| |
| <!DOCTYPE html> |
| <html lang="en" dir=ZgotmplZ> |
| |
| <head> |
| |
| |
| |
| <link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css"> |
| <script src="/bootstrap/js/bootstrap.bundle.min.js"></script> |
| <link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css"> |
| <script src="/js/anchor.min.js"></script> |
| <script src="/js/flink.js"></script> |
| <link rel="canonical" href="https://flink.apache.org/2022/01/07/apache-flink-ml-2.0.0-release-announcement/"> |
| |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| <meta name="description" content="The Apache Flink community is excited to announce the release of Flink ML 2.0.0! Flink ML is a library that provides APIs and infrastructure for building stream-batch unified machine learning algorithms, that can be easy-to-use and performant with (near-) real-time latency. |
| This release involves a major refactor of the earlier Flink ML library and introduces major features that extend the Flink ML API and the iteration runtime, such as supporting stages with multi-input multi-output, graph-based stage composition, and a new stream-batch unified iteration library."> |
| <meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="Apache Flink ML 2.0.0 Release Announcement" /> |
| <meta property="og:description" content="The Apache Flink community is excited to announce the release of Flink ML 2.0.0! Flink ML is a library that provides APIs and infrastructure for building stream-batch unified machine learning algorithms, that can be easy-to-use and performant with (near-) real-time latency. |
| This release involves a major refactor of the earlier Flink ML library and introduces major features that extend the Flink ML API and the iteration runtime, such as supporting stages with multi-input multi-output, graph-based stage composition, and a new stream-batch unified iteration library." /> |
| <meta property="og:type" content="article" /> |
| <meta property="og:url" content="https://flink.apache.org/2022/01/07/apache-flink-ml-2.0.0-release-announcement/" /><meta property="article:section" content="posts" /> |
| <meta property="article:published_time" content="2022-01-07T08:00:00+00:00" /> |
| <meta property="article:modified_time" content="2022-01-07T08:00:00+00:00" /> |
| <title>Apache Flink ML 2.0.0 Release Announcement | Apache Flink</title> |
| <link rel="manifest" href="/manifest.json"> |
| <link rel="icon" href="/favicon.png" type="image/x-icon"> |
| <link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU="> |
| <script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script> |
| <!-- |
| Made with Book Theme |
| https://github.com/alex-shpak/hugo-book |
| --> |
| |
| <meta name="generator" content="Hugo 0.124.1"> |
| |
| |
| <script> |
| var _paq = window._paq = window._paq || []; |
| |
| |
| _paq.push(['disableCookies']); |
| |
| _paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="//analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '1']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| |
| </head> |
| |
| <body dir=ZgotmplZ> |
| |
| |
| |
| <header> |
| <nav class="navbar navbar-expand-xl"> |
| <div class="container-fluid"> |
| <a class="navbar-brand" href="/"> |
| <img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle"> |
| <span>Apache Flink</span> |
| </a> |
| <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation"> |
| <i class="fa fa-bars navbar-toggler-icon"></i> |
| </button> |
| <div class="collapse navbar-collapse" id="navbarSupportedContent"> |
| <ul class="navbar-nav"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="nav-item dropdown"> |
| <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a> |
| <ul class="dropdown-menu"> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/security/">Security</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a> |
| |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| <li class="nav-item dropdown"> |
| <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a> |
| <ul class="dropdown-menu"> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| <li class="nav-item dropdown"> |
| <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a> |
| <ul class="dropdown-menu"> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| <li class="nav-item dropdown"> |
| <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a> |
| <ul class="dropdown-menu"> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a> |
| |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="nav-item"> |
| |
| |
| <a class="nav-link" href="/posts/">Flink Blog</a> |
| |
| |
| </li> |
| |
| |
| |
| |
| |
| <li class="nav-item"> |
| |
| |
| <a class="nav-link" href="/downloads/">Downloads</a> |
| |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </ul> |
| <div class="book-search"> |
| <div class="book-search-spinner hidden"> |
| <i class="fa fa-refresh fa-spin"></i> |
| </div> |
| <form class="search-bar d-flex" onsubmit="return false;"su> |
| <input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/"> |
| <i class="fa fa-search search"></i> |
| <i class="fa fa-circle-o-notch fa-spin spinner"></i> |
| </form> |
| <div class="book-search-spinner hidden"></div> |
| <ul id="book-search-results"></ul> |
| </div> |
| </div> |
| </div> |
| </nav> |
| <div class="navbar-clearfix"></div> |
| </header> |
| |
| |
| <main class="flex"> |
| <section class="container book-page"> |
| |
| <article class="markdown"> |
| <h1> |
| <a href="/2022/01/07/apache-flink-ml-2.0.0-release-announcement/">Apache Flink ML 2.0.0 Release Announcement</a> |
| </h1> |
| |
| |
| |
| January 7, 2022 - |
| |
| |
| |
| Dong Lin |
| |
| |
| Yun Gao |
| |
| |
| |
| |
| <p><p>The Apache Flink community is excited to announce the release of Flink ML |
| 2.0.0! Flink ML is a library that provides APIs and infrastructure for building |
| stream-batch unified machine learning algorithms, that can be easy-to-use and |
| performant with (near-) real-time latency.</p> |
| <p>This release involves a major refactor of the earlier Flink ML library and |
| introduces major features that extend the Flink ML API and the iteration |
| runtime, such as supporting stages with multi-input multi-output, graph-based |
| stage composition, and a new stream-batch unified iteration library. Moreover, |
| we added five algorithm implementations in this release, which is the start of |
| a long-term initiative to provide a large number of off-the-shelf algorithms in |
| Flink ML with state-of-the-art performance.</p> |
| <p>We believe this release is an important step towards extending Apache Flink to |
| a wide range of machine learning use cases, especially the real-time machine |
| learning scenarios.</p> |
| <p>We encourage you to <a href="https://flink.apache.org/downloads.html">download the release</a> and share your feedback with |
| the community through the Flink <a href="https://flink.apache.org/community.html#mailing-lists">mailing lists</a> or |
| <a href="https://issues.apache.org/jira/browse/flink">JIRA</a>! We hope you like the new |
| release and we’d be eager to learn about your experience with it.</p> |
| <h1 id="notable-features"> |
| Notable Features |
| <a class="anchor" href="#notable-features">#</a> |
| </h1> |
| <h2 id="api-and-infrastructure"> |
| API and Infrastructure |
| <a class="anchor" href="#api-and-infrastructure">#</a> |
| </h2> |
| <h3 id="supporting-stages-requiring-multi-input-multi-output"> |
| Supporting stages requiring multi-input multi-output |
| <a class="anchor" href="#supporting-stages-requiring-multi-input-multi-output">#</a> |
| </h3> |
| <p>Stages in a machine learning workflow might take multiple inputs and return |
| multiple outputs. For example, a graph embedding algorithm might need to read |
| two tables, which represent the edge and node of the graph respectively. And a |
| workflow might need a stage that splits the input dataset into two output |
| datasets, for training and testing respectively.</p> |
| <p>With this capability, algorithm developers can assemble a machine learning |
| workflow as a directed acyclic graph (DAG) of pre-defined stages. And this |
| workflow can be configured and deployed without users knowing the |
| implementation details of this graph. This improvement could considerably |
| expand the applicability and usability of Flink ML.</p> |
| <h3 id="supporting-online-learning-with-apis-exposing-model-data"> |
| Supporting online learning with APIs exposing model data |
| <a class="anchor" href="#supporting-online-learning-with-apis-exposing-model-data">#</a> |
| </h3> |
| <p>In a native online learning scenario, we have a long-running job that keeps |
| processing training data and updating a machine learning model. And we could |
| have multiple jobs deployed in web servers which do online inference. It is |
| necessary to transmit the latest model data from the training job to those |
| inference jobs in (near-) real-time latency.</p> |
| <p>The traditional Estimator/Transformer paradigm does not provide APIs to expose |
| this model data in a streaming manner. Users have to repeatedly call fit() to |
| update model data. Although users might be able to update model data once every |
| few minutes, it is likely very inefficient, if not impossible, to update model |
| data once every few seconds with this approach.</p> |
| <p>With |
| <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=184615783">FLIP-173</a>, |
| model data can be exposed as an unbounded stream via the getModelData() API. |
| Then algorithm users can transfer the model data to web servers in real-time |
| and use the up-to-date model data to do online inference. This feature could |
| significantly strengthen Flink ML’s capability to support online learning |
| applications.</p> |
| <h3 id="improved-usability-for-managing-parameters"> |
| Improved usability for managing parameters |
| <a class="anchor" href="#improved-usability-for-managing-parameters">#</a> |
| </h3> |
| <p>We care a lot about usability and developer velocity in Flink ML. In this |
| release, we refactored and significantly simplified the experience of defining, |
| getting and setting parameters for algorithms.</p> |
| <p>With |
| <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=181311361">FLIP-174</a>, |
| parameters can be defined as static variables of an interface, and any |
| algorithm that implements the interface could inherit these variable |
| definitions without additional work. Commonly used parameter validators are |
| provided as part of the infrastructure.</p> |
| <h3 id="tools-for-composing-dag-of-stages-into-a-new-stage"> |
| Tools for composing DAG of stages into a new stage |
| <a class="anchor" href="#tools-for-composing-dag-of-stages-into-a-new-stage">#</a> |
| </h3> |
| <p>One of the most useful tool in the existing ML libraries (e.g. Scikit-learn, |
| Flink, Spark) is |
| <a href="https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html">Pipeline</a>, |
| which allows users to compose an estimator from an ordered list of estimators |
| and transformers, without having to explicitly implement the fit/transform for |
| the estimator/transformer.</p> |
| <p><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=181311363">FLIP-175</a> |
| extended this capability from pipeline to DAG. Users can now compose an |
| estimator from a DAG of estimator and transformers. This capability of |
| composition allows developers to slice a complex workflow into simpler modules |
| and re-use the modules across multiple workflows. We believe this capability |
| could significantly improve the experience of building and deploying complex |
| workflows using Flink ML.</p> |
| <h2 id="stream-batch-unified-iteration-library"> |
| Stream-batch Unified Iteration Library |
| <a class="anchor" href="#stream-batch-unified-iteration-library">#</a> |
| </h2> |
| <p>To support training machine learning algorithms and adjust the model parameters |
| dynamically based on the prediction result, it is necessary to have native |
| support for processing data iteratively. It is known that Flink uses DAG to |
| describe the process logic, thus we need to provide the iteration library on |
| top of Flink separately. Besides, since we need to support both offline |
| training and online training / adjustment, the iteration library should support |
| both streaming and batch cases.</p> |
| <p><a href="https://cwiki.apache.org/confluence/x/hAEBCw">FLIP-176</a> implements a |
| stream-batch unified iteration library. It provides the function of |
| transmitting records back to the precedent operators and the ability to track |
| the progress of rounds inside the iteration. Users could directly use |
| DataStream API and Table API to express the execution logic inside the |
| iteration. Besides, the new iteration library also extends Flink’s |
| checkpointing mechanism to also support exactly-once failover for jobs using |
| iterations.</p> |
| <h2 id="python-sdk"> |
| Python SDK |
| <a class="anchor" href="#python-sdk">#</a> |
| </h2> |
| <p>Nowadays many machine learning practitioners are used to developing machine |
| learning workflows in Python due to its ease-of-use and excellent ecosystem. To |
| meet the needs of these users, a Python package dedicated for Flink ML is |
| created starting from this release. The Python package currently provides APIs |
| similar to their Java counterparts for developing machine learning algorithms.</p> |
| <p>Users can install Flink ML Python package through pip using the following |
| command:</p> |
| <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"><span class="line"><span class="cl">pip install apache-flink-ml |
| </span></span></code></pre></div><p>In the future we will enhance the Python SDK to enable its interoperability |
| with Flink ML’s Java library, for example, allowing users to express machine |
| learning workflows in Python, where workflows consist of a mixture of stages |
| from the Flink ML Java library as well as stages implemented in Python (e.g. a |
| TensorFlow program).</p> |
| <h2 id="algorithm-library"> |
| Algorithm Library |
| <a class="anchor" href="#algorithm-library">#</a> |
| </h2> |
| <p>Now that the Flink ML API re-design is done, we started the initiative to add |
| off-the-shelf algorithms in Flink ML. The release of Flink-ML 2.0.0 is closely |
| related to project Alink - an Apache Flink ecosystem project open sourced by |
| Alibaba. The connection between the Flink community and developers of the Alink |
| project dates back to 2017. The project Alink developers have a significant |
| contribution in designing the new Flink ML APIs, refactoring, optimizing and |
| migrating algorithms from Alink to Flink. Our long-term goal is to provide a |
| library of performant algorithms that are easy to use, debug and customize for |
| your needs.</p> |
| <p>We have implemented five algorithms in this release, i.e. logistic regression, |
| k-means, k-nearest neighbors, naive bayes and one-hot encoder. For now these |
| algorithms focus on validating the APIs and iteration runtime. In addition to |
| adding more and more algorithms, we will also stress test and optimize their |
| performance to make sure these algorithms have state-of-the-art performance. |
| Stay tuned!</p> |
| <h1 id="related-work"> |
| Related Work |
| <a class="anchor" href="#related-work">#</a> |
| </h1> |
| <h2 id="flink-ml-project-moved-to-a-separate-repository"> |
| Flink ML project moved to a separate repository |
| <a class="anchor" href="#flink-ml-project-moved-to-a-separate-repository">#</a> |
| </h2> |
| <p>To accelerate the development of Flink ML, the effort has moved to the new |
| repository <a href="https://github.com/apache/flink-ml">flink-ml</a> under the Flink |
| project. We here follow a similar approach like the Stateful Functions effort, |
| where a separate repository has helped to speed up the development by allowing |
| for more light-weight contribution workflows and separate release cycles.</p> |
| <h2 id="github-organization-created-for-flink-ecosystem-projects"> |
| Github organization created for Flink ecosystem projects |
| <a class="anchor" href="#github-organization-created-for-flink-ecosystem-projects">#</a> |
| </h2> |
| <p>To facilitate the community collaboration on ecosystem projects that extend the |
| capability of the Apache Flink, Apache Flink PMC has granted the permission to |
| use flink-extended as the name of this <a href="https://github.com/flink-extended">GitHub |
| organization</a>, which provides a neutral |
| place to host the code of ecosystem projects.</p> |
| <p>Two Flink ML related projects have been moved to this organization. |
| <a href="https://github.com/flink-extended/dl-on-flink">dl-on-flink</a> provides the |
| capability to implement Flink ML stages using TensorFlow. And |
| <a href="https://github.com/flink-extended/clink">clink</a> is a library that facilitates |
| the implementation of Flink ML stages using C++ in order to support e.g. |
| real-time feature engineering.</p> |
| <p>We hope you can join this effort and share your Flink ecosystem projects in |
| this Github organization. And stay tuned for more updates on ecosystem |
| projects.</p> |
| <h1 id="upgrade-notes"> |
| Upgrade Notes |
| <a class="anchor" href="#upgrade-notes">#</a> |
| </h1> |
| <p>Please review this note for a list of adjustments to make and issues to check |
| if you plan to upgrade to Flink ML 2.0.0.</p> |
| <p>This note discusses any critical information about incompatibilities and |
| breaking changes, performance changes, and any other changes that might impact |
| your production deployment of Flink ML.</p> |
| <ul> |
| <li> |
| <p><strong>Module names are changed</strong>.</p> |
| <p>We have replaced the <code>flink-ml-api</code> module with the <code>flink-ml-core_2.12</code> |
| module.</p> |
| <p>For users who have a dependency on <code>flink-ml-api</code>, please replace it with |
| <code>flink-ml-core_2.12</code></p> |
| </li> |
| <li> |
| <p><strong>PipelineStage and its subclasses are changed</strong>.</p> |
| <p><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=184615783">FLIP-173</a> |
| made major changes to PipelineStage and its subclasses. Changes include class |
| rename, method signature change, method removal etc.</p> |
| <p>Users who use PipelineStage and its subclasses should use the new APIs |
| introduced in FLIP-173.</p> |
| </li> |
| <li> |
| <p><strong>Param-related classes are changed</strong>.</p> |
| <p><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=181311361">FLIP-174</a> |
| made major changes to the param-related classes. Changes include class rename, |
| method signature change, method removal etc.</p> |
| <p>Users who use classes such as Params and WithParams should use the new APIs |
| introduced in FLIP-174.</p> |
| </li> |
| <li> |
| <p><strong>Flink dependency is changed from 1.12 to 1.14</strong>.</p> |
| <p>This change introduces all the breaking changes listed in the Flink 1.14 |
| <a href="https://nightlies.apache.org/flink/flink-docs-release-1.14/release-notes/flink-1.14">release notes</a>. |
| One major change is that the DataSet API is not supported anymore.</p> |
| <p>Users who use DataSet::iterate should switch to using the datastream-based |
| iteration API introduced in <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=184615300">FLIP-176</a>.</p> |
| </li> |
| </ul> |
| <h1 id="release-notes-and-resources"> |
| Release Notes and Resources |
| <a class="anchor" href="#release-notes-and-resources">#</a> |
| </h1> |
| <p>Please take a look at the <a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12315522&version=12351079">release notes</a> |
| for a detailed list of changes and new features.</p> |
| <p>The binary distribution and source artifacts are now available on the updated |
| <a href="https://flink.apache.org/downloads.html">Downloads page</a> of the Flink website, |
| and the most recent distribution of Flink ML Python package is available on |
| <a href="https://pypi.org/project/apache-flink-ml">PyPI</a>.</p> |
| <h1 id="list-of-contributors"> |
| List of Contributors |
| <a class="anchor" href="#list-of-contributors">#</a> |
| </h1> |
| <p>The Apache Flink community would like to thank each one of the contributors |
| that have made this release possible:</p> |
| <p>Yun Gao, Dong Lin, Zhipeng Zhang, huangxingbo, Yunfeng Zhou, Jiangjie (Becket) |
| Qin, weibo, abdelrahman-ik.</p> |
| </p> |
| </article> |
| |
| |
| |
| |
| |
| |
| |
| <div class="edit-this-page"> |
| <p> |
| <a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a> |
| </p> |
| <p> |
| <a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2022-01-07-release-ml-2.0.0.md"> |
| Edit This Page<i class="fa fa-edit fa-fw"></i> |
| </a> |
| </p> |
| </div> |
| |
| </section> |
| |
| <aside class="book-toc"> |
| |
| |
| |
| <nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3> |
| <ul> |
| <li><a href="#notable-features">Notable Features</a> |
| <ul> |
| <li><a href="#api-and-infrastructure">API and Infrastructure</a> |
| <ul> |
| <li><a href="#supporting-stages-requiring-multi-input-multi-output">Supporting stages requiring multi-input multi-output</a></li> |
| <li><a href="#supporting-online-learning-with-apis-exposing-model-data">Supporting online learning with APIs exposing model data</a></li> |
| <li><a href="#improved-usability-for-managing-parameters">Improved usability for managing parameters</a></li> |
| <li><a href="#tools-for-composing-dag-of-stages-into-a-new-stage">Tools for composing DAG of stages into a new stage</a></li> |
| </ul> |
| </li> |
| <li><a href="#stream-batch-unified-iteration-library">Stream-batch Unified Iteration Library</a></li> |
| <li><a href="#python-sdk">Python SDK</a></li> |
| <li><a href="#algorithm-library">Algorithm Library</a></li> |
| </ul> |
| </li> |
| <li><a href="#related-work">Related Work</a> |
| <ul> |
| <li><a href="#flink-ml-project-moved-to-a-separate-repository">Flink ML project moved to a separate repository</a></li> |
| <li><a href="#github-organization-created-for-flink-ecosystem-projects">Github organization created for Flink ecosystem projects</a></li> |
| </ul> |
| </li> |
| <li><a href="#upgrade-notes">Upgrade Notes</a></li> |
| <li><a href="#release-notes-and-resources">Release Notes and Resources</a></li> |
| <li><a href="#list-of-contributors">List of Contributors</a></li> |
| </ul> |
| </nav> |
| |
| |
| </aside> |
| <aside class="expand-toc hidden"> |
| <a class="toc" onclick="expandToc()" href="javascript:void(0)"> |
| <i class="fa fa-bars" aria-hidden="true"></i> |
| </a> |
| </aside> |
| |
| </main> |
| |
| <footer> |
| |
| |
| |
| <div class="separator"></div> |
| <div class="panels"> |
| <div class="wrapper"> |
| <div class="panel"> |
| <ul> |
| <li> |
| <a href="https://flink-packages.org/">flink-packages.org</a> |
| </li> |
| <li> |
| <a href="https://www.apache.org/">Apache Software Foundation</a> |
| </li> |
| <li> |
| <a href="https://www.apache.org/licenses/">License</a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li> |
| <a href="/zh/"> |
| <i class="fa fa-globe" aria-hidden="true"></i> 中文版 |
| </a> |
| </li> |
| |
| |
| |
| </ul> |
| </div> |
| <div class="panel"> |
| <ul> |
| <li> |
| <a href="/what-is-flink/security">Security</a--> |
| </li> |
| <li> |
| <a href="https://www.apache.org/foundation/sponsorship.html">Donate</a> |
| </li> |
| <li> |
| <a href="https://www.apache.org/foundation/thanks.html">Thanks</a> |
| </li> |
| </ul> |
| </div> |
| <div class="panel icons"> |
| <div> |
| <a href="/posts"> |
| <div class="icon flink-blog-icon"></div> |
| <span>Flink blog</span> |
| </a> |
| </div> |
| <div> |
| <a href="https://github.com/apache/flink"> |
| <div class="icon flink-github-icon"></div> |
| <span>Github</span> |
| </a> |
| </div> |
| <div> |
| <a href="https://twitter.com/apacheflink"> |
| <div class="icon flink-twitter-icon"></div> |
| <span>Twitter</span> |
| </a> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| <hr/> |
| |
| <div class="container disclaimer"> |
| <p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p> |
| </div> |
| |
| |
| |
| </footer> |
| |
| </body> |
| </html> |
| |
| |
| |
| |
| |
| |
