| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> |
| <title>Apache Flink: Apache Flink 1.11.0 Release Announcement</title> |
| <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> |
| <link rel="icon" href="/favicon.ico" type="image/x-icon"> |
| |
| <!-- Bootstrap --> |
| <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css"> |
| <link rel="stylesheet" href="/css/flink.css"> |
| <link rel="stylesheet" href="/css/syntax.css"> |
| |
| <!-- Blog RSS feed --> |
| <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> |
| |
| <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> |
| <!-- We need to load Jquery in the header for custom google analytics event tracking--> |
| <script src="/js/jquery.min.js"></script> |
| |
| <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> |
| <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> |
| <!--[if lt IE 9]> |
| <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script> |
| <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script> |
| <![endif]--> |
| </head> |
| <body> |
| |
| |
| <!-- Main content. --> |
| <div class="container"> |
| <div class="row"> |
| |
| |
| <div id="sidebar" class="col-sm-3"> |
| |
| |
| <!-- Top navbar. --> |
| <nav class="navbar navbar-default"> |
| <!-- The logo. --> |
| <div class="navbar-header"> |
| <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| </button> |
| <div class="navbar-logo"> |
| <a href="/"> |
| <img alt="Apache Flink" src="/img/flink-header-logo.svg" width="147px" height="73px"> |
| </a> |
| </div> |
| </div><!-- /.navbar-header --> |
| |
| <!-- The navigation links. --> |
| <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> |
| <ul class="nav navbar-nav navbar-main"> |
| |
| <!-- First menu section explains visitors what Flink is --> |
| |
| <!-- What is Stream Processing? --> |
| <!-- |
| <li><a href="/streamprocessing1.html">What is Stream Processing?</a></li> |
| --> |
| |
| <!-- What is Flink? --> |
| <li><a href="/flink-architecture.html">What is Apache Flink?</a></li> |
| |
| |
| |
| <!-- What is Stateful Functions? --> |
| |
| <li><a href="/stateful-functions.html">What is Stateful Functions?</a></li> |
| |
| <!-- Use cases --> |
| <li><a href="/usecases.html">Use Cases</a></li> |
| |
| <!-- Powered by --> |
| <li><a href="/poweredby.html">Powered By</a></li> |
| |
| |
| |
| <!-- Second menu section aims to support Flink users --> |
| |
| <!-- Downloads --> |
| <li><a href="/downloads.html">Downloads</a></li> |
| |
| <!-- Getting Started --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Getting Started<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/getting-started/index.html" target="_blank">With Flink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1/getting-started/project-setup.html" target="_blank">With Flink Stateful Functions <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="/training.html">Training Course</a></li> |
| </ul> |
| </li> |
| |
| <!-- Documentation --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11" target="_blank">Flink 1.11 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-master" target="_blank">Flink Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1" target="_blank">Flink Stateful Functions 2.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-master" target="_blank">Flink Stateful Functions Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| </ul> |
| </li> |
| |
| <!-- getting help --> |
| <li><a href="/gettinghelp.html">Getting Help</a></li> |
| |
| <!-- Blog --> |
| <li class="active"><a href="/blog/"><b>Flink Blog</b></a></li> |
| |
| |
| <!-- Flink-packages --> |
| <li> |
| <a href="https://flink-packages.org" target="_blank">flink-packages.org <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| <!-- Third menu section aim to support community and contributors --> |
| |
| <!-- Community --> |
| <li><a href="/community.html">Community & Project Info</a></li> |
| |
| <!-- Roadmap --> |
| <li><a href="/roadmap.html">Roadmap</a></li> |
| |
| <!-- Contribute --> |
| <li><a href="/contributing/how-to-contribute.html">How to Contribute</a></li> |
| |
| |
| <!-- GitHub --> |
| <li> |
| <a href="https://github.com/apache/flink" target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| |
| <!-- Language Switcher --> |
| <li> |
| |
| |
| <!-- link to the Chinese home page when current is blog page --> |
| <a href="/zh">中文版</a> |
| |
| |
| </li> |
| |
| </ul> |
| |
| <ul class="nav navbar-nav navbar-bottom"> |
| <hr /> |
| |
| <!-- Twitter --> |
| <li><a href="https://twitter.com/apacheflink" target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <!-- Visualizer --> |
| <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <hr /> |
| |
| <li><a href="https://apache.org" target="_blank">Apache Software Foundation <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <li> |
| <style> |
| .smalllinks:link { |
| display: inline-block !important; background: none; padding-top: 0px; padding-bottom: 0px; padding-right: 0px; min-width: 75px; |
| } |
| </style> |
| |
| <a class="smalllinks" href="https://www.apache.org/licenses/" target="_blank">License</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/security/" target="_blank">Security</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/sponsorship.html" target="_blank">Donate</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| </li> |
| |
| </ul> |
| </div><!-- /.navbar-collapse --> |
| </nav> |
| |
| </div> |
| <div class="col-sm-9"> |
| <div class="row-fluid"> |
| <div class="col-sm-12"> |
| <div class="row"> |
| <h1>Apache Flink 1.11.0 Release Announcement</h1> |
| <p><i></i></p> |
| |
| <article> |
| <p>06 Jul 2020 Marta Paes (<a href="https://twitter.com/morsapaes">@morsapaes</a>)</p> |
| |
| <p>The Apache Flink community is proud to announce the release of Flink 1.11.0! More than 200 contributors worked on over 1.3k issues to bring significant improvements to usability as well as new features to Flink users across the whole API stack. Some highlights that we’re particularly excited about are:</p> |
| |
| <ul> |
| <li> |
| <p>The core engine is introducing <strong>unaligned checkpoints</strong>, a major change to Flink’s fault tolerance mechanism that improves checkpointing performance under heavy backpressure.</p> |
| </li> |
| <li> |
| <p>A <strong>new Source API</strong> that simplifies the implementation of (custom) sources by unifying batch and streaming execution, as well as offloading internals such as event-time handling, watermark generation or idleness detection to Flink.</p> |
| </li> |
| <li> |
| <p>Flink SQL is introducing <strong>Support for Change Data Capture (CDC)</strong> to easily consume and interpret database changelogs from tools like Debezium. The renewed <strong>FileSystem Connector</strong> also expands the set of use cases and formats supported in the Table API/SQL, enabling scenarios like streaming data directly from Kafka to Hive.</p> |
| </li> |
| <li> |
| <p>Multiple performance optimizations to PyFlink, including support for <strong>vectorized User-defined Functions (Pandas UDFs)</strong>. This improves interoperability with libraries like Pandas and NumPy, making Flink more powerful for data science and ML workloads.</p> |
| </li> |
| </ul> |
| |
| <p>Read on for all major new features and improvements, important changes to be aware of and what to expect moving forward!</p> |
| |
| <div class="page-toc"> |
| <ul id="markdown-toc"> |
| <li><a href="#new-features-and-improvements" id="markdown-toc-new-features-and-improvements">New Features and Improvements</a> <ul> |
| <li><a href="#unaligned-checkpoints-beta" id="markdown-toc-unaligned-checkpoints-beta">Unaligned Checkpoints (Beta)</a></li> |
| <li><a href="#unified-watermark-generators" id="markdown-toc-unified-watermark-generators">Unified Watermark Generators</a></li> |
| <li><a href="#new-data-source-api-beta" id="markdown-toc-new-data-source-api-beta">New Data Source API (Beta)</a></li> |
| <li><a href="#application-mode-deployments" id="markdown-toc-application-mode-deployments">Application Mode Deployments</a></li> |
| <li><a href="#other-improvements" id="markdown-toc-other-improvements">Other Improvements</a></li> |
| <li><a href="#table-apisql-support-for-change-data-capture-cdc" id="markdown-toc-table-apisql-support-for-change-data-capture-cdc">Table API/SQL: Support for Change Data Capture (CDC)</a></li> |
| <li><a href="#table-apisql-jdbc-catalog-interface-and-postgres-catalog" id="markdown-toc-table-apisql-jdbc-catalog-interface-and-postgres-catalog">Table API/SQL: JDBC Catalog Interface and Postgres Catalog</a></li> |
| <li><a href="#table-apisql-filesystem-connector-with-support-for-avro-orc-and-parquet" id="markdown-toc-table-apisql-filesystem-connector-with-support-for-avro-orc-and-parquet">Table API/SQL: FileSystem Connector with Support for Avro, ORC and Parquet</a></li> |
| <li><a href="#table-apisql-support-for-python-udfs" id="markdown-toc-table-apisql-support-for-python-udfs">Table API/SQL: Support for Python UDFs</a></li> |
| <li><a href="#other-improvements-to-the-table-apisql" id="markdown-toc-other-improvements-to-the-table-apisql">Other Improvements to the Table API/SQL</a></li> |
| <li><a href="#pyflink-support-for-pandas-udfs" id="markdown-toc-pyflink-support-for-pandas-udfs">PyFlink: Support for Pandas UDFs</a></li> |
| <li><a href="#other-improvements-to-pyflink" id="markdown-toc-other-improvements-to-pyflink">Other Improvements to PyFlink</a></li> |
| </ul> |
| </li> |
| <li><a href="#important-changes" id="markdown-toc-important-changes">Important Changes</a></li> |
| <li><a href="#release-notes" id="markdown-toc-release-notes">Release Notes</a></li> |
| <li><a href="#list-of-contributors" id="markdown-toc-list-of-contributors">List of Contributors</a></li> |
| </ul> |
| |
| </div> |
| |
| <p>The binary distribution and source artifacts are now available on the updated <a href="/downloads.html">Downloads page</a> of the Flink website, and the most recent distribution of PyFlink is available on <a href="https://pypi.org/project/apache-flink/">PyPI</a>. Please review the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/release-notes/flink-1.11.html">release notes</a> carefully, and check the complete <a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12346364&styleName=Html&projectId=12315522">release changelog</a> and <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/flink-docs-release-1.11/">updated documentation</a> for more details.</p> |
| |
| <p>We encourage you to download the release and share your feedback with the community through the <a href="https://flink.apache.org/community.html#mailing-lists">Flink mailing lists</a> or <a href="https://issues.apache.org/jira/projects/FLINK/summary">JIRA</a>.</p> |
| |
| <h2 id="new-features-and-improvements">New Features and Improvements</h2> |
| |
| <h3 id="unaligned-checkpoints-beta">Unaligned Checkpoints (Beta)</h3> |
| |
| <p>Triggering a checkpoint in Flink will cause a <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/internals/stream_checkpointing.html#barriers">checkpoint barrier</a> to flow from the sources of your topology all the way towards the sinks. For operators that receive more than one input stream, the barriers flowing through each channel need to be aligned before the operator can snapshot its state and forward the checkpoint barrier — typically, this alignment will take just a few milliseconds to complete, but it can become a bottleneck in backpressured pipelines as:</p> |
| |
| <ul> |
| <li> |
| <p>Checkpoint barriers will flow much slower through backpressured channels, effectively blocking the remaining channels and their upstream operators during checkpointing;</p> |
| </li> |
| <li> |
| <p>Slow checkpoint barrier propagation leads to longer checkpointing times and can, worst case, result in little to no progress in the application.</p> |
| </li> |
| </ul> |
| |
| <p>To improve the performance of checkpointing under backpressure scenarios, the community is rolling out the first iteration of unaligned checkpoints (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-76%3A+Unaligned+Checkpoints">FLIP-76</a>) with Flink 1.11. Compared to the original checkpointing mechanism (Fig. 1), this approach doesn’t wait for barrier alignment across input channels, instead allowing barriers to overtake in-flight records (i.e., data stored in buffers) and forwarding them downstream before the synchronous part of the checkpoint takes place (Fig. 2).</p> |
| |
| <div style="line-height:60%;"> |
| <br /> |
| </div> |
| |
| <div class="row"> |
| <div class="col-lg-6"> |
| <div class="text-center"> |
| <figure> |
| <img src="/img/blog/2020-07-06-release-1.11.0/image1.gif" width="600px" alt="Aligned Checkpoints" /> |
| <br /><br /> |
| <figcaption><i><b>Fig.1:</b> Aligned Checkpoints</i></figcaption> |
| </figure> |
| </div> |
| </div> |
| <div class="col-lg-6"> |
| <div class="text-center"> |
| <figure> |
| <img src="/img/blog/2020-07-06-release-1.11.0/image2.png" width="600px" alt="Unaligned Checkpoints" /> |
| <br /><br /> |
| <figcaption><i><b>Fig.2:</b> Unaligned Checkpoints</i></figcaption> |
| </figure> |
| </div> |
| </div> |
| </div> |
| |
| <div style="line-height:150%;"> |
| <br /> |
| </div> |
| |
| <p>Because in-flight records have to be persisted as part of the snapshot, unaligned checkpoints will lead to increased checkpoints sizes. On the upside, <strong>checkpointing times are heavily reduced</strong>, so users will see more progress (even in unstable environments) as more up-to-date checkpoints will lighten the recovery process. You can learn more about the current limitations of unaligned checkpoints in the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/ops/state/checkpoints.html#unaligned-checkpoints">documentation</a>, and track the improvement work planned for this feature in <a href="https://issues.apache.org/jira/browse/FLINK-14551">FLINK-14551</a>.</p> |
| |
| <p>As with any beta feature, we appreciate early feedback that you might want to share with the community after giving unaligned checkpoints a try!</p> |
| |
| <p><span class="label label-info">Info</span> To enable this feature, you need to configure the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/streaming/api/environment/CheckpointConfig.html"><code>enableUnalignedCheckpoints</code></a> option in your <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/stream/state/checkpointing.html#enabling-and-configuring-checkpointing">checkpoint config</a>. Please note that unaligned checkpoints can only be enabled if <code>checkpointingMode</code> is set to <code>CheckpointingMode.EXACTLY_ONCE</code>.</p> |
| |
| <h3 id="unified-watermark-generators">Unified Watermark Generators</h3> |
| |
| <p>So far, watermark generation (prev. also called <em>assignment</em>) relied on two different interfaces: <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/streaming/api/functions/AssignerWithPunctuatedWatermarks.html"><code>AssignerWithPunctuatedWatermarks</code></a> and <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/streaming/api/functions/AssignerWithPeriodicWatermarks.html"><code>AssignerWithPeriodicWatermarks</code></a>; that were closely intertwined with timestamp extraction. This made it difficult to implement long-requested features like support for idleness detection, besides leading to code duplication and maintenance burden. With <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-126%3A+Unify+%28and+separate%29+Watermark+Assigners">FLIP-126</a>, the legacy watermark assigners are unified into a single interface: the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/api/common/eventtime/WatermarkGenerator.html"><code>WatermarkGenerator</code></a>; and detached from the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/api/common/eventtime/TimestampAssigner.html"><code>TimestampAssigner</code></a>.</p> |
| |
| <p>This gives users more control over watermark emission and simplifies the implementation of new connectors that need to support watermark assignment and timestamp extraction at the source (see <em><a href="#new-data-source-api-beta">New Data Source API</a></em>). Multiple <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11//dev/event_timestamps_watermarks.html#introduction-to-watermark-strategies">strategies for watermarking</a> are available out-of-the-box as convenience methods in Flink 1.11 (e.g. <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/api/common/eventtime/WatermarkStrategy.html#forBoundedOutOfOrderness-java.time.Duration-"><code>forBoundedOutOfOrderness</code></a>, <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/api/common/eventtime/WatermarkStrategy.html#forMonotonousTimestamps--"><code>forMonotonousTimestamps</code></a>), though you can also choose to customize your own.</p> |
| |
| <p><strong>Support for Watermark Idleness Detection</strong></p> |
| |
| <p>The <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/api/common/eventtime/WatermarkStrategy.html#withIdleness-java.time.Duration-"><code>WatermarkStrategy.withIdleness()</code></a> method allows you to mark a stream as idle if no events arrive within a configured time (i.e. a timeout duration), which in turn allows handling event time skew properly and preventing idle partitions from holding back the event time progress of the entire application. Users can already benefit from <strong>per-partition idleness detection</strong> in the Kafka connector, which has been adapted to use the new interfaces (<a href="https://issues.apache.org/jira/browse/FLINK-17669">FLINK-17669</a>).</p> |
| |
| <p><span class="label label-info">Note</span> <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-126%3A+Unify+%28and+separate%29+Watermark+Assigners">FLIP-126</a> introduces no breaking changes, but we recommend that users give preference to the new <code>WatermarkGenerator</code> interface moving forward, in preparation for the deprecation of the legacy watermark assigners in future releases.</p> |
| |
| <h3 id="new-data-source-api-beta">New Data Source API (Beta)</h3> |
| |
| <p>Up to this point, writing a production-grade source connector for Flink was a non-trivial task that required users to be somewhat familiar with Flink internals and account for implementation details like event time assignment, watermark generation or idleness detection in their code. Flink 1.11 introduces a new Data Source API (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-27%3A+Refactor+Source+Interface">FLIP-27</a>) to overcome these limitations, as well as the need to rewrite separate code for batch and streaming execution.</p> |
| |
| <center> |
| <figure> |
| <img src="/img/blog/2020-07-06-release-1.11.0/image3.png" width="600px" alt="Data Source API" /> |
| </figure> |
| </center> |
| |
| <div style="line-height:150%;"> |
| <br /> |
| </div> |
| |
| <p>Separating the work of split discovery and the actual reading of the consumed data (i.e. the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/stream/sources.html#data-source-concepts"><em>splits</em></a>) in different components — resp. the <code>SplitEnumerator</code> and <code>SourceReader</code> — allows mixing and matching different enumeration strategies and split readers.</p> |
| |
| <p>As an example, the existing Kafka connector has multiple strategies for partition discovery that are intermingled with the rest of the code. With the new interfaces in place, it would only need a single reader implementation and there could be several split enumerators for the different partition discovery strategies.</p> |
| |
| <p><strong>Batch and Streaming Unification</strong></p> |
| |
| <p>Source connectors implemented using the Data Source API will be able to work both as a bounded (<em>batch</em>) and unbounded (<em>streaming</em>) source. The difference between both cases is minimal: for bounded input, the <code>SplitEnumerator</code> will generate a fixed set of splits and each split is finite; for unbounded input, either the splits are not finite or the <code>SplitEnumerator</code> keeps generating new splits.</p> |
| |
| <p><strong>Implicit Watermark and Event Time Handling</strong></p> |
| |
| <p>The <code>TimestampAssigner</code> and <code>WatermarkGenerator</code> run transparently as part of the <code>SourceReader</code> component, so users also don’t have to implement any timestamp extraction or watermark generation code.</p> |
| |
| <p><span class="label label-info">Note</span> The existing source connectors have not yet been reimplemented using the Data Source API — this is planned for upcoming releases. If you’re looking to implement a new source, please refer to the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/stream/sources.html#data-sources">Data Source documentation</a> and <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/stream/sources.html#the-data-source-api">the tips on source development</a>.</p> |
| |
| <h3 id="application-mode-deployments">Application Mode Deployments</h3> |
| |
| <p>Prior to Flink 1.11, jobs in a Flink application could either be submitted to a long-running <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/concepts/flink-architecture.html#flink-session-cluster">Flink Session Cluster</a> (<em>session mode</em>) or a dedicated <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/concepts/flink-architecture.html#flink-job-cluster">Flink Job Cluster</a> (<em>job mode</em>). For both these modes, the <code>main()</code> method of user programs runs on the <em>client</em> side. This presents some challenges: on one hand, if the client is part of a large installation, it can easily become a bottleneck for <code>JobGraph</code> generation; and on the other, it’s not a good fit for containerized environments like Docker or Kubernetes.</p> |
| |
| <p>From this release on, Flink gets an additional deployment mode: <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/ops/deployment/#application-mode">Application Mode</a> (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-85+Flink+Application+Mode">FLIP-85</a>); where the <code>main()</code> method runs on the cluster, rather than the client. The job submission becomes a one-step process: you package your application logic and dependencies into an executable job JAR and the cluster entrypoint (<a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/client/deployment/application/ApplicationClusterEntryPoint.html"><code>ApplicationClusterEntryPoint</code></a>) is responsible for calling the <code>main()</code> method to extract the <code>JobGraph</code>.</p> |
| |
| <p>In Flink 1.11, the community worked to already support <em>application mode</em> in Kubernetes (<a href="https://issues.apache.org/jira/browse/FLINK-10934">FLINK-10934</a>).</p> |
| |
| <h3 id="other-improvements">Other Improvements</h3> |
| |
| <p><strong>Unified Memory Configuration for JobManagers (<a href="https://jira.apache.org/jira/browse/FLINK-16614">FLIP-116</a>)</strong></p> |
| |
| <p>Following the work started in Flink 1.10 to improve memory management and configuration, this release introduces a new memory model that aligns the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/ops/memory/mem_setup_master.html">JobManagers’ configuration options</a> and terminology with that introduced in <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-49%3A+Unified+Memory+Configuration+for+TaskExecutors">FLIP-49</a> for TaskManagers. This affects all deployment types: standalone, YARN, Mesos and the new active Kubernetes integration.</p> |
| |
| <p><span class="label label-danger">Attention</span> Reusing a previous Flink configuration without any adjustments can result in differently computed memory parameters for the JVM and, as a result, performance changes or even failures. Make sure to check the <a href="https://ci.apache.org/projects/flink/flink-docs-master/ops/memory/mem_migration.html#migrate-job-manager-memory-configuration">migration guide</a> if you’re planning to update to the latest version.</p> |
| |
| <p><strong>Improvements to the Flink WebUI (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-75%3A+Flink+Web+UI+Improvement+Proposal">FLIP-75</a>)</strong></p> |
| |
| <p>In Flink 1.11, the community kicked off a series of improvements to the Flink WebUI. The first to roll out are better TaskManager and JobManager log display (<a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=147427143">FLIP-103</a>), as well as a new thread dump utility (<a href="https://issues.apache.org/jira/browse/FLINK-14816">FLINK-14816</a>). Some additional work planned for upcoming releases includes better backpressure detection, more flexible and configurable exception display and support for displaying the history of subtask failure attempts.</p> |
| |
| <p><strong>Docker Image Unification (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-111%3A+Docker+image+unification">FLIP-111</a>)</strong></p> |
| |
| <p>With this release, all Docker-related resources have been consolidated into <a href="https://github.com/apache/flink-docker">apache/flink-docker</a> and the entry point script has been extended to allow users to run the default Docker image in different modes without the need to create a custom image. The <a href="https://ci.apache.org/projects/flink/flink-docs-master/ops/deployment/docker.html#customize-flink-image">updated documentation</a> describes in detail how to use and customize the official Flink Docker image for different environments and deployment modes.</p> |
| |
| <hr /> |
| |
| <h3 id="table-apisql-support-for-change-data-capture-cdc">Table API/SQL: Support for Change Data Capture (CDC)</h3> |
| |
| <p>Change Data Capture (CDC) has become a popular pattern to capture committed changes from a database and propagate those changes to downstream consumers, for example to keep multiple datastores in sync and avoid common pitfalls such as <a href="https://thorben-janssen.com/dual-writes/">dual writes</a>. Being able to easily ingest and interpret these changelogs into the Table API/SQL has been a highly demanded feature in the Flink community — and it’s now possible with Flink 1.11.</p> |
| |
| <p>To extend the scope of the Table API/SQL to use cases like CDC, Flink 1.11 introduces new table source and sink interfaces with <strong>changelog mode</strong> (see <em><a href="#other-improvements-to-the-table-apisql">New TableSource and TableSink Interfaces</a></em>) and support for the <a href="https://debezium.io/">Debezium</a> and <a href="https://github.com/alibaba/canal">Canal</a> formats (<a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=147427289">FLIP-105</a>). This means that dynamic tables sources are no longer limited to append-only operations and can ingest these external changelogs (<code>INSERT</code> events), interpret them into change operations (<code>INSERT</code>, <code>UPDATE</code>, <code>DELETE</code> events) and emit them downstream with the change type.</p> |
| |
| <center> |
| <figure> |
| <img src="/img/blog/2020-07-06-release-1.11.0/image4.png" width="500px" alt="CDC" /> |
| </figure> |
| </center> |
| |
| <div style="line-height:150%;"> |
| <br /> |
| </div> |
| |
| <p>Users have to specify either <code>“format=debezium-json”</code> or <code>“format=canal-json”</code> in their <code>CREATE TABLE</code> statement to consume changelogs using SQL DDL.</p> |
| |
| <div class="highlight"><pre><code class="language-sql"><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">my_table</span> <span class="p">(</span> |
| <span class="p">...</span> |
| <span class="p">)</span> <span class="k">WITH</span> <span class="p">(</span> |
| <span class="s1">'connector'</span><span class="o">=</span><span class="s1">'...'</span><span class="p">,</span> <span class="c1">-- e.g. 'kafka'</span> |
| <span class="s1">'format'</span><span class="o">=</span><span class="s1">'debezium-json'</span><span class="p">,</span> |
| <span class="s1">'debezium-json.schema-include'</span><span class="o">=</span><span class="s1">'true'</span> <span class="c1">-- default: false (Debezium can be configured to include or exclude the message schema)</span> |
| <span class="s1">'debezium-json.ignore-parse-errors'</span><span class="o">=</span><span class="s1">'true'</span> <span class="c1">-- default: false</span> |
| <span class="p">);</span></code></pre></div> |
| |
| <p>Flink 1.11 only supports Kafka as a changelog source out-of-the-box and JSON-encoded changelogs, with Avro (Debezium) and Protobuf (Canal) planned for future releases. There are also plans to support MySQL binlogs and Kafka compacted topics as sources, as well as to extend changelog support to batch execution.</p> |
| |
| <p><span class="label label-danger">Attention</span> There is a known issue (<a href="https://issues.apache.org/jira/browse/FLINK-18461">FLINK-18461</a>) that prevents changelog sources from being used to write to upsert sinks (e.g. MySQL, HBase, Elasticsearch). This will be fixed in the next patch release (1.11.1).</p> |
| |
| <h3 id="table-apisql-jdbc-catalog-interface-and-postgres-catalog">Table API/SQL: JDBC Catalog Interface and Postgres Catalog</h3> |
| |
| <p>Flink 1.11 introduces a generic JDBC catalog interface (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-93%3A+JDBC+catalog+and+Postgres+catalog">FLIP-93</a>) that enables users of the Table API/SQL to <strong>derive table schemas automatically</strong> from connections to relational databases over <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/table/connect.html#jdbc-connector">JDBC</a>. This eliminates the previous need for manual schema definition and type conversion, and also allows to check for schema errors at compile time instead of runtime.</p> |
| |
| <p>The first implementation, rolling out with the new release, is the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/table/catalogs.html#postgrescatalog">Postgres catalog</a>.</p> |
| |
| <h3 id="table-apisql-filesystem-connector-with-support-for-avro-orc-and-parquet">Table API/SQL: FileSystem Connector with Support for Avro, ORC and Parquet</h3> |
| |
| <p>To improve the user experience for end-to-end streaming ETL use cases, the Flink community worked on a new FileSystem Connector for the Table API/SQL (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-115%3A+Filesystem+connector+in+Table">FLIP-115</a>). The implementation is based on Flink’s <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/ops/filesystems/index.html">FileSystem abstraction</a> and reuses <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/connectors/streamfile_sink.html">StreamingFileSink</a> to ensure the same set of capabilities and consistent behaviour with the DataStream API.</p> |
| |
| <p>This also means that Table API/SQL users can now make use of all formats already supported by StreamingFileSink, like (Avro) Parquet, as well as the new formats introduced with this release, like Avro (<a href="https://issues.apache.org/jira/browse/FLINK-11395">FLINK-11395</a>) and Orc (<a href="https://issues.apache.org/jira/browse/FLINK-10114">FLINK-10114</a>).</p> |
| |
| <div class="highlight"><pre><code class="language-sql"><span class="k">CREATE</span> <span class="k">TABLE</span> <span class="n">my_table</span> <span class="p">(</span> |
| <span class="n">column_name1</span> <span class="nb">INT</span><span class="p">,</span> |
| <span class="n">column_name2</span> <span class="n">STRING</span><span class="p">,</span> |
| <span class="p">...</span> |
| <span class="n">part_name1</span> <span class="nb">INT</span><span class="p">,</span> |
| <span class="n">part_name2</span> <span class="n">STRING</span> |
| <span class="p">)</span> <span class="n">PARTITIONED</span> <span class="k">BY</span> <span class="p">(</span><span class="n">part_name1</span><span class="p">,</span> <span class="n">part_name2</span><span class="p">)</span> <span class="k">WITH</span> <span class="p">(</span> |
| <span class="s1">'connector'</span> <span class="o">=</span> <span class="s1">'filesystem'</span><span class="p">,</span> |
| <span class="s1">'path'</span> <span class="o">=</span> <span class="s1">'file:///path/to/file,</span> |
| <span class="s1"> '</span><span class="n">format</span><span class="s1">' = '</span><span class="p">...</span><span class="err">'</span><span class="p">,</span> <span class="c1">-- supported formats: Avro, ORC, Parquet, CSV, JSON </span> |
| <span class="p">...</span> |
| <span class="p">);</span></code></pre></div> |
| |
| <p>The new all-rounder FileSystem Connector transparently handles batch and streaming execution, provides exactly-once guarantees and has full partition support, greatly expanding the scope of usage of the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/table/connect.html#file-system-connector">legacy connector</a>. This allows users to easily implement common use cases like <strong>directly streaming data from Kafka to Hive</strong>.</p> |
| |
| <p>You can track the upcoming improvements to the FileSystem Connector in <a href="https://issues.apache.org/jira/browse/FLINK-17778">FLINK-17778</a>.</p> |
| |
| <h3 id="table-apisql-support-for-python-udfs">Table API/SQL: Support for Python UDFs</h3> |
| |
| <p>Prior to this release, users of the Table API/SQL were limited to defining UDFs in either Java or Scala. In Flink 1.11, the community worked on expanding the usage scope of the Python language beyond PyFlink and providing support for Python UDFs in the SQL DDL syntax (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-106%3A+Support+Python+UDF+in+SQL+Function+DDL">FLIP-106</a>), as well as the SQL Client (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-114%3A+Support+Python+UDF+in+SQL+Client">FLIP-114</a>). Users can also register Python UDFs in the system catalog via SQL DDL or the Java Catalog API, so that functions can be shared between jobs.</p> |
| |
| <h3 id="other-improvements-to-the-table-apisql">Other Improvements to the Table API/SQL</h3> |
| |
| <p><strong>DDL and DML Compatibility for the Hive Connector (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-123%3A+DDL+and+DML+compatibility+for+Hive+connector">FLIP-123</a>)</strong></p> |
| |
| <p>Starting from Flink 1.11, users can write SQL statements directly using Hive syntax (HiveQL) in the Table API/SQL and the SQL Client. For this purpose, an additional dialect was introduced and users can now dynamically switch between Flink (<code>default</code>) and Hive (<code>hive</code>) on a per-statement basis. For a complete list of supported DDL and DML statements, check the Hive dialect <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/table/hive/hive_dialect.html#hive-dialect">documentation</a>.</p> |
| |
| <p><strong>Extensions and Improvements to the Flink SQL Syntax</strong></p> |
| |
| <ul> |
| <li> |
| <p>Flink 1.11 introduces the concept of <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/table/sql/create.html#create-table">primary key constraints</a> to leverage runtime optimizations in Flink SQL DDL (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP+87%3A+Primary+key+constraints+in+Table+API">FLIP-87</a>);</p> |
| </li> |
| <li> |
| <p>View objects are now fully supported in SQL DDL using the <code>CREATE</code>/<code>ALTER</code>/<code>DROP VIEW</code> statements (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-71%3A+E2E+View+support+in+FLINK+SQL">FLIP-71</a>);</p> |
| </li> |
| <li> |
| <p>Users can now specify or override table options in their DQL/DML statements using <a href="https://ci.apache.org/projects/flink/flink-docs-master/dev/table/sql/hints.html#dynamic-table-options">dynamic table options</a> (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-113%3A+Supports+Dynamic+Table+Options+for+Flink+SQL">FLIP-113</a>).</p> |
| </li> |
| <li> |
| <p>To make connector properties less verbose and improve exception handling, some key properties have been refactored (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-122%3A+New+Connector+Property+Keys+for+New+Factory">FLIP-122</a>). This change does not break compatibility, so users can still use the old property keys.</p> |
| </li> |
| </ul> |
| |
| <p><strong>New TableSource and TableSink Interfaces (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-95%3A+New+TableSource+and+TableSink+interfaces">FLIP-95</a>)</strong></p> |
| |
| <p>Flink 1.11 introduces new table source and sink interfaces (resp. <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/table/connector/source/DynamicTableSource.html"><code>DynamicTableSource</code></a> and <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/table/connector/sink/DynamicTableSink.html"><code>DynamicTableSink</code></a>) that unify batch and streaming execution, provide more efficient data processing with the Blink planner and offer support for handling changelogs (see <em><a href="#table-apisql-support-for-change-data-capture-cdc">Support for Change Data Capture (CDC)</a></em>). The new interfaces also make it easier for users to <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/table/sourceSinks.html#full-stack-example">implement custom connectors</a> or modify existing ones. For an end-to-end example on how to implement a custom scan table source with a decoding format that supports changelog semantics, check out the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/table/sourceSinks.html#full-stack-example">documentation</a>.</p> |
| |
| <p><span class="label label-info">Note</span> Although compatibility is not immediately affected, we recommend that Table API/SQL users update any sources and sinks to the new interface stack.</p> |
| |
| <p><strong>Refactored TableEnvironment Interface (<a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=134745878">FLIP-84</a>)</strong></p> |
| |
| <p>The semantics to describe similar behaviours in the <code>TableEnvironment</code> and <code>Table</code> interfaces have diverged over time, leading to an inconsistent and sometimes unclear user experience. To improve this and make programming more fluent in the Table API/SQL, Flink 1.11 introduces new methods that unify behaviours like execution triggering (e.g. <code>executeSql()</code>) and result representation (e.g. <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/table/api/TableResult.html#print--"><code>print()</code></a>, <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/api/java/org/apache/flink/table/api/TableResult.html#collect--"><code>collect()</code></a>), and also lay the groundwork for important features like <a href="https://lists.apache.org/thread.html/r076e63bf6c8ed42d1b9ed2b406029696274a3a90cc360bc3a03e65d2%40%3Cdev.flink.apache.org%3E">multi-statement execution support</a> in future releases.</p> |
| |
| <p><span class="label label-info">Note</span> The methods deprecated with FLIP-84 will not be immediately removed, but we recommend that users adopt the newly introduced methods. For a complete list of new and deprecated methods, check the “Summary” section of <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=134745878">FLIP-84</a>.</p> |
| |
| <p><strong>New Type Inference for Table API UDFs (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-65%3A+New+type+inference+for+Table+API+UDFs">FLIP-65</a>)</strong></p> |
| |
| <p>In Flink 1.9, the community started working on a <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/table/types.html#data-types">new data type system</a> for the Table API to improve its compliance with the SQL standard (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-37%3A+Rework+of+the+Table+API+Type+System">FLIP-37</a>). This work is now close to being completed in Flink 1.11, with the exposure of Table API UDFs to the new type system (scalar and table functions, with aggregate functions planned for the next release).</p> |
| |
| <hr /> |
| |
| <h3 id="pyflink-support-for-pandas-udfs">PyFlink: Support for Pandas UDFs</h3> |
| |
| <p>Up to this release, Python UDFs in PyFlink only supported scalar values of standard Python types. This presented some limitations:</p> |
| |
| <ul> |
| <li> |
| <p>High serialization/deserialization overhead in the process of transferring data between the JVM and the Python processes;</p> |
| </li> |
| <li> |
| <p>Hard to integrate with common Python libraries for high-performance numerical processing like pandas and NumPy.</p> |
| </li> |
| </ul> |
| |
| <p>To overcome these limitations, the community introduced support for (scalar) <strong>vectorized Python UDFs</strong> based on <a href="https://pandas.pydata.org/pandas-docs/stable/getting_started/overview.html">pandas</a> in Flink 1.11 (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-97%3A+Support+Scalar+Vectorized+Python+UDF+in+PyFlink">FLIP-97</a>). The performance of vectorized UDFs is usually much higher, as the serialization/deserialization overhead is minimized by falling back to <a href="https://arrow.apache.org/">Apache Arrow</a>; and handling <code>pandas.Series</code> as input/output allows to take full advantage of the pandas and NumPy libraries. This makes Pandas UDFs a popular solution to parallelize Machine Learning and other large-scale, distributed data science workloads (e.g. feature engineering, distributed model application).</p> |
| |
| <div class="highlight"><pre><code class="language-python"><span class="nd">@udf</span><span class="p">(</span><span class="n">input_types</span><span class="o">=</span><span class="p">[</span><span class="n">DataTypes</span><span class="o">.</span><span class="n">BIGINT</span><span class="p">(),</span> <span class="n">DataTypes</span><span class="o">.</span><span class="n">BIGINT</span><span class="p">()],</span> <span class="n">result_type</span><span class="o">=</span><span class="n">DataTypes</span><span class="o">.</span><span class="n">BIGINT</span><span class="p">(),</span> <span class="n">udf_type</span><span class="o">=</span><span class="s">"pandas"</span><span class="p">)</span> |
| <span class="k">def</span> <span class="nf">add</span><span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">i</span> <span class="o">+</span> <span class="n">j</span></code></pre></div> |
| |
| <p>To mark a UDF as a Pandas UDF, you only need to add an extra parameter <code>udf_type=”pandas”</code> in the udf decorator, as described in the <a href="https://ci.apache.org/projects/flink/flink-docs-master/dev/table/python/vectorized_python_udfs.html#vectorized-user-defined-functions">documentation</a>.</p> |
| |
| <h3 id="other-improvements-to-pyflink">Other Improvements to PyFlink</h3> |
| |
| <p><strong>Conversion fromPandas/toPandas (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-120%3A+Support+conversion+between+PyFlink+Table+and+Pandas+DataFrame">FLIP-120</a>)</strong></p> |
| |
| <p>Arrow is also supported as an optimization to convert between PyFlink tables and <a href="https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html"><code>pandas.DataFrames</code></a>, enabling users to switch processing engines seamlessly without the need for an intermediate connector. For examples on how to use the new <code>fromPandas()</code> and <code>toPandas()</code> methods in PyFlink, check out the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/table/python/conversion_of_pandas.html#conversions-between-pyflink-table-and-pandas-dataframe">documentation</a>.</p> |
| |
| <p><strong>Support for User-defined Table Functions (UDTFs) (<a href="https://jira.apache.org/jira/browse/FLINK-14500">FLINK-14500</a>)</strong></p> |
| |
| <p>From Flink 1.11, you can define and register custom <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/table/python/python_udfs.html#table-functions">UDTFs</a> in PyFlink. Similar to a Python UDF, a UDTF takes zero, one or multiple scalar values as input, but can return an arbitrary number of rows as output instead of a single value.</p> |
| |
| <p><strong>Cython Performance Optimization for UDFs (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-121%3A+Support+Cython+Optimizing+Python+User+Defined+Function">FLIP-121</a>)</strong></p> |
| |
| <p><a href="https://cython.readthedocs.io/en/latest/src/quickstart/cythonize.html">Cython</a> is a compiled superset of the Python language that is often used to improve the performance of large-scale numeric processing in Python, as it optimizes execution to machine code-level speed and pairs well with popular C-based libraries like NumPy. From Flink 1.11, you can build <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/flinkDev/building.html#build-pyflink">PyFlink with Cython support</a> and “Cythonize” your Python UDFs to substantially improve code execution speed (up to 30x faster, compared to Python UDFs in Flink 1.10).</p> |
| |
| <p><strong>User-defined Metrics in Python UDFs (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-112%3A+Support+User-Defined+Metrics+in++Python+UDF">FLIP-112</a>)</strong></p> |
| |
| <p>To make it easier for users to monitor and debug the execution of Python UDFs, PyFlink now allows gathering and exposing metrics to external systems, as well as defining user scopes and variables. You can access the metrics system from a UDF by calling <code>function_context.get_metric_group()</code> in the open method, as described in the <a href="https://ci.apache.org/projects/flink/flink-docs-master/dev/table/python/metrics.html#registering-metrics">documentation</a>.</p> |
| |
| <hr /> |
| |
| <h2 id="important-changes">Important Changes</h2> |
| |
| <ul> |
| <li> |
| <p>[<a href="https://jira.apache.org/jira/browse/FLINK-17339">FLINK-17339</a>] The Blink planner is the <strong>default</strong> in the Table API/SQL starting from Flink 1.11. This was already the case for the SQL Client since Flink 1.10. The old Flink planner is still supported, but not actively developed.</p> |
| </li> |
| <li> |
| <p>[<a href="https://issues.apache.org/jira/browse/FLINK-5763">FLINK-5763</a>] Savepoints now contain all their state inside a single directory (both metadata and program state). This makes it straightforward to figure out which files make up the state of a savepoint and allows users to <strong>relocate savepoints</strong> by simply moving a directory.</p> |
| </li> |
| <li> |
| <p>[<a href="https://issues.apache.org/jira/browse/FLINK-16408">FLINK-16408</a>] To reduce pressure on the JVM metaspace, the user code class loader is being reused by a <code>TaskExecutor</code> as long as there is at least a single slot allocated for the respective job. This changes Flink’s recovery behaviour slightly, so that it will not reload static fields.</p> |
| </li> |
| <li> |
| <p>[<a href="https://issues.apache.org/jira/browse/FLINK-11086">FLINK-11086</a>] Flink now supports Hadoop versions above <strong>Hadoop 3.0.0</strong>. Note that the Flink project does not provide any updated “flink-shaded-hadoop-*” jars. Users need to provide Hadoop dependencies through the <code>HADOOP_CLASSPATH</code> environment variable (recommended) or the lib/ folder.</p> |
| </li> |
| <li> |
| <p>[<a href="https://issues.apache.org/jira/browse/FLINK-16963">FLINK-16963</a>] All <code>MetricReporters</code> that come with Flink have been converted to plugins. These should no longer be placed into <code>/lib</code> (which may result in dependency conflicts), but <code>/plugins/<some_directory></code> instead.</p> |
| </li> |
| <li> |
| <p>[<a href="https://issues.apache.org/jira/browse/FLINK-12639">FLINK-12639</a>] The Flink <strong>documentation</strong> is undergoing some <strong>rework</strong>, so you might notice that the navigation and organization of content look slightly different starting from Flink 1.11.</p> |
| </li> |
| </ul> |
| |
| <h2 id="release-notes">Release Notes</h2> |
| |
| <p>Please review the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/release-notes/flink-1.11.html">release notes</a> carefully for a detailed list of changes and new features if you plan to upgrade your setup to Flink 1.11. This version is API-compatible with previous 1.x releases for APIs annotated with the @Public annotation.</p> |
| |
| <h2 id="list-of-contributors">List of Contributors</h2> |
| |
| <p>The Apache Flink community would like to thank all the 200+ contributors that have made this release possible:</p> |
| |
| <p>Aitozi, Alexander Fedulov, Alexey Trenikhin, Aljoscha Krettek, Andrey Zagrebin, Arvid Heise, Ayush Saxena, Bairos, Bartosz Krasinski, Benchao Li, Benoit Hanotte, Benoît Paris, Bhagavan Das, Canbin Zheng, Cedric Chen, Chesnay Schepler, Colm O hEigeartaigh, Congxian Qiu, CrazyTomatoOo, Danish Amjad, Danny Chan, David Anderson, Dawid Wysakowicz, Dian Fu, Dominik Wosiński, Echo Lee, Ethan Marsh, Etienne Chauchot, Fabian Hueske, Fabian Paul, Flavio Pompermaier, Gao Yun, Gary Yao, Ghildiyal, Grebennikov Roman, GuoWei Ma, Guru Prasad, Gyula Fora, Hequn Cheng, Hu Guang, HuFeiHu, HuangXingBo, Igal Shilman, Ismael Juma, Jacob Sevart, Jark Wu, Jaskaran Bindra, Jason K, Jeff Yang, Jeff Zhang, Jerry Wang, Jiangjie (Becket) Qin, Jiayi, Jiayi Liao, Jiayi-Liao, Jincheng Sun, Jing Zhang, Jingsong Lee, JingsongLi, Jun Qin, JunZhang, Jörn Kottmann, Kevin Bohinski, Konstantin Knauf, Kostas Kloudas, Kurt Young, Leonard Xu, Lining Jing, Liupengcheng, LululuAlu, Marta Paes Moreira, Matt Welke, Max Kuklinski, Maximilian Michels, Nico Kruber, Niels Basjes, Oleksandr Nitavskyi, Paul Lam, Paul Lin, PengFei Li, PengchengLiu, Piotr Nowojski, Prem Santosh, Qingsheng Ren, Rafi Aroch, Raymond Farrelly, Richard Deurwaarder, Robert Metzger, RocMarshal, Roey Shem Tov, Roman, Roman Khachatryan, Rong Rong, RoyRuan, Rui Li, Seth Wiesman, Shaobin.Ou, Shengkai, Shuiqiang Chen, Shuo Cheng, Sivaprasanna, Sivaprasanna S, SteNicholas, Stefan Richter, Stephan Ewen, Steve OU, Steve Whelan, Tartarus, Terry Wang, Thomas Weise, Till Rohrmann, Timo Walther, TsReaper, Tzu-Li (Gordon) Tai, Victor Wong, Wei Zhong, Weike DONG, Xiaogang Zhou, Xintong Song, Xu Bai, Xuannan, Yadong Xie, Yang Wang, Yangze Guo, Yichao Yang, Ying, Yu Li, Yuan Mei, Yun Gao, Yun Tang, Yuval Itzchakov, Zakelly, Zhao, Zhenghua Gao, Zhijiang, Zhu Zhu, acqua.csq, austin ce, azagrebin, bdine, bowen.li, caoyingjie, caozhen, caozhen1937, chaojianok, chen, chendonglin, comsir, cpugputpu, czhang2, dianfu, edu05, eduardowt, fangliang, felixzheng, fmyblack, gauss, gk0916, godfrey he, godfreyhe, guliziduo, guowei.mgw, hehuiyuan, hequn8128, hpeter, huangxingbo, huzheng, ifndef-SleePy, jingwen-ywb, jrthe42, kevin.cyj, klion26, lamber-ken, leesf, libenchao, lijiewang.wlj, liuyongvs, lsy, lumen, machinedoll, mans2singh, molsionmo, oliveryunchang, openinx, paul8263, ptmagic, qqibrow, sev7e0, shuai-xu, shuai.xu, shuiqiangchen, snuyanzin, spafka, sunhaibotb, sunjincheng121, testfixer, tison, vinoyang, vthinkxie, wangtong, wangxianghu, wangxiyuan, wangxlong, wangyang0918, wenlong.lwl, whlwanghailong, william, windWheel, wooplevip, wuxuyang, xushiwei, xuyang1706, yanghua, yangyichao-mango, yuzhao.cyz, zentol, zhanglibing, zhangmang, zhangzhanchun, zhengcanbin, zhengshuli, zhenxianyimeng, zhijiang, zhongyong jin, zhule, zhuxiaoshang, zjuwangg, zoudan, zoudaokoulife, zzchun, “lzh576177775”, 骚sir, 厉颖, 张军, 曹建华, 漫步云端</p> |
| |
| </article> |
| </div> |
| |
| <div class="row"> |
| <div id="disqus_thread"></div> |
| <script type="text/javascript"> |
| /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ |
| var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname |
| |
| /* * * DON'T EDIT BELOW THIS LINE * * */ |
| (function() { |
| var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; |
| dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; |
| (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); |
| })(); |
| </script> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| <hr /> |
| |
| <div class="row"> |
| <div class="footer text-center col-sm-12"> |
| <p>Copyright © 2014-2019 <a href="http://apache.org">The Apache Software Foundation</a>. All Rights Reserved.</p> |
| <p>Apache Flink, Flink®, Apache®, the squirrel logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> |
| <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> |
| </div> |
| </div> |
| </div><!-- /.container --> |
| |
| <!-- Include all compiled plugins (below), or include individual files as needed --> |
| <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.matchHeight/0.7.0/jquery.matchHeight-min.js"></script> |
| <script src="/js/codetabs.js"></script> |
| <script src="/js/stickysidebar.js"></script> |
| |
| <!-- Google Analytics --> |
| <script> |
| (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ |
| (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), |
| m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) |
| })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); |
| |
| ga('create', 'UA-52545728-1', 'auto'); |
| ga('send', 'pageview'); |
| </script> |
| </body> |
| </html> |
