| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> |
| <title>Apache Flink: Looking Ahead to Apache Flink 1.4.0 and 1.5.0</title> |
| <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> |
| <link rel="icon" href="/favicon.ico" type="image/x-icon"> |
| |
| <!-- Bootstrap --> |
| <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css"> |
| <link rel="stylesheet" href="/css/flink.css"> |
| <link rel="stylesheet" href="/css/syntax.css"> |
| |
| <!-- Blog RSS feed --> |
| <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> |
| |
| <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> |
| <!-- We need to load Jquery in the header for custom google analytics event tracking--> |
| <script src="/js/jquery.min.js"></script> |
| |
| <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> |
| <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> |
| <!--[if lt IE 9]> |
| <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script> |
| <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script> |
| <![endif]--> |
| </head> |
| <body> |
| |
| |
| <!-- Main content. --> |
| <div class="container"> |
| <div class="row"> |
| |
| |
| <div id="sidebar" class="col-sm-3"> |
| |
| |
| <!-- Top navbar. --> |
| <nav class="navbar navbar-default"> |
| <!-- The logo. --> |
| <div class="navbar-header"> |
| <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| </button> |
| <div class="navbar-logo"> |
| <a href="/"> |
| <img alt="Apache Flink" src="/img/flink-header-logo.svg" width="147px" height="73px"> |
| </a> |
| </div> |
| </div><!-- /.navbar-header --> |
| |
| <!-- The navigation links. --> |
| <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> |
| <ul class="nav navbar-nav navbar-main"> |
| |
| <!-- First menu section explains visitors what Flink is --> |
| |
| <!-- What is Stream Processing? --> |
| <!-- |
| <li><a href="/streamprocessing1.html">What is Stream Processing?</a></li> |
| --> |
| |
| <!-- What is Flink? --> |
| <li><a href="/flink-architecture.html">What is Apache Flink?</a></li> |
| |
| |
| |
| <!-- What is Stateful Functions? --> |
| |
| <li><a href="/stateful-functions.html">What is Stateful Functions?</a></li> |
| |
| <!-- Use cases --> |
| <li><a href="/usecases.html">Use Cases</a></li> |
| |
| <!-- Powered by --> |
| <li><a href="/poweredby.html">Powered By</a></li> |
| |
| |
| |
| <!-- Second menu section aims to support Flink users --> |
| |
| <!-- Downloads --> |
| <li><a href="/downloads.html">Downloads</a></li> |
| |
| <!-- Getting Started --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Getting Started<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.10/getting-started/index.html" target="_blank">With Flink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1/getting-started/project-setup.html" target="_blank">With Flink Stateful Functions <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="/training.html">Training Course</a></li> |
| </ul> |
| </li> |
| |
| <!-- Documentation --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.10" target="_blank">Flink 1.10 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-master" target="_blank">Flink Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1" target="_blank">Flink Stateful Functions 2.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-master" target="_blank">Flink Stateful Functions Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| </ul> |
| </li> |
| |
| <!-- getting help --> |
| <li><a href="/gettinghelp.html">Getting Help</a></li> |
| |
| <!-- Blog --> |
| <li class="active"><a href="/blog/"><b>Flink Blog</b></a></li> |
| |
| |
| <!-- Flink-packages --> |
| <li> |
| <a href="https://flink-packages.org" target="_blank">flink-packages.org <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| <!-- Third menu section aim to support community and contributors --> |
| |
| <!-- Community --> |
| <li><a href="/community.html">Community & Project Info</a></li> |
| |
| <!-- Roadmap --> |
| <li><a href="/roadmap.html">Roadmap</a></li> |
| |
| <!-- Contribute --> |
| <li><a href="/contributing/how-to-contribute.html">How to Contribute</a></li> |
| |
| |
| <!-- GitHub --> |
| <li> |
| <a href="https://github.com/apache/flink" target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| |
| <!-- Language Switcher --> |
| <li> |
| |
| |
| <!-- link to the Chinese home page when current is blog page --> |
| <a href="/zh">中文版</a> |
| |
| |
| </li> |
| |
| </ul> |
| |
| <ul class="nav navbar-nav navbar-bottom"> |
| <hr /> |
| |
| <!-- Twitter --> |
| <li><a href="https://twitter.com/apacheflink" target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <!-- Visualizer --> |
| <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <hr /> |
| |
| <li><a href="https://apache.org" target="_blank">Apache Software Foundation <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <li> |
| <style> |
| .smalllinks:link { |
| display: inline-block !important; background: none; padding-top: 0px; padding-bottom: 0px; padding-right: 0px; min-width: 75px; |
| } |
| </style> |
| |
| <a class="smalllinks" href="https://www.apache.org/licenses/" target="_blank">License</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/security/" target="_blank">Security</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/sponsorship.html" target="_blank">Donate</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| </li> |
| |
| </ul> |
| </div><!-- /.navbar-collapse --> |
| </nav> |
| |
| </div> |
| <div class="col-sm-9"> |
| <div class="row-fluid"> |
| <div class="col-sm-12"> |
| <div class="row"> |
| <h1>Looking Ahead to Apache Flink 1.4.0 and 1.5.0</h1> |
| <p><i></i></p> |
| |
| <article> |
| <p>22 Nov 2017 Stephan Ewen (<a href="https://twitter.com/StephanEwen">@StephanEwen</a>), Aljoscha Krettek (<a href="https://twitter.com/aljoscha">@aljoscha</a>), & Mike Winters (<a href="https://twitter.com/wints">@wints</a>)</p> |
| |
| <p>The Apache Flink 1.4.0 release is on track to happen in the next couple of weeks, and for all of the |
| readers out there who haven’t been following the release discussion on <a href="http://flink.apache.org/community.html#mailing-lists">Flink’s developer mailing |
| list</a>, we’d like to provide some details on |
| what’s coming in Flink 1.4.0 as well as a preview of what the Flink community will save for 1.5.0.</p> |
| |
| <p>Both releases include ambitious features that we believe will move Flink to an entirely new level in |
| terms of the types of problems it can solve and applications it can support. The community deserves |
| lots of credit for its hard work over the past few months, and we’re excited to see these features |
| in the hands of users.</p> |
| |
| <p>This post will describe how the community plans to get there and the rationale behind the approach.</p> |
| |
| <h2 id="coming-soon-major-changes-to-flinks-runtime">Coming soon: Major Changes to Flink’s Runtime</h2> |
| |
| <p>There are 3 significant improvements to the Apache Flink engine that the community has nearly |
| completed and that will have a meaningful impact on Flink’s operability and performance.</p> |
| |
| <ol> |
| <li>Rework of the deployment model and distributed process</li> |
| <li>Transition from configurable, fixed-interval network I/O to event-driven network I/O and application-level flow control for better backpressure handling</li> |
| <li>Faster recovery from failure</li> |
| </ol> |
| |
| <p>Next, we’ll go through each of these improvements in more detail.</p> |
| |
| <h2 id="reworking-flinks-deployment-model-and-distributed-processing">Reworking Flink’s Deployment Model and Distributed Processing</h2> |
| |
| <p><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=65147077">FLIP-6</a> (FLIP is short for |
| FLink Improvement Proposal and FLIPs are proposals for bigger changes to Flink) is an initiative |
| that’s been in the works for more than a year and represents a major refactor of Flink’s deployment |
| model and distributed process. The underlying motivation for FLIP-6 was the fact that Flink is being |
| adopted by a wider range of developer communities–both developers coming from the big data and |
| analytics space as well as developers coming from the event-driven applications space.</p> |
| |
| <p>Modern, stateful stream processing has served as a convergence for these two developer communities. |
| Despite a significant overlap of the core concepts in the applications being built, each group of |
| developers has its own set of common tools, deployment models, and expected behaviors when working |
| with a stream processing framework like Flink.</p> |
| |
| <p>FLIP-6 will ensure that Flink fits naturally in both of these contexts, behaving as though it’s |
| native to each ecosystem and operating seamlessly within a broader technology stack. A few of the |
| specific changes in FLIP-6 that will have such an impact:</p> |
| |
| <ul> |
| <li>Leveraging cluster management frameworks to support full resource elasticity</li> |
| <li>First-class support for containerized environments such as Kubernetes and Docker</li> |
| <li>REST-based client-cluster communication to ease operations and 3rd party integrations</li> |
| </ul> |
| |
| <p>FLIP-6, along with already-introduced features like |
| <a href="https://data-artisans.com/blog/apache-flink-at-mediamath-rescaling-stateful-applications">rescalable state</a>, |
| lays the groundwork for dynamic scaling in Flink, meaning that Flink programs will be able to scale up or down |
| automatically based on required resources–a huge step forward in terms of ease of operability and |
| the efficiency of Flink applications.</p> |
| |
| <h2 id="lower-latency-via-improvements-to-the-apache-flink-network-stack">Lower Latency via Improvements to the Apache Flink Network Stack</h2> |
| |
| <p>Speed will always be a key consideration for users who build stream processing applications, and |
| Flink 1.5 will include a rework of the network stack that will even further improve Flink’s latency. |
| At the heart of this work is a transition from configurable, fixed-interval network I/O to event- |
| driven network I/O and application-level flow control, ensuring that Flink will use all available |
| network capacity, as well as credit-based flow control which offers more fine-grained backpressuring |
| for improved checkpoint alignments.</p> |
| |
| <p>In our testing (<a href="https://www.slideshare.net/FlinkForward/flink-forward-berlin-2017-nico-kruber-building-a-network-stack-for-optimal-throughput-lowlatency-tradeoffs#26">see slide 26 here</a>), |
| we’ve seen a substantial improvement in latency using event-driven network I/O, and the community |
| is also doing work to make sure we’re able to provide this increase in speed without a measurable |
| throughput tradeoff.</p> |
| |
| <h2 id="faster-recovery-from-failures">Faster Recovery from Failures</h2> |
| |
| <p>Flink 1.3.0 introduced incremental checkpoints, making it possible to take a checkpoint of state |
| updates since the last successfully-completed checkpoint only rather than the previous behavior of |
| only taking checkpoints of the entire state of the application. This has led to significant |
| performance improvements for users with large state.</p> |
| |
| <p>Flink 1.5 will introduce task-local recovery, which means that Flink will store a second copy of the |
| most recent checkpoint on the local disk (or even in main memory) of a task manager. The primary |
| copy still goes to durable storage so that it’s resilient to machine failures.</p> |
| |
| <p>In case of failover, the scheduler will try to reschedule tasks to their previous task manager (in |
| other words, to the same machine again) if this is possible. The task can then recover from the |
| locally-kept state. This makes it possible to avoid reading all state from the distributed file |
| system (which is remote over the network). Especially in applications with very large state, not |
| having to read many gigabytes over the network and instead from local disk will result in |
| significant performance gains in recovery.</p> |
| |
| <h2 id="the-proposed-timeline-for-flink-14-and-flink-15">The Proposed Timeline for Flink 1.4 and Flink 1.5</h2> |
| |
| <p>The good news is that all 3 of the features described above are well underway, and in fact, much of |
| the work is already covered by open pull requests.</p> |
| |
| <p>But given these features’ importance and the complexity of the work involved, the community expected |
| that the QA and testing required would be extensive and would delay the release of the otherwise- |
| ready features also on the list for the next release.</p> |
| |
| <p>And so the community decided to withhold the 3 features above (deployment model rework, improvements |
| to the network stack, and faster recovery) to be included a separate Flink 1.5 release that will |
| come shortly after the Flink 1.4 release. Flink 1.5 is estimated to come just a couple of months |
| after 1.4 rather than the typical 4-month cycle in between major releases.</p> |
| |
| <p>The soon-to-be-released Flink 1.4 represents the current state of Flink without merging those 3 |
| features. And Flink 1.4 is a substantial release in its own right, including, but not limited to, |
| the following:</p> |
| |
| <ul> |
| <li><strong>A significantly improved dependency structure</strong>, removing many of Flink’s dependencies and subtle runtime conflicts. This increases overall stability and removes friction when embedding Flink or calling Flink “library style”.</li> |
| <li><strong>Reversed class loading for dynamically-loaded user code</strong>, allowing for different dependencies than those included in the core framework.</li> |
| <li><strong>An Apache Kafka 0.11 exactly-once producer</strong>, making it possible to build end-to-end exactly once applications with Flink and Kafka.</li> |
| <li><strong>Streaming SQL JOIN based on processing time and event time</strong>, which gives users the full advantage of Flink’s time handling while using a SQL JOIN.</li> |
| <li><strong>Table API / Streaming SQL Source and Sink Additions</strong>, including a Kafka 0.11 source and JDBC sink.</li> |
| <li><strong>Hadoop-free Flink</strong>, meaning that users who don’t rely on any Hadoop components (such as YARN or HDFS) in their Flink applications can use Flink without Hadoop for the first time.</li> |
| <li><strong>Improvements to queryable state</strong>, including a more container-friendly architecture, a more user-friendly API that hides configuration parameters, and the groundwork to be able to expose window state (the state of an in-flight window) in the future.</li> |
| <li><strong>Connector improvements and fixes</strong> for a range of connectors including Kafka, Apache Cassandra, Amazon Kinesis, and more.</li> |
| <li><strong>Improved RPC performance</strong> for faster recovery from failure</li> |
| </ul> |
| |
| <p>The community decided it was best to get these features into a stable version of Flink as soon as |
| possible, and the separation of what could have been a single (and very substantial) Flink 1.4 |
| release into 1.4 and 1.5 serves that purpose.</p> |
| |
| <p>We’re excited by what each of these represents for Apache Flink, and we’d like to extend our thanks |
| to the Flink community for all of their hard work.</p> |
| |
| <p>If you’d like to follow along with release discussions, <a href="http://flink.apache.org/community.html#mailing-lists">please subscribe to the dev@ mailing |
| list</a>.</p> |
| |
| |
| </article> |
| </div> |
| |
| <div class="row"> |
| <div id="disqus_thread"></div> |
| <script type="text/javascript"> |
| /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ |
| var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname |
| |
| /* * * DON'T EDIT BELOW THIS LINE * * */ |
| (function() { |
| var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; |
| dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; |
| (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); |
| })(); |
| </script> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| <hr /> |
| |
| <div class="row"> |
| <div class="footer text-center col-sm-12"> |
| <p>Copyright © 2014-2019 <a href="http://apache.org">The Apache Software Foundation</a>. All Rights Reserved.</p> |
| <p>Apache Flink, Flink®, Apache®, the squirrel logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> |
| <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> |
| </div> |
| </div> |
| </div><!-- /.container --> |
| |
| <!-- Include all compiled plugins (below), or include individual files as needed --> |
| <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.matchHeight/0.7.0/jquery.matchHeight-min.js"></script> |
| <script src="/js/codetabs.js"></script> |
| <script src="/js/stickysidebar.js"></script> |
| |
| <!-- Google Analytics --> |
| <script> |
| (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ |
| (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), |
| m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) |
| })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); |
| |
| ga('create', 'UA-52545728-1', 'auto'); |
| ga('send', 'pageview'); |
| </script> |
| </body> |
| </html> |
