blob: 10fdd71cf17239cb6f9a45d9a62275c9f52dbf1d [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" dir=ZgotmplZ>
<head>
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
<script src="/bootstrap/js/bootstrap.bundle.min.js"></script>
<link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css">
<script src="/js/anchor.min.js"></script>
<script src="/js/flink.js"></script>
<link rel="canonical" href="https://flink.apache.org/2017/11/21/looking-ahead-to-apache-flink-1.4.0-and-1.5.0/">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="The Apache Flink 1.4.0 release is on track to happen in the next couple of weeks, and for all of the readers out there who haven’t been following the release discussion on Flink’s developer mailing list, we’d like to provide some details on what’s coming in Flink 1.4.0 as well as a preview of what the Flink community will save for 1.5.0.
Both releases include ambitious features that we believe will move Flink to an entirely new level in terms of the types of problems it can solve and applications it can support.">
<meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="Looking Ahead to Apache Flink 1.4.0 and 1.5.0" />
<meta property="og:description" content="The Apache Flink 1.4.0 release is on track to happen in the next couple of weeks, and for all of the readers out there who haven’t been following the release discussion on Flink’s developer mailing list, we’d like to provide some details on what’s coming in Flink 1.4.0 as well as a preview of what the Flink community will save for 1.5.0.
Both releases include ambitious features that we believe will move Flink to an entirely new level in terms of the types of problems it can solve and applications it can support." />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://flink.apache.org/2017/11/21/looking-ahead-to-apache-flink-1.4.0-and-1.5.0/" /><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2017-11-21T10:00:00+00:00" />
<meta property="article:modified_time" content="2017-11-21T10:00:00+00:00" />
<title>Looking Ahead to Apache Flink 1.4.0 and 1.5.0 | Apache Flink</title>
<link rel="manifest" href="/manifest.json">
<link rel="icon" href="/favicon.png" type="image/x-icon">
<link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU=">
<script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script>
<!--
Made with Book Theme
https://github.com/alex-shpak/hugo-book
-->
<meta name="generator" content="Hugo 0.124.1">
<script>
var _paq = window._paq = window._paq || [];
_paq.push(['disableCookies']);
_paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="//analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '1']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
</head>
<body dir=ZgotmplZ>
<header>
<nav class="navbar navbar-expand-xl">
<div class="container-fluid">
<a class="navbar-brand" href="/">
<img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle">
<span>Apache Flink</span>
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<i class="fa fa-bars navbar-toggler-icon"></i>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/security/">Security</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a>
</li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link" href="/posts/">Flink Blog</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/downloads/">Downloads</a>
</li>
</ul>
<div class="book-search">
<div class="book-search-spinner hidden">
<i class="fa fa-refresh fa-spin"></i>
</div>
<form class="search-bar d-flex" onsubmit="return false;"su>
<input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/">
<i class="fa fa-search search"></i>
<i class="fa fa-circle-o-notch fa-spin spinner"></i>
</form>
<div class="book-search-spinner hidden"></div>
<ul id="book-search-results"></ul>
</div>
</div>
</div>
</nav>
<div class="navbar-clearfix"></div>
</header>
<main class="flex">
<section class="container book-page">
<article class="markdown">
<h1>
<a href="/2017/11/21/looking-ahead-to-apache-flink-1.4.0-and-1.5.0/">Looking Ahead to Apache Flink 1.4.0 and 1.5.0</a>
</h1>
November 21, 2017 -
Stephan Ewen
<a href="https://twitter.com/StephanEwen">(@StephanEwen)</a>
Aljoscha Krettek
<a href="https://twitter.com/aljoscha">(@aljoscha)</a>
Mike Winters
<a href="https://twitter.com/wints">(@wints)</a>
<p><p>The Apache Flink 1.4.0 release is on track to happen in the next couple of weeks, and for all of the
readers out there who haven’t been following the release discussion on <a href="http://flink.apache.org/community.html#mailing-lists">Flink’s developer mailing
list</a>, we’d like to provide some details on
what’s coming in Flink 1.4.0 as well as a preview of what the Flink community will save for 1.5.0.</p>
<p>Both releases include ambitious features that we believe will move Flink to an entirely new level in
terms of the types of problems it can solve and applications it can support. The community deserves
lots of credit for its hard work over the past few months, and we’re excited to see these features
in the hands of users.</p>
<p>This post will describe how the community plans to get there and the rationale behind the approach.</p>
<h2 id="coming-soon-major-changes-to-flinks-runtime">
Coming soon: Major Changes to Flink’s Runtime
<a class="anchor" href="#coming-soon-major-changes-to-flinks-runtime">#</a>
</h2>
<p>There are 3 significant improvements to the Apache Flink engine that the community has nearly
completed and that will have a meaningful impact on Flink’s operability and performance.</p>
<ol>
<li>Rework of the deployment model and distributed process</li>
<li>Transition from configurable, fixed-interval network I/O to event-driven network I/O and application-level flow control for better backpressure handling</li>
<li>Faster recovery from failure</li>
</ol>
<p>Next, we’ll go through each of these improvements in more detail.</p>
<h2 id="reworking-flinks-deployment-model-and-distributed-processing">
Reworking Flink’s Deployment Model and Distributed Processing
<a class="anchor" href="#reworking-flinks-deployment-model-and-distributed-processing">#</a>
</h2>
<p><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=65147077">FLIP-6</a> (FLIP is short for
FLink Improvement Proposal and FLIPs are proposals for bigger changes to Flink) is an initiative
that’s been in the works for more than a year and represents a major refactor of Flink’s deployment
model and distributed process. The underlying motivation for FLIP-6 was the fact that Flink is being
adopted by a wider range of developer communities&ndash;both developers coming from the big data and
analytics space as well as developers coming from the event-driven applications space.</p>
<p>Modern, stateful stream processing has served as a convergence for these two developer communities.
Despite a significant overlap of the core concepts in the applications being built, each group of
developers has its own set of common tools, deployment models, and expected behaviors when working
with a stream processing framework like Flink.</p>
<p>FLIP-6 will ensure that Flink fits naturally in both of these contexts, behaving as though it’s
native to each ecosystem and operating seamlessly within a broader technology stack. A few of the
specific changes in FLIP-6 that will have such an impact:</p>
<ul>
<li>Leveraging cluster management frameworks to support full resource elasticity</li>
<li>First-class support for containerized environments such as Kubernetes and Docker</li>
<li>REST-based client-cluster communication to ease operations and 3rd party integrations</li>
</ul>
<p>FLIP-6, along with already-introduced features like
<a href="https://data-artisans.com/blog/apache-flink-at-mediamath-rescaling-stateful-applications">rescalable state</a>,
lays the groundwork for dynamic scaling in Flink, meaning that Flink programs will be able to scale up or down
automatically based on required resources&ndash;a huge step forward in terms of ease of operability and
the efficiency of Flink applications.</p>
<h2 id="lower-latency-via-improvements-to-the-apache-flink-network-stack">
Lower Latency via Improvements to the Apache Flink Network Stack
<a class="anchor" href="#lower-latency-via-improvements-to-the-apache-flink-network-stack">#</a>
</h2>
<p>Speed will always be a key consideration for users who build stream processing applications, and
Flink 1.5 will include a rework of the network stack that will even further improve Flink&rsquo;s latency.
At the heart of this work is a transition from configurable, fixed-interval network I/O to event-
driven network I/O and application-level flow control, ensuring that Flink will use all available
network capacity, as well as credit-based flow control which offers more fine-grained backpressuring
for improved checkpoint alignments.</p>
<p>In our testing (<a href="https://www.slideshare.net/FlinkForward/flink-forward-berlin-2017-nico-kruber-building-a-network-stack-for-optimal-throughput-lowlatency-tradeoffs#26">see slide 26 here</a>),
we’ve seen a substantial improvement in latency using event-driven network I/O, and the community
is also doing work to make sure we’re able to provide this increase in speed without a measurable
throughput tradeoff.</p>
<h2 id="faster-recovery-from-failures">
Faster Recovery from Failures
<a class="anchor" href="#faster-recovery-from-failures">#</a>
</h2>
<p>Flink 1.3.0 introduced incremental checkpoints, making it possible to take a checkpoint of state
updates since the last successfully-completed checkpoint only rather than the previous behavior of
only taking checkpoints of the entire state of the application. This has led to significant
performance improvements for users with large state.</p>
<p>Flink 1.5 will introduce task-local recovery, which means that Flink will store a second copy of the
most recent checkpoint on the local disk (or even in main memory) of a task manager. The primary
copy still goes to durable storage so that it’s resilient to machine failures.</p>
<p>In case of failover, the scheduler will try to reschedule tasks to their previous task manager (in
other words, to the same machine again) if this is possible. The task can then recover from the
locally-kept state. This makes it possible to avoid reading all state from the distributed file
system (which is remote over the network). Especially in applications with very large state, not
having to read many gigabytes over the network and instead from local disk will result in
significant performance gains in recovery.</p>
<h2 id="the-proposed-timeline-for-flink-14-and-flink-15">
The Proposed Timeline for Flink 1.4 and Flink 1.5
<a class="anchor" href="#the-proposed-timeline-for-flink-14-and-flink-15">#</a>
</h2>
<p>The good news is that all 3 of the features described above are well underway, and in fact, much of
the work is already covered by open pull requests.</p>
<p>But given these features’ importance and the complexity of the work involved, the community expected
that the QA and testing required would be extensive and would delay the release of the otherwise-
ready features also on the list for the next release.</p>
<p>And so the community decided to withhold the 3 features above (deployment model rework, improvements
to the network stack, and faster recovery) to be included a separate Flink 1.5 release that will
come shortly after the Flink 1.4 release. Flink 1.5 is estimated to come just a couple of months
after 1.4 rather than the typical 4-month cycle in between major releases.</p>
<p>The soon-to-be-released Flink 1.4 represents the current state of Flink without merging those 3
features. And Flink 1.4 is a substantial release in its own right, including, but not limited to,
the following:</p>
<ul>
<li><strong>A significantly improved dependency structure</strong>, removing many of Flink’s dependencies and subtle runtime conflicts. This increases overall stability and removes friction when embedding Flink or calling Flink &ldquo;library style&rdquo;.</li>
<li><strong>Reversed class loading for dynamically-loaded user code</strong>, allowing for different dependencies than those included in the core framework.</li>
<li><strong>An Apache Kafka 0.11 exactly-once producer</strong>, making it possible to build end-to-end exactly once applications with Flink and Kafka.</li>
<li><strong>Streaming SQL JOIN based on processing time and event time</strong>, which gives users the full advantage of Flink’s time handling while using a SQL JOIN.</li>
<li><strong>Table API / Streaming SQL Source and Sink Additions</strong>, including a Kafka 0.11 source and JDBC sink.</li>
<li><strong>Hadoop-free Flink</strong>, meaning that users who don’t rely on any Hadoop components (such as YARN or HDFS) in their Flink applications can use Flink without Hadoop for the first time.</li>
<li><strong>Improvements to queryable state</strong>, including a more container-friendly architecture, a more user-friendly API that hides configuration parameters, and the groundwork to be able to expose window state (the state of an in-flight window) in the future.</li>
<li><strong>Connector improvements and fixes</strong> for a range of connectors including Kafka, Apache Cassandra, Amazon Kinesis, and more.</li>
<li><strong>Improved RPC performance</strong> for faster recovery from failure</li>
</ul>
<p>The community decided it was best to get these features into a stable version of Flink as soon as
possible, and the separation of what could have been a single (and very substantial) Flink 1.4
release into 1.4 and 1.5 serves that purpose.</p>
<p>We’re excited by what each of these represents for Apache Flink, and we’d like to extend our thanks
to the Flink community for all of their hard work.</p>
<p>If you’d like to follow along with release discussions, <a href="http://flink.apache.org/community.html#mailing-lists">please subscribe to the dev@ mailing
list</a>.</p>
</p>
</article>
<div class="edit-this-page">
<p>
<a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a>
</p>
<p>
<a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2017-11-21-release-1.4-and-1.5-timeline.md">
Edit This Page<i class="fa fa-edit fa-fw"></i>
</a>
</p>
</div>
</section>
<aside class="book-toc">
<nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3>
<ul>
<li>
<ul>
<li><a href="#coming-soon-major-changes-to-flinks-runtime">Coming soon: Major Changes to Flink’s Runtime</a></li>
<li><a href="#reworking-flinks-deployment-model-and-distributed-processing">Reworking Flink’s Deployment Model and Distributed Processing</a></li>
<li><a href="#lower-latency-via-improvements-to-the-apache-flink-network-stack">Lower Latency via Improvements to the Apache Flink Network Stack</a></li>
<li><a href="#faster-recovery-from-failures">Faster Recovery from Failures</a></li>
<li><a href="#the-proposed-timeline-for-flink-14-and-flink-15">The Proposed Timeline for Flink 1.4 and Flink 1.5</a></li>
</ul>
</li>
</ul>
</nav>
</aside>
<aside class="expand-toc hidden">
<a class="toc" onclick="expandToc()" href="javascript:void(0)">
<i class="fa fa-bars" aria-hidden="true"></i>
</a>
</aside>
</main>
<footer>
<div class="separator"></div>
<div class="panels">
<div class="wrapper">
<div class="panel">
<ul>
<li>
<a href="https://flink-packages.org/">flink-packages.org</a>
</li>
<li>
<a href="https://www.apache.org/">Apache Software Foundation</a>
</li>
<li>
<a href="https://www.apache.org/licenses/">License</a>
</li>
<li>
<a href="/zh/">
<i class="fa fa-globe" aria-hidden="true"></i>&nbsp;中文版
</a>
</li>
</ul>
</div>
<div class="panel">
<ul>
<li>
<a href="/what-is-flink/security">Security</a-->
</li>
<li>
<a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
</li>
<li>
<a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
</li>
</ul>
</div>
<div class="panel icons">
<div>
<a href="/posts">
<div class="icon flink-blog-icon"></div>
<span>Flink blog</span>
</a>
</div>
<div>
<a href="https://github.com/apache/flink">
<div class="icon flink-github-icon"></div>
<span>Github</span>
</a>
</div>
<div>
<a href="https://twitter.com/apacheflink">
<div class="icon flink-twitter-icon"></div>
<span>Twitter</span>
</a>
</div>
</div>
</div>
</div>
<hr/>
<div class="container disclaimer">
<p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</footer>
</body>
</html>