blob: 39e0d31374bdfcf6d0802ab6eda8a72a7f7e8995 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
<title>Apache Flink: Flink Community Update - April'20</title>
<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
<!-- Bootstrap -->
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css">
<link rel="stylesheet" href="/css/flink.css">
<link rel="stylesheet" href="/css/syntax.css">
<!-- Blog RSS feed -->
<link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" />
<!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
<!-- We need to load Jquery in the header for custom google analytics event tracking-->
<script src="/js/jquery.min.js"></script>
<!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
<!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
<!--[if lt IE 9]>
<script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
<![endif]-->
</head>
<body>
<!-- Main content. -->
<div class="container">
<div class="row">
<div id="sidebar" class="col-sm-3">
<!-- Top navbar. -->
<nav class="navbar navbar-default">
<!-- The logo. -->
<div class="navbar-header">
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<div class="navbar-logo">
<a href="/">
<img alt="Apache Flink" src="/img/flink-header-logo.svg" width="147px" height="73px">
</a>
</div>
</div><!-- /.navbar-header -->
<!-- The navigation links. -->
<div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
<ul class="nav navbar-nav navbar-main">
<!-- First menu section explains visitors what Flink is -->
<!-- What is Stream Processing? -->
<!--
<li><a href="/streamprocessing1.html">What is Stream Processing?</a></li>
-->
<!-- What is Flink? -->
<li><a href="/flink-architecture.html">What is Apache Flink?</a></li>
<!-- What is Stateful Functions? -->
<li><a href="/stateful-functions.html">What is Stateful Functions?</a></li>
<!-- Use cases -->
<li><a href="/usecases.html">Use Cases</a></li>
<!-- Powered by -->
<li><a href="/poweredby.html">Powered By</a></li>
&nbsp;
<!-- Second menu section aims to support Flink users -->
<!-- Downloads -->
<li><a href="/downloads.html">Downloads</a></li>
<!-- Getting Started -->
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" href="#">Getting Started<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/getting-started/index.html" target="_blank">With Flink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1/getting-started/project-setup.html" target="_blank">With Flink Stateful Functions <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<li><a href="/training.html">Training Course</a></li>
</ul>
</li>
<!-- Documentation -->
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11" target="_blank">Flink 1.11 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<li><a href="https://ci.apache.org/projects/flink/flink-docs-master" target="_blank">Flink Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1" target="_blank">Flink Stateful Functions 2.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-master" target="_blank">Flink Stateful Functions Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
</ul>
</li>
<!-- getting help -->
<li><a href="/gettinghelp.html">Getting Help</a></li>
<!-- Blog -->
<li class="active"><a href="/blog/"><b>Flink Blog</b></a></li>
<!-- Flink-packages -->
<li>
<a href="https://flink-packages.org" target="_blank">flink-packages.org <small><span class="glyphicon glyphicon-new-window"></span></small></a>
</li>
&nbsp;
<!-- Third menu section aim to support community and contributors -->
<!-- Community -->
<li><a href="/community.html">Community &amp; Project Info</a></li>
<!-- Roadmap -->
<li><a href="/roadmap.html">Roadmap</a></li>
<!-- Contribute -->
<li><a href="/contributing/how-to-contribute.html">How to Contribute</a></li>
<!-- GitHub -->
<li>
<a href="https://github.com/apache/flink" target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a>
</li>
&nbsp;
<!-- Language Switcher -->
<li>
<!-- link to the Chinese home page when current is blog page -->
<a href="/zh">中文版</a>
</li>
</ul>
<ul class="nav navbar-nav navbar-bottom">
<hr />
<!-- Twitter -->
<li><a href="https://twitter.com/apacheflink" target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<!-- Visualizer -->
<li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<hr />
<li><a href="https://apache.org" target="_blank">Apache Software Foundation <small><span class="glyphicon glyphicon-new-window"></span></small></a></li>
<li>
<style>
.smalllinks:link {
display: inline-block !important; background: none; padding-top: 0px; padding-bottom: 0px; padding-right: 0px; min-width: 75px;
}
</style>
<a class="smalllinks" href="https://www.apache.org/licenses/" target="_blank">License</a> <small><span class="glyphicon glyphicon-new-window"></span></small>
<a class="smalllinks" href="https://www.apache.org/security/" target="_blank">Security</a> <small><span class="glyphicon glyphicon-new-window"></span></small>
<a class="smalllinks" href="https://www.apache.org/foundation/sponsorship.html" target="_blank">Donate</a> <small><span class="glyphicon glyphicon-new-window"></span></small>
<a class="smalllinks" href="https://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a> <small><span class="glyphicon glyphicon-new-window"></span></small>
</li>
</ul>
</div><!-- /.navbar-collapse -->
</nav>
</div>
<div class="col-sm-9">
<div class="row-fluid">
<div class="col-sm-12">
<div class="row">
<h1>Flink Community Update - April'20</h1>
<p><i></i></p>
<article>
<p>01 Apr 2020 Marta Paes (<a href="https://twitter.com/morsapaes">@morsapaes</a>)</p>
<p>While things slow down around us, the Apache Flink community is privileged to remain as active as ever. This blogpost combs through the past few months to give you an update on the state of things in Flink — from core releases to Stateful Functions; from some good old community stats to a new development blog.</p>
<p>And since now it’s more important than ever to keep up the spirits, we’d like to invite you to join the <a href="https://www.flink-forward.org/sf-2020">Flink Forward Virtual Conference</a>, on April 22-24 (see <a href="#upcoming-events">Upcoming Events</a>). Hope to see you there!</p>
<div class="page-toc">
<ul id="markdown-toc">
<li><a href="#the-year-so-far-in-flink" id="markdown-toc-the-year-so-far-in-flink">The Year (so far) in Flink</a> <ul>
<li><a href="#flink-110-release" id="markdown-toc-flink-110-release">Flink 1.10 Release</a></li>
<li><a href="#stateful-functions-contribution-and-20-release" id="markdown-toc-stateful-functions-contribution-and-20-release">Stateful Functions Contribution and 2.0 Release</a></li>
<li><a href="#building-up-to-flink-111" id="markdown-toc-building-up-to-flink-111">Building up to Flink 1.11</a></li>
<li><a href="#new-committers-and-pmc-members" id="markdown-toc-new-committers-and-pmc-members">New Committers and PMC Members</a> <ul>
<li><a href="#new-pmc-members" id="markdown-toc-new-pmc-members">New PMC Members</a></li>
<li><a href="#new-committers" id="markdown-toc-new-committers">New Committers</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="#the-bigger-picture" id="markdown-toc-the-bigger-picture">The Bigger Picture</a> <ul>
<li><a href="#a-look-into-the-flink-repository" id="markdown-toc-a-look-into-the-flink-repository">A Look into the Flink Repository</a></li>
<li><a href="#flink-community-packages" id="markdown-toc-flink-community-packages">Flink Community Packages</a></li>
<li><a href="#flink-engine-room" id="markdown-toc-flink-engine-room">Flink “Engine Room”</a></li>
</ul>
</li>
<li><a href="#upcoming-events" id="markdown-toc-upcoming-events">Upcoming Events</a> <ul>
<li><a href="#flink-forward-virtual-conference" id="markdown-toc-flink-forward-virtual-conference">Flink Forward Virtual Conference</a></li>
<li><a href="#others" id="markdown-toc-others">Others</a></li>
</ul>
</li>
</ul>
</div>
<h1 id="the-year-so-far-in-flink">The Year (so far) in Flink</h1>
<h2 id="flink-110-release">Flink 1.10 Release</h2>
<p>To kick off the new year, the Flink community <a href="https://flink.apache.org/news/2020/02/11/release-1.10.0.html">released Flink 1.10</a> with the record contribution of over 200 engineers. This release introduced significant improvements to the overall performance and stability of Flink jobs, a preview of native Kubernetes integration and advances in Python support (PyFlink). Flink 1.10 also marked the completion of the <a href="https://flink.apache.org/news/2019/08/22/release-1.9.0.html#preview-of-the-new-blink-sql-query-processor">Blink integration</a>, hardening streaming SQL and bringing mature batch processing to Flink with production-ready Hive integration and TPC-DS coverage.</p>
<p>The community is now discussing the <a href="http://apache-flink-mailing-list-archive.1008284.n3.nabble.com/DISCUSS-Releasing-Flink-1-10-1-td38689.html#a38690">release of Flink 1.10.1</a>, covering some outstanding bugs from Flink 1.10.</p>
<h2 id="stateful-functions-contribution-and-20-release">Stateful Functions Contribution and 2.0 Release</h2>
<p>Last January, the first version of Stateful Functions (<a href="https://statefun.io/">statefun.io</a>) code was pushed to the <a href="https://github.com/apache/flink-statefun">Flink repository</a>. Stateful Functions started out as an API to build general purpose event-driven applications on Flink, taking advantage of its advanced state management mechanism to cut the “middleman” that usually handles state coordination in such applications (e.g. a database).</p>
<p>In a <a href="http://apache-flink-mailing-list-archive.1008284.n3.nabble.com/DISCUSS-Update-on-Flink-Stateful-Functions-what-are-the-next-steps-tp38646.html">recent update</a>, some new features were announced, like multi-language support (including a Python SDK), function unit testing and Stateful Functions’ own flavor of the <a href="https://ci.apache.org/projects/flink/flink-docs-stable/dev/libs/state_processor_api.html">State Processor API</a>. The release cycle will be independent from core Flink releases and the Release Candidate (RC) has been created — so, <strong>you can expect Stateful Functions 2.0 to be released very soon!</strong></p>
<h2 id="building-up-to-flink-111">Building up to Flink 1.11</h2>
<p>Amidst the usual outpour of discussion threads, JIRA tickets and FLIPs, the community is working full steam on bringing Flink 1.11 to life in the next few months. The feature freeze is currently scheduled for late April, so the release is expected around mid May.
The upcoming release will focus on new features and integrations that broaden the scope of Flink use cases, as well as core runtime enhancements to streamline the operations of complex deployments.</p>
<p>Some of the plans on the use case side include support for changelog streams in the Table API/SQL (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-105%3A+Support+to+Interpret+and+Emit+Changelog+in+Flink+SQL">FLIP-105</a>), easy streaming data ingestion into Apache Hive (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-115%3A+Filesystem+connector+in+Table">FLIP-115</a>) and support for Pandas DataFrames in PyFlink. On the operational side, the much anticipated new Source API (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-27%3A+Refactor+Source+Interface">FLIP-27</a>) will unify batch and streaming sources, and improve out-of-the-box event-time behavior; while unaligned checkpoints (<a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-76%3A+Unaligned+Checkpoints">FLIP-76</a>) and some changes to network memory management will allow to speed up checkpointing under backpressure.</p>
<p>Throw into the mix improvements around type systems, the WebUI, metrics reporting and supported formats, this release is bound to keep the community busy. For a complete overview of the ongoing development, check <a href="http://apache-flink-mailing-list-archive.1008284.n3.nabble.com/DISCUSS-Features-of-Apache-Flink-1-11-td38724.html#a38793">this discussion</a> and follow the weekly updates on the Flink <a href="https://flink.apache.org/community.html#mailing-lists">@community mailing list</a>.</p>
<h2 id="new-committers-and-pmc-members">New Committers and PMC Members</h2>
<p>The Apache Flink community has welcomed <strong>1 PMC (Project Management Committee) Member</strong> and <strong>5 new Committers</strong> since the last update (September 2019):</p>
<h3 id="new-pmc-members">New PMC Members</h3>
<div class="highlight"><pre><code>Jark Wu
</code></pre></div>
<h3 id="new-committers">New Committers</h3>
<div class="highlight"><pre><code>Zili Chen, Jingsong Lee, Yu Li, Dian Fu, Zhu Zhu
</code></pre></div>
<p>Congratulations to all and thank you for your hardworking commitment to Flink!</p>
<h1 id="the-bigger-picture">The Bigger Picture</h1>
<h2 id="a-look-into-the-flink-repository">A Look into the Flink Repository</h2>
<p>In the <a href="https://flink.apache.org/news/2019/09/10/community-update.html">last update</a>, we shared some numbers around Flink releases and mailing list activity. This time, we’re looking into the activity in the Flink repository and how it’s evolving.</p>
<center>
<img src="/img/blog/2020-03-30-flink-community-update/2020-03-30-flink-community-update_1.png" width="725px" alt="GitHub 1" />
</center>
<p>There is a clear upward trend in the number of contributions to the repository, based on the number of commits. This reflects the <strong>fast pace of development</strong> the project is experiencing and also the <strong>successful integration of the China-based Flink contributors</strong> started early last year. To complement these observations, the repository registered a <strong>1.5x increase in the number of individual contributors in 2019</strong>, compared to the previous year.</p>
<p>But did this increase in capacity produce any other measurable benefits?</p>
<center>
<img src="/img/blog/2020-03-30-flink-community-update/2020-03-30-flink-community-update_2.png" width="725px" alt="GitHub 2" />
</center>
<p>If we look at the average time of Pull Request (PR) “resolution”, it seems like it did: <strong>the average time it takes to close a PR these days has been steadily decreasing</strong> since last year, sitting between 5-6 days for the past few months.</p>
<p>These are great indicators of the health of Flink as an open source project!</p>
<h2 id="flink-community-packages">Flink Community Packages</h2>
<p>If you missed the launch of <a href="http://flink-packages.org/">flink-packages.org</a>, here’s a reminder! Ververica has <a href="https://www.ververica.com/blog/announcing-flink-community-packages">created (and open sourced)</a> a website that showcases the work of the community to push forward the ecosystem surrounding Flink. There, you can explore existing packages (like the Pravega and Pulsar Flink connectors, or the Flink Kubernetes operators developed by Google and Lyft) and also submit your own contributions to the ecosystem.</p>
<h2 id="flink-engine-room">Flink “Engine Room”</h2>
<p>The community has recently launched the <a href="https://cwiki.apache.org/confluence/pages/viewrecentblogposts.action?key=FLINK">“Engine Room”</a>, a dedicated space in Flink’s Wiki for knowledge sharing between contributors. The goal of this initiative is to make ongoing development on Flink internals more transparent across different work streams, and also to help new contributors get on board with best practices. The first blogpost is already up and sheds light on the <a href="https://cwiki.apache.org/confluence/display/FLINK/2020/03/22/Migrating+Flink%27s+CI+Infrastructure+from+Travis+CI+to+Azure+Pipelines">migration of Flink’s CI infrastructure from Travis to Azure Pipelines</a>.</p>
<h1 id="upcoming-events">Upcoming Events</h1>
<h2 id="flink-forward-virtual-conference">Flink Forward Virtual Conference</h2>
<p>The organization of Flink Forward had to make the hard decision of cancelling this year’s event in San Francisco. But all is not lost! <strong>Flink Forward SF will be held online on April 22-24 and you can register (for free)</strong> <a href="https://www.flink-forward.org/sf-2020">here</a>. Join the community for interactive talks and Q&amp;A sessions with core Flink contributors and companies like Splunk, Lyft, Netflix or Google.</p>
<h2 id="others">Others</h2>
<p>Events across the globe have come to a halt due to the growing concerns around COVID-19, so this time we’ll leave you with some interesting content to read instead. In addition to this written content, you can also recap last year’s sessions from <a href="https://www.youtube.com/playlist?list=PLDX4T_cnKjD207Aa8b5CsZjc7Z_KRezGz">Flink Forward Berlin</a> and <a href="https://www.youtube.com/playlist?list=PLDX4T_cnKjD3ANoNinSx3Au-poZTHvbF5">Flink Forward China</a>!</p>
<table class="table table-bordered">
<thead>
<tr>
<th>Type</th>
<th>Links</th>
</tr>
</thead>
<tbody>
<tr>
<td><span class="glyphicon glyphicon glyphicon-bookmark" aria-hidden="true"></span> Blogposts</td>
<td><ul>
<li><a href="https://medium.com/bird-engineering/replayable-process-functions-in-flink-time-ordering-and-timers-28007a0210e1">Replayable Process Functions: Time, Ordering, and Timers @Bird</a></li>
<li><a href="https://engineering.salesforce.com/application-log-intelligence-performance-insights-at-salesforce-using-flink-92955f30573f">Application Log Intelligence &amp; Performance Insights at Salesforce Using Flink @Salesforce</a></li>
</ul>
<ul>
<li><a href="https://flink.apache.org/news/2020/01/29/state-unlocked-interacting-with-state-in-apache-flink.html">State Unlocked: Interacting with State in Apache Flink</a></li>
<li><a href="https://flink.apache.org/news/2020/01/15/demo-fraud-detection.html">Advanced Flink Application Patterns Vol.1: Case Study of a Fraud Detection System</a></li>
<li><a href="https://flink.apache.org/news/2020/03/24/demo-fraud-detection-2.html">Advanced Flink Application Patterns Vol.2: Dynamic Updates of Application Logic</a></li>
<li><a href="https://flink.apache.org/ecosystem/2020/02/22/apache-beam-how-beam-runs-on-top-of-flink.html">Apache Beam: How Beam Runs on Top of Flink</a></li>
<li><a href="https://flink.apache.org/features/2020/03/27/flink-for-data-warehouse.html">Flink as Unified Engine for Modern Data Warehousing: Production-Ready Hive Integration</a></li>
</ul>
</td>
</tr>
<tr>
<td><span class="glyphicon glyphicon-console" aria-hidden="true"></span> Tutorials</td>
<td><ul>
<li><a href="https://medium.com/@zjffdu/flink-on-zeppelin-part-3-streaming-5fca1e16754">Flink on Zeppelin — (Part 3). Streaming</a></li>
<li><a href="https://aws.amazon.com/blogs/big-data/streaming-etl-with-apache-flink-and-amazon-kinesis-data-analytics">Streaming ETL with Apache Flink and Amazon Kinesis Data Analytics</a></li>
<li><a href="https://flink.apache.org/news/2020/02/20/ddl.html">No Java Required: Configuring Sources and Sinks in SQL</a></li>
<li><a href="https://flink.apache.org/news/2020/02/07/a-guide-for-unit-testing-in-apache-flink.html">A Guide for Unit Testing in Apache Flink</a></li>
</ul>
</td>
</tr>
</tbody>
</table>
<p>If you’d like to keep a closer eye on what’s happening in the community, subscribe to the Flink <a href="https://flink.apache.org/community.html#mailing-lists">@community mailing list</a> to get fine-grained weekly updates, upcoming event announcements and more.</p>
</article>
</div>
<div class="row">
<div id="disqus_thread"></div>
<script type="text/javascript">
/* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */
var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname
/* * * DON'T EDIT BELOW THIS LINE * * */
(function() {
var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
(document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
})();
</script>
</div>
</div>
</div>
</div>
</div>
<hr />
<div class="row">
<div class="footer text-center col-sm-12">
<p>Copyright © 2014-2019 <a href="http://apache.org">The Apache Software Foundation</a>. All Rights Reserved.</p>
<p>Apache Flink, Flink®, Apache®, the squirrel logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p>
<p><a href="/privacy-policy.html">Privacy Policy</a> &middot; <a href="/blog/feed.xml">RSS feed</a></p>
</div>
</div>
</div><!-- /.container -->
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.matchHeight/0.7.0/jquery.matchHeight-min.js"></script>
<script src="/js/codetabs.js"></script>
<script src="/js/stickysidebar.js"></script>
<!-- Google Analytics -->
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-52545728-1', 'auto');
ga('send', 'pageview');
</script>
</body>
</html>