| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> |
| <title>Apache Flink: Apache Flink in 2017: Year in Review</title> |
| <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> |
| <link rel="icon" href="/favicon.ico" type="image/x-icon"> |
| |
| <!-- Bootstrap --> |
| <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css"> |
| <link rel="stylesheet" href="/css/flink.css"> |
| <link rel="stylesheet" href="/css/syntax.css"> |
| |
| <!-- Blog RSS feed --> |
| <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> |
| |
| <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> |
| <!-- We need to load Jquery in the header for custom google analytics event tracking--> |
| <script src="/js/jquery.min.js"></script> |
| |
| <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> |
| <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> |
| <!--[if lt IE 9]> |
| <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script> |
| <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script> |
| <![endif]--> |
| </head> |
| <body> |
| |
| |
| <!-- Main content. --> |
| <div class="container"> |
| <div class="row"> |
| |
| |
| <div id="sidebar" class="col-sm-3"> |
| |
| |
| <!-- Top navbar. --> |
| <nav class="navbar navbar-default"> |
| <!-- The logo. --> |
| <div class="navbar-header"> |
| <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| </button> |
| <div class="navbar-logo"> |
| <a href="/"> |
| <img alt="Apache Flink" src="/img/flink-header-logo.svg" width="147px" height="73px"> |
| </a> |
| </div> |
| </div><!-- /.navbar-header --> |
| |
| <!-- The navigation links. --> |
| <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> |
| <ul class="nav navbar-nav navbar-main"> |
| |
| <!-- First menu section explains visitors what Flink is --> |
| |
| <!-- What is Stream Processing? --> |
| <!-- |
| <li><a href="/streamprocessing1.html">What is Stream Processing?</a></li> |
| --> |
| |
| <!-- What is Flink? --> |
| <li><a href="/flink-architecture.html">What is Apache Flink?</a></li> |
| |
| |
| |
| <!-- What is Stateful Functions? --> |
| |
| <li><a href="/stateful-functions.html">What is Stateful Functions?</a></li> |
| |
| <!-- Use cases --> |
| <li><a href="/usecases.html">Use Cases</a></li> |
| |
| <!-- Powered by --> |
| <li><a href="/poweredby.html">Powered By</a></li> |
| |
| |
| |
| <!-- Second menu section aims to support Flink users --> |
| |
| <!-- Downloads --> |
| <li><a href="/downloads.html">Downloads</a></li> |
| |
| <!-- Getting Started --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Getting Started<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/getting-started/index.html" target="_blank">With Flink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1/getting-started/project-setup.html" target="_blank">With Flink Stateful Functions <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="/training.html">Training Course</a></li> |
| </ul> |
| </li> |
| |
| <!-- Documentation --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11" target="_blank">Flink 1.11 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-master" target="_blank">Flink Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1" target="_blank">Flink Stateful Functions 2.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-master" target="_blank">Flink Stateful Functions Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| </ul> |
| </li> |
| |
| <!-- getting help --> |
| <li><a href="/gettinghelp.html">Getting Help</a></li> |
| |
| <!-- Blog --> |
| <li class="active"><a href="/blog/"><b>Flink Blog</b></a></li> |
| |
| |
| <!-- Flink-packages --> |
| <li> |
| <a href="https://flink-packages.org" target="_blank">flink-packages.org <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| <!-- Third menu section aim to support community and contributors --> |
| |
| <!-- Community --> |
| <li><a href="/community.html">Community & Project Info</a></li> |
| |
| <!-- Roadmap --> |
| <li><a href="/roadmap.html">Roadmap</a></li> |
| |
| <!-- Contribute --> |
| <li><a href="/contributing/how-to-contribute.html">How to Contribute</a></li> |
| |
| |
| <!-- GitHub --> |
| <li> |
| <a href="https://github.com/apache/flink" target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| |
| <!-- Language Switcher --> |
| <li> |
| |
| |
| <!-- link to the Chinese home page when current is blog page --> |
| <a href="/zh">中文版</a> |
| |
| |
| </li> |
| |
| </ul> |
| |
| <ul class="nav navbar-nav navbar-bottom"> |
| <hr /> |
| |
| <!-- Twitter --> |
| <li><a href="https://twitter.com/apacheflink" target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <!-- Visualizer --> |
| <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <hr /> |
| |
| <li><a href="https://apache.org" target="_blank">Apache Software Foundation <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <li> |
| <style> |
| .smalllinks:link { |
| display: inline-block !important; background: none; padding-top: 0px; padding-bottom: 0px; padding-right: 0px; min-width: 75px; |
| } |
| </style> |
| |
| <a class="smalllinks" href="https://www.apache.org/licenses/" target="_blank">License</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/security/" target="_blank">Security</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/sponsorship.html" target="_blank">Donate</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| </li> |
| |
| </ul> |
| </div><!-- /.navbar-collapse --> |
| </nav> |
| |
| </div> |
| <div class="col-sm-9"> |
| <div class="row-fluid"> |
| <div class="col-sm-12"> |
| <div class="row"> |
| <h1>Apache Flink in 2017: Year in Review</h1> |
| <p><i></i></p> |
| |
| <article> |
| <p>21 Dec 2017 Chris Ward (<a href="https://twitter.com/chrischinch">@chrischinch</a>) & Mike Winters (<a href="https://twitter.com/wints">@wints</a>)</p> |
| |
| <p>2017 was another exciting year for the Apache Flink® community, with 3 major version releases (<a href="http://flink.apache.org/news/2017/02/06/release-1.2.0.html">Flink 1.2.0 in February</a>, <a href="http://flink.apache.org/news/2017/06/01/release-1.3.0.html">Flink 1.3.0 in June</a>, and <a href="http://flink.apache.org/news/2017/12/12/release-1.4.0.html">Flink 1.4.0 in December</a>) and the first-ever <a href="https://sf-2017.flink-forward.org/">Flink Forward in San Francisco</a>, giving Flink community members in another corner of the globe an opportunity to connect. Users shared details about their innovative production deployments, redefining what is possible with a modern stream processing framework like Flink.</p> |
| |
| <p>In this post, we’ll look back on the project’s progress over the course of 2017, and we’ll also preview what 2018 has in store.</p> |
| |
| <div class="page-toc"> |
| <ul id="markdown-toc"> |
| <li><a href="#community-growth" id="markdown-toc-community-growth">Community Growth</a> <ul> |
| <li><a href="#github" id="markdown-toc-github">Github</a></li> |
| <li><a href="#meetups" id="markdown-toc-meetups">Meetups</a></li> |
| </ul> |
| </li> |
| <li><a href="#flink-forward-2017" id="markdown-toc-flink-forward-2017">Flink Forward 2017</a></li> |
| <li><a href="#features-and-ecosystem" id="markdown-toc-features-and-ecosystem">Features and Ecosystem</a> <ul> |
| <li><a href="#flink-ecosystem-growth" id="markdown-toc-flink-ecosystem-growth">Flink Ecosystem Growth</a></li> |
| <li><a href="#feature-timeline-in-2017" id="markdown-toc-feature-timeline-in-2017">Feature Timeline in 2017</a></li> |
| </ul> |
| </li> |
| <li><a href="#looking-ahead-to-2018" id="markdown-toc-looking-ahead-to-2018">Looking ahead to 2018</a></li> |
| </ul> |
| |
| </div> |
| |
| <h2 id="community-growth">Community Growth</h2> |
| |
| <h3 id="github">Github</h3> |
| |
| <p>First, here’s a summary of community statistics from <a href="https://github.com/apache/flink">GitHub</a>. At the time of writing:</p> |
| |
| <ul> |
| <li><strong>Contributors</strong> have increased from 258 in December 2016 to 352 in December 2017 (up <strong>36%</strong>)</li> |
| <li><strong>Stars</strong> have increased from 1830 in December 2016 to 3036 in December 2017 (up <strong>65%</strong>)</li> |
| <li><strong>Forks</strong> have increased from 1255 in December 2016 to 2070 in December 2017 (up <strong>65%</strong>)</li> |
| </ul> |
| |
| <p>The community also welcomed <strong>10 new committers in 2017</strong>: Kostas Kloudas, Jark Wu, Stefan Richter, Kurt Young, Theodore Vasiloudis, Xiaogang Shi, Dawid Wysakowicz, Shaoxuan Wang, Jincheng Sun and Haohui Mai.</p> |
| |
| <p>We also welcomed <strong>3 new members to the <a href="http://www.apache.org/foundation/governance/pmcs.html">project management committee (PMC)</a></strong>: Greg Hogan, Tzu-Li (Gordon) Tai and Chesnay Schepler.</p> |
| |
| <p><img src="/img/blog/github-stats-2017.png" alt="Apache Flink GitHub Stats" /></p> |
| |
| <p>Next, let’s take a look at a few other project stats, starting with number of commits. If we run:</p> |
| |
| <div class="highlight"><pre><code class="language-bash">git log --pretty<span class="o">=</span>oneline --after<span class="o">=</span>12/31/2016 <span class="p">|</span> wc -l</code></pre></div> |
| |
| <p>Inside the Flink repository, we’ll see a total of <strong>2316</strong> commits so far in 2017, bringing the all-time total commits to <strong>12,532</strong>.</p> |
| |
| <p>Now, let’s go a bit deeper, here are instructions to take a look at this data yourself.</p> |
| |
| <p>Download and install gitstats from the <a href="http://gitstats.sourceforge.net/">project homepage</a>, then clone the Apache Flink git repository:</p> |
| |
| <div class="highlight"><pre><code class="language-bash">git clone git@github.com:apache/flink.git</code></pre></div> |
| |
| <p>Generate the statistics</p> |
| |
| <div class="highlight"><pre><code class="language-bash">gitstats flink/ flink-stats/</code></pre></div> |
| |
| <p>View all the statistics as an HTML page using your default browser:</p> |
| |
| <div class="highlight"><pre><code class="language-bash">open flink-stats/index.html</code></pre></div> |
| |
| <p>Flink surpassed 1 million lines of code in 2016, and that trend continued in 2017 with the code base now clocking in at <strong>1,257,949</strong> lines.</p> |
| |
| <p><img src="/img/blog/flink-lines-of-code-2017.png" alt="Flink Total Lines of Code" /></p> |
| |
| <p>Monday remains the day of the week with the most commits over the project’s history, but Wednesday is catching up:</p> |
| |
| <p><img src="/img/blog/flink-dow-2017.png" alt="Flink Commits by Day of Week" /></p> |
| |
| <p>5 pm remains the preferred commit time, closely followed by 4 pm:</p> |
| |
| <p><img src="/img/blog/flink-hod-2017.png" alt="Flink Commits by Hour of Day" /></p> |
| |
| <h3 id="meetups">Meetups</h3> |
| |
| <p><a href="https://www.meetup.com/topics/apache-flink/">Apache Flink Meetup membership</a> grew by <strong>20%</strong> this year to a total of <strong>19,767</strong> members at <strong>39</strong> meetups listing Flink as a topic. With meetups on five continents, the Flink community is proud to be truly global.</p> |
| |
| <p><img src="/img/blog/flink-meetups-dec2017.png" alt="Apache Flink Meetup Map" /></p> |
| |
| <h2 id="flink-forward-2017">Flink Forward 2017</h2> |
| |
| <p>2017 was the first year we ran a Flink Forward conference in both <a href="https://berlin-2017.flink-forward.org">Berlin</a> (September 11-13) and <a href="https://sf-2017.flink-forward.org">San Francisco</a> (April 10-11), and over 350 members of our community attended each event for speaker sessions, training, and discussion about Flink.</p> |
| |
| <p><a href="https://www.slideshare.net/FlinkForward/presentations">Slides</a> and <a href="https://www.youtube.com/channel/UCY8_lgiZLZErZPF47a2hXMA">videos</a> are available for all speaker sessions, and if you’re interested in learning more about how organizations use Flink in production, we encourage you to browse and watch a couple.</p> |
| |
| <p>For 2018, Flink Forward will be back in <a href="https://flink-forward.org/">September in Berlin</a>, and in <a href="https://sf-2018.flink-forward.org/">April in San Francisco</a>.</p> |
| |
| <p><img src="/img/blog/speaker-logos-ff2017.png" alt="Flink Forward Speakers" /></p> |
| |
| <h2 id="features-and-ecosystem">Features and Ecosystem</h2> |
| |
| <h3 id="flink-ecosystem-growth">Flink Ecosystem Growth</h3> |
| |
| <p>Flink was added to a selection of distributions and integrations during 2017, making it easier for a wider user base to get started with Flink:</p> |
| |
| <ul> |
| <li><a href="https://hub.docker.com/r/_/flink/">Official Docker image</a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.3/setup/mesos.html">Official DC/OS and Mesos support</a></li> |
| <li><a href="https://data-artisans.com/blog/dellemc-launches-open-source-pravega-complete-apache-flink-connector">A Flink connector</a> for <a href="http://pravega.io">Pravega</a>, Dell/EMC’s streaming storage system.</li> |
| <li>Uber announced AthenaX, a streaming SQL platform <a href="https://data-artisans.com/blog/uber-introduces-open-source-athenax-streaming-sql-platform-apache-flink">powered by Apache Flink</a>.</li> |
| <li>dataArtisans announced an early access program of a SaaS product based on Apache Flink, <a href="https://data-artisans.com/blog/da-platform-2-stateful-stream-processing-with-apache-flink-made-easier">dA Platform 2</a>.</li> |
| </ul> |
| |
| <h3 id="feature-timeline-in-2017">Feature Timeline in 2017</h3> |
| |
| <p>Just in time for the end of the year, our 1.4 release <a href="http://flink.apache.org/news/2017/12/12/release-1.4.0.html">read the full release announcement</a> landed in mid-December culminating 5 months of work and the resolution of more than 900 issues. This is the fifth major release in the 1.x.y series.</p> |
| |
| <p>Here’s a selection of major features added to Flink over the course of 2017:</p> |
| |
| <p><img src="/img/blog/flink-releases-2017.png" alt="Flink Release Timeline 2017" /></p> |
| |
| <p>If you take a look at <a href="https://issues.apache.org/jira/browse/FLINK-5016?jql=project%20%3D%20FLINK%20AND%20issuetype%20in%20(Bug%2C%20Improvement%2C%20%22New%20Feature%22)%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20resolved%20%3E%3D%202017-01-01%20AND%20resolved%20%3C%3D%202017-12-31%20ORDER%20BY%20resolved%20ASC">the resolved issues and enhancements for 2017 on Jira</a> you can see that the community resolved over 1,831 issues and feature additions.</p> |
| |
| <p><a href="/news/2016/12/19/2016-year-in-review.html#looking-ahead-to-2017">Regarding roadmap commitments from 2016</a>, there is mixed news, with some items a part of current releases, others scheduled for upcoming releases and some that remain under discussion.</p> |
| |
| <h2 id="looking-ahead-to-2018">Looking ahead to 2018</h2> |
| |
| <p>A good source of information about the Flink community’s roadmap is the list of <a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Improvement+Proposals">Flink Improvement Proposals (FLIPs)</a> in the project wiki. Below, we’ll highlight a selection of FLIPs accepted by the community as well as some that are still under discussion.</p> |
| |
| <p>Work is already underway on a number of these features, and some will be included in Flink 1.5 at the beginning of 2018.</p> |
| |
| <ul> |
| <li><strong>Improved BLOB storage architecture</strong>, as described in <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-19:+Improved+BLOB+storage+architecture">FLIP-19</a> to consolidate API usage and improve concurrency.</li> |
| <li><strong>Integration of SQL and CEP</strong>, as described in <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-20:+Integration+of+SQL+and+CEP">FLIP-20</a> to allow developers to create complex event processing (CEP) patterns using SQL statements.</li> |
| <li><strong>Unified checkpoints and savepoints</strong>, as described in <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-10:+Unify+Checkpoints+and+Savepoints">FLIP-10</a>, to allow savepoints to be triggered automatically–important for program updates for the sake of error handling because savepoints allow the user to modify both the job and Flink version whereas checkpoints can only be recovered with the same job.</li> |
| <li><strong>An improved Flink deployment and process model</strong>, as described in <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=65147077">FLIP-6</a>, to allow for better integration with Flink and cluster managers and deployment technologies such as Mesos, Docker, and Kubernetes.</li> |
| <li><strong>Fine-grained recovery from task failures</strong>, as described in <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-1+:+Fine+Grained+Recovery+from+Task+Failures">FLIP-1</a> to improve recovery efficiency and only re-execute failed tasks, reducing the amount of state that Flink needs to transfer on recovery.</li> |
| <li><strong>An SQL Client</strong>, as described in <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-24+-+SQL+Client">FLIP-24</a> to add a service and a client to execute SQL queries against batch and streaming tables.</li> |
| <li><strong>Serving of machine learning models</strong>, as described in <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-23+-+Model+Serving">FLIP-23</a> to add a library that allows users to apply offline-trained machine learning models to data streams.</li> |
| </ul> |
| |
| <p>If you’re interested in getting involved with Flink, we encourage you to take a look at the FLIPs and to join the discussion via the <a href="http://flink.apache.org/community.html#mailing-lists">Flink mailing lists</a>.</p> |
| |
| <p>Lastly, we’d like to extend a sincere thank you to all the Flink community for making 2017 a great year!</p> |
| |
| </article> |
| </div> |
| |
| <div class="row"> |
| <div id="disqus_thread"></div> |
| <script type="text/javascript"> |
| /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ |
| var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname |
| |
| /* * * DON'T EDIT BELOW THIS LINE * * */ |
| (function() { |
| var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; |
| dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; |
| (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); |
| })(); |
| </script> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| <hr /> |
| |
| <div class="row"> |
| <div class="footer text-center col-sm-12"> |
| <p>Copyright © 2014-2019 <a href="http://apache.org">The Apache Software Foundation</a>. All Rights Reserved.</p> |
| <p>Apache Flink, Flink®, Apache®, the squirrel logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> |
| <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> |
| </div> |
| </div> |
| </div><!-- /.container --> |
| |
| <!-- Include all compiled plugins (below), or include individual files as needed --> |
| <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.matchHeight/0.7.0/jquery.matchHeight-min.js"></script> |
| <script src="/js/codetabs.js"></script> |
| <script src="/js/stickysidebar.js"></script> |
| |
| <!-- Google Analytics --> |
| <script> |
| (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ |
| (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), |
| m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) |
| })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); |
| |
| ga('create', 'UA-52545728-1', 'auto'); |
| ga('send', 'pageview'); |
| </script> |
| </body> |
| </html> |
