| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> |
| <title>Apache Flink: Apache Flink in 2016: Year in Review</title> |
| <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> |
| <link rel="icon" href="/favicon.ico" type="image/x-icon"> |
| |
| <!-- Bootstrap --> |
| <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css"> |
| <link rel="stylesheet" href="/css/flink.css"> |
| <link rel="stylesheet" href="/css/syntax.css"> |
| |
| <!-- Blog RSS feed --> |
| <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> |
| |
| <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> |
| <!-- We need to load Jquery in the header for custom google analytics event tracking--> |
| <script src="/js/jquery.min.js"></script> |
| |
| <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> |
| <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> |
| <!--[if lt IE 9]> |
| <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script> |
| <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script> |
| <![endif]--> |
| </head> |
| <body> |
| |
| |
| <!-- Main content. --> |
| <div class="container"> |
| <div class="row"> |
| |
| |
| <div id="sidebar" class="col-sm-3"> |
| |
| |
| <!-- Top navbar. --> |
| <nav class="navbar navbar-default"> |
| <!-- The logo. --> |
| <div class="navbar-header"> |
| <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| </button> |
| <div class="navbar-logo"> |
| <a href="/"> |
| <img alt="Apache Flink" src="/img/flink-header-logo.svg" width="147px" height="73px"> |
| </a> |
| </div> |
| </div><!-- /.navbar-header --> |
| |
| <!-- The navigation links. --> |
| <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> |
| <ul class="nav navbar-nav navbar-main"> |
| |
| <!-- First menu section explains visitors what Flink is --> |
| |
| <!-- What is Stream Processing? --> |
| <!-- |
| <li><a href="/streamprocessing1.html">What is Stream Processing?</a></li> |
| --> |
| |
| <!-- What is Flink? --> |
| <li><a href="/flink-architecture.html">What is Apache Flink?</a></li> |
| |
| |
| |
| <!-- What is Stateful Functions? --> |
| |
| <li><a href="/stateful-functions.html">What is Stateful Functions?</a></li> |
| |
| <!-- Use cases --> |
| <li><a href="/usecases.html">Use Cases</a></li> |
| |
| <!-- Powered by --> |
| <li><a href="/poweredby.html">Powered By</a></li> |
| |
| |
| |
| <!-- Second menu section aims to support Flink users --> |
| |
| <!-- Downloads --> |
| <li><a href="/downloads.html">Downloads</a></li> |
| |
| <!-- Getting Started --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Getting Started<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.10/getting-started/index.html" target="_blank">With Flink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1/getting-started/project-setup.html" target="_blank">With Flink Stateful Functions <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="/training.html">Training Course</a></li> |
| </ul> |
| </li> |
| |
| <!-- Documentation --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.10" target="_blank">Flink 1.10 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-master" target="_blank">Flink Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1" target="_blank">Flink Stateful Functions 2.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-master" target="_blank">Flink Stateful Functions Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| </ul> |
| </li> |
| |
| <!-- getting help --> |
| <li><a href="/gettinghelp.html">Getting Help</a></li> |
| |
| <!-- Blog --> |
| <li class="active"><a href="/blog/"><b>Flink Blog</b></a></li> |
| |
| |
| <!-- Flink-packages --> |
| <li> |
| <a href="https://flink-packages.org" target="_blank">flink-packages.org <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| <!-- Third menu section aim to support community and contributors --> |
| |
| <!-- Community --> |
| <li><a href="/community.html">Community & Project Info</a></li> |
| |
| <!-- Roadmap --> |
| <li><a href="/roadmap.html">Roadmap</a></li> |
| |
| <!-- Contribute --> |
| <li><a href="/contributing/how-to-contribute.html">How to Contribute</a></li> |
| |
| |
| <!-- GitHub --> |
| <li> |
| <a href="https://github.com/apache/flink" target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| |
| <!-- Language Switcher --> |
| <li> |
| |
| |
| <!-- link to the Chinese home page when current is blog page --> |
| <a href="/zh">中文版</a> |
| |
| |
| </li> |
| |
| </ul> |
| |
| <ul class="nav navbar-nav navbar-bottom"> |
| <hr /> |
| |
| <!-- Twitter --> |
| <li><a href="https://twitter.com/apacheflink" target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <!-- Visualizer --> |
| <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <hr /> |
| |
| <li><a href="https://apache.org" target="_blank">Apache Software Foundation <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <li> |
| <style> |
| .smalllinks:link { |
| display: inline-block !important; background: none; padding-top: 0px; padding-bottom: 0px; padding-right: 0px; min-width: 75px; |
| } |
| </style> |
| |
| <a class="smalllinks" href="https://www.apache.org/licenses/" target="_blank">License</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/security/" target="_blank">Security</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/sponsorship.html" target="_blank">Donate</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| </li> |
| |
| </ul> |
| </div><!-- /.navbar-collapse --> |
| </nav> |
| |
| </div> |
| <div class="col-sm-9"> |
| <div class="row-fluid"> |
| <div class="col-sm-12"> |
| <div class="row"> |
| <h1>Apache Flink in 2016: Year in Review</h1> |
| <p><i></i></p> |
| |
| <article> |
| <p>19 Dec 2016 by Mike Winters</p> |
| |
| <p>2016 was an exciting year for the Apache Flink® community, and the |
| <a href="http://flink.apache.org/news/2016/03/08/release-1.0.0.html" target="_blank">release of Flink 1.0 in March</a> |
| marked the first time in Flink’s history that the community guaranteed API backward compatibility for all |
| versions in a series. This step forward for Flink was followed by many new and exciting production deployments |
| in organizations of all shapes and sizes, all around the globe.</p> |
| |
| <p>In this post, we’ll look back on the project’s progress over the course of 2016, and |
| we’ll also preview what 2017 has in store.</p> |
| |
| <div class="page-toc"> |
| <ul id="markdown-toc"> |
| <li><a href="#community-growth" id="markdown-toc-community-growth">Community Growth</a> <ul> |
| <li><a href="#github" id="markdown-toc-github">Github</a></li> |
| <li><a href="#meetups" id="markdown-toc-meetups">Meetups</a></li> |
| </ul> |
| </li> |
| <li><a href="#flink-forward-2016" id="markdown-toc-flink-forward-2016">Flink Forward 2016</a></li> |
| <li><a href="#features-and-ecosystem" id="markdown-toc-features-and-ecosystem">Features and Ecosystem</a> <ul> |
| <li><a href="#flink-ecosystem-growth" id="markdown-toc-flink-ecosystem-growth">Flink Ecosystem Growth</a></li> |
| <li><a href="#feature-timeline-in-2016" id="markdown-toc-feature-timeline-in-2016">Feature Timeline in 2016</a></li> |
| </ul> |
| </li> |
| <li><a href="#looking-ahead-to-2017" id="markdown-toc-looking-ahead-to-2017">Looking ahead to 2017</a></li> |
| </ul> |
| |
| </div> |
| |
| <h2 id="community-growth">Community Growth</h2> |
| |
| <h3 id="github">Github</h3> |
| <p>First, here’s a summary of community statistics from <a href="https://github.com/apache/flink" target="_blank">GitHub</a>. At the time of writing:</p> |
| <ul> |
| <li><b>Contributors</b> have increased from 150 in December 2015 to 258 in December 2016 (up <b>72%</b>)</li> |
| <li><b>Stars</b> have increased from 813 in December 2015 to 1830 in December 2016 (up <b>125%</b>)</li> |
| <li><b>Forks</b> have increased from 544 in December 2015 to 1255 in December 2016 (up <b>130%</b>)</li> |
| </ul> |
| |
| <p>The community also welcomed <b>3 new committers in 2016</b>: Chengxiang Li, Greg Hogan, and Tzu-Li (Gordon) Tai.</p> |
| |
| <p><br /><img src="/img/blog/github-stats-2016.png" width="775" alt="Apache Flink GitHub Stats" /> |
| <br /> |
| <br /></p> |
| |
| <p>Next, let’s take a look at a few other project stats, starting with number of commits. If we run:</p> |
| |
| <div class="highlight"><pre><code>git log --pretty=oneline --after=12/31/2015 | wc -l |
| </code></pre></div> |
| <p>…inside the Flink repository, we’ll see a total of <strong>1884</strong> commits so far in 2016, bringing the all-time total commits to <strong>10,015</strong>.</p> |
| |
| <p>Now, let’s go a bit deeper. And here are instructions in case you’d like to take a look at this data yourself.</p> |
| |
| <ul> |
| <li>Download gitstats from the <a href="http://gitstats.sourceforge.net/">project homepage</a>. Or, on OS X with homebrew, type:</li> |
| </ul> |
| |
| <div class="highlight"><pre><code>brew install --HEAD homebrew/head-only/gitstats |
| </code></pre></div> |
| |
| <ul> |
| <li>Clone the Apache Flink git repository:</li> |
| </ul> |
| |
| <div class="highlight"><pre><code>git clone git@github.com:apache/flink.git |
| </code></pre></div> |
| |
| <ul> |
| <li>Generate the statistics</li> |
| </ul> |
| |
| <div class="highlight"><pre><code>gitstats flink/ flink-stats/ |
| </code></pre></div> |
| |
| <ul> |
| <li>View all the statistics as an html page using your defaulf browser:</li> |
| </ul> |
| |
| <div class="highlight"><pre><code>open flink-stats/index.html |
| </code></pre></div> |
| <p>2016 is the year that Flink surpassed 1 million lines of code, now clocking in at <strong>1,034,137</strong> lines.</p> |
| |
| <p><img src="/img/blog/flink-lines-of-code-2016.png" align="center" width="550" alt="Flink Total Lines of Code" /></p> |
| |
| <p>Monday remains the day of the week with the most commits over the project’s history:</p> |
| |
| <p><img src="/img/blog/flink-dow-2016.png" align="center" width="550" alt="Flink Commits by Day of Week" /></p> |
| |
| <p>And 5pm is still solidly the preferred commit time:</p> |
| |
| <p><img src="/img/blog/flink-hod-2016.png" align="center" width="550" alt="Flink Commits by Hour of Day" /></p> |
| |
| <p><br /></p> |
| |
| <h3 id="meetups">Meetups</h3> |
| <p><a href="https://www.meetup.com/topics/apache-flink/" target="_blank">Apache Flink Meetup membership</a> grew by <b>240%</b> |
| this year, and at the time of writing, there are 41 meetups comprised of 16,541 members listing Flink as a topic–up from 16 groups with 4,864 members in December 2015. |
| The Flink community is proud to be truly global in nature.</p> |
| |
| <p><img src="/img/blog/flink-meetups-dec2016.png" width="775" alt="Apache Flink Meetup Map" /></p> |
| |
| <h2 id="flink-forward-2016">Flink Forward 2016</h2> |
| |
| <p>The <a href="http://2016.flink-forward.org/" target="_blank">second annual Flink Forward conference </a>took place in |
| Berlin on September 12-14, and over 350 members of the Flink community came together for speaker sessions, training, |
| and discussion about Flink. <a href="http://2016.flink-forward.org/program/sessions/" target="_blank">Slides and videos</a> |
| from speaker sessions are available online, and we encourage you to take a look if you’re interested in learning more |
| about how Flink is used in production in a wide range of organizations.</p> |
| |
| <p>Flink Forward will be expanding to <a href="http://sf.flink-forward.org/" target="_blank">San Francisco in April 2017</a>, and the <a href="http://berlin.flink-forward.org/" target="_blank">third-annual Berlin event |
| is scheduled for September 2017.</a></p> |
| |
| <p><img src="/img/blog/speaker-logos-ff2016.png" width="775" alt="Flink Forward Speakers" /></p> |
| |
| <h2 id="features-and-ecosystem">Features and Ecosystem</h2> |
| |
| <h3 id="flink-ecosystem-growth">Flink Ecosystem Growth</h3> |
| |
| <p>Flink was added to a selection of distributions during 2016, making it easier |
| for an even larger base of users to start working with Flink:</p> |
| |
| <ul> |
| <li><a href="https://aws.amazon.com/blogs/big-data/use-apache-flink-on-amazon-emr/" target="_blank"> |
| Amazon EMR</a></li> |
| <li><a href="https://cloud.google.com/dataproc/docs/release-notes/service#november_29_2016" target="_blank"> |
| Google Cloud Dataproc</a></li> |
| <li><a href="https://www.lightbend.com/blog/introducing-lightbend-fast-data-platform" target="_blank"> |
| Lightbend Fast Data Platform</a></li> |
| </ul> |
| |
| <p>In addition, the Apache Beam and Flink communities teamed up to build a Flink runner for Beam that, according to the Google team, is <a href="https://cloud.google.com/blog/big-data/2016/05/why-apache-beam-a-google-perspective" target="_blank">“sophisticated enough to be a compelling alternative to Cloud Dataflow when running on premise or on non-Google clouds”</a>.</p> |
| |
| <h3 id="feature-timeline-in-2016">Feature Timeline in 2016</h3> |
| |
| <p>Here’s a selection of major features added to Flink over the course of 2016:</p> |
| |
| <p><img src="/img/blog/flink-releases-2016.png" width="775" alt="Flink Release Timeline 2016" /></p> |
| |
| <p>If you spend time in the <a href="https://issues.apache.org/jira/browse/FLINK-4554?jql=project%20%3D%20FLINK%20AND%20issuetype%20%3D%20%22New%20Feature%22%20AND%20status%20%3D%20Resolved%20ORDER%20BY%20resolved%20DESC" target="_blank">Apache Flink JIRA project</a>, you’ll see that the Flink community has addressed every single one of the roadmap items identified |
| in <a href="http://flink.apache.org/news/2015/12/18/a-year-in-review.html" target="_blank">2015’s year in review post</a>. Here’s to making that an annual tradition. :)</p> |
| |
| <h2 id="looking-ahead-to-2017">Looking ahead to 2017</h2> |
| |
| <p>A good source of information about the Flink community’s roadmap is the list of |
| <a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Improvement+Proposals" target="_blank">Flink |
| Improvement Proposals (FLIPs)</a> in the project wiki. Below, we’ll highlight a selection of FLIPs |
| that have been accepted by the community as well as some that are still under discussion.</p> |
| |
| <p>We should note that work is already underway on a number of these features, and some will even be included in Flink 1.2 at the beginning of 2017.</p> |
| |
| <ul> |
| <li> |
| <p><strong>A new Flink deployment and process model</strong>, as described in <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=65147077" target="_blank">FLIP-6<a></a>. This work ensures that Flink supports a wide |
| range of deployment types and cluster managers, making it possible to run Flink smoothly in any environment.</a></p> |
| </li> |
| <li> |
| <p><strong>Dynamic scaling</strong> for both key-value state <a href="https://github.com/apache/flink/pull/2440" target="_blank">(as described in |
| this PR)<a></a> <em>and</em> non-partitioned state <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-8%3A+Rescalable+Non-Partitioned+State" target="_blank">(as described in FLIP-8)<a></a>, ensuring that it’s always possible to split or merge state when scaling up or down, respectively.</a></a></p> |
| </li> |
| <li> |
| <p><strong>Asynchronous I/O</strong>, as described in <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=65870673" target="_blank">FLIP-12 |
| </a>, which makes I/O access a less time-consuming process without adding complexity or the need for extra checkpoint coordination.</p> |
| </li> |
| <li> |
| <p><strong>Enhancements to the window evictor</strong>, as described in <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-4+%3A+Enhance+Window+Evictor" target="_blank">FLIP-4</a>, |
| to provide users with more control over how elements are evicted from a window.</p> |
| </li> |
| <li> |
| <p><strong>Fined-grained recovery from task failures</strong>, as described in <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-1+%3A+Fine+Grained+Recovery+from+Task+Failures" target="_blank">FLIP-1</a>, |
| to make it possible to restart only what needs to be restarted during recovery, building on cached intermediate results.</p> |
| </li> |
| <li> |
| <p><strong>Unified checkpoints and savepoints</strong>, as described in <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-10%3A+Unify+Checkpoints+and+Savepoints" target="_blank">FLIP-10</a>, to |
| allow savepoints to be triggered automatically–important for program updates for the sake of error handling because savepoints allow the user to modify both |
| the job and Flink version whereas checkpoints can only be recovered with the same job.</p> |
| </li> |
| <li> |
| <p><strong>Table API window aggregations</strong>, as described in <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-11%3A+Table+API+Stream+Aggregations" target="_blank">FLIP-11</a>, to support group-window and row-window aggregates on streaming and batch tables.</p> |
| </li> |
| <li> |
| <p><strong>Side inputs</strong>, as described in <a href="https://docs.google.com/document/d/1hIgxi2Zchww_5fWUHLoYiXwSBXjv-M5eOv-MKQYN3m4/edit" target="_blank">this design document</a>, to |
| enable the joining of a main, high-throughput stream with one more more inputs with static or slowly-changing data.</p> |
| </li> |
| </ul> |
| |
| <p>If you’re interested in getting involved with Flink, we encourage you to take a look at the FLIPs and to join the discussion via the <a href="http://flink.apache.org/community.html#mailing-lists">Flink mailing lists</a>.</p> |
| |
| <p>Lastly, we’d like to extend a sincere thank you to all of the Flink community for making 2016 a great year!</p> |
| |
| </article> |
| </div> |
| |
| <div class="row"> |
| <div id="disqus_thread"></div> |
| <script type="text/javascript"> |
| /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ |
| var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname |
| |
| /* * * DON'T EDIT BELOW THIS LINE * * */ |
| (function() { |
| var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; |
| dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; |
| (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); |
| })(); |
| </script> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| <hr /> |
| |
| <div class="row"> |
| <div class="footer text-center col-sm-12"> |
| <p>Copyright © 2014-2019 <a href="http://apache.org">The Apache Software Foundation</a>. All Rights Reserved.</p> |
| <p>Apache Flink, Flink®, Apache®, the squirrel logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> |
| <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> |
| </div> |
| </div> |
| </div><!-- /.container --> |
| |
| <!-- Include all compiled plugins (below), or include individual files as needed --> |
| <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.matchHeight/0.7.0/jquery.matchHeight-min.js"></script> |
| <script src="/js/codetabs.js"></script> |
| <script src="/js/stickysidebar.js"></script> |
| |
| <!-- Google Analytics --> |
| <script> |
| (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ |
| (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), |
| m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) |
| })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); |
| |
| ga('create', 'UA-52545728-1', 'auto'); |
| ga('send', 'pageview'); |
| </script> |
| </body> |
| </html> |
