Google Git
Sign in
apache / flink-web / 0f9478d1de8c2317e67a6a63df5961eba5bbc8db / . / content / 2015 / 12 / 18 / flink-2015-a-year-in-review-and-a-lookout-to-2016 / index.html
blob: 065d2d9013d4a7ab6463939080578f8fe1b922ca [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" dir=ZgotmplZ>
<head>
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
<script src="/bootstrap/js/bootstrap.bundle.min.js"></script>
<link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css">
<script src="/js/anchor.min.js"></script>
<script src="/js/flink.js"></script>
<link rel="canonical" href="https://flink.apache.org/2015/12/18/flink-2015-a-year-in-review-and-a-lookout-to-2016/">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="With 2015 ending, we thought that this would be good time to reflect on the amazing work done by the Flink community over this past year, and how much this community has grown.
Overall, we have seen Flink grow in terms of functionality from an engine to one of the most complete open-source stream processing frameworks available. The community grew from a relatively small and geographically focused team, to a truly global, and one of the largest big data communities in the the Apache Software Foundation.">
<meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="Flink 2015: A year in review, and a lookout to 2016" />
<meta property="og:description" content="With 2015 ending, we thought that this would be good time to reflect on the amazing work done by the Flink community over this past year, and how much this community has grown.
Overall, we have seen Flink grow in terms of functionality from an engine to one of the most complete open-source stream processing frameworks available. The community grew from a relatively small and geographically focused team, to a truly global, and one of the largest big data communities in the the Apache Software Foundation." />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://flink.apache.org/2015/12/18/flink-2015-a-year-in-review-and-a-lookout-to-2016/" /><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2015-12-18T10:00:00+00:00" />
<meta property="article:modified_time" content="2015-12-18T10:00:00+00:00" />
<title>Flink 2015: A year in review, and a lookout to 2016 | Apache Flink</title>
<link rel="manifest" href="/manifest.json">
<link rel="icon" href="/favicon.png" type="image/x-icon">
<link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU=">
<script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script>
<!--
Made with Book Theme
https://github.com/alex-shpak/hugo-book
-->
<meta name="generator" content="Hugo 0.124.1">
<script>
var _paq = window._paq = window._paq || [];
_paq.push(['disableCookies']);
_paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="//analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '1']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
</head>
<body dir=ZgotmplZ>
<header>
<nav class="navbar navbar-expand-xl">
<div class="container-fluid">
<a class="navbar-brand" href="/">
<img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle">
<span>Apache Flink</span>
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<i class="fa fa-bars navbar-toggler-icon"></i>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/security/">Security</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a>
</li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link" href="/posts/">Flink Blog</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/downloads/">Downloads</a>
</li>
</ul>
<div class="book-search">
<div class="book-search-spinner hidden">
<i class="fa fa-refresh fa-spin"></i>
</div>
<form class="search-bar d-flex" onsubmit="return false;"su>
<input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/">
<i class="fa fa-search search"></i>
<i class="fa fa-circle-o-notch fa-spin spinner"></i>
</form>
<div class="book-search-spinner hidden"></div>
<ul id="book-search-results"></ul>
</div>
</div>
</div>
</nav>
<div class="navbar-clearfix"></div>
</header>
<main class="flex">
<section class="container book-page">
<article class="markdown">
<h1>
<a href="/2015/12/18/flink-2015-a-year-in-review-and-a-lookout-to-2016/">Flink 2015: A year in review, and a lookout to 2016</a>
</h1>
December 18, 2015 -
<p><p>With 2015 ending, we thought that this would be good time to reflect
on the amazing work done by the Flink community over this past year,
and how much this community has grown.</p>
<p>Overall, we have seen Flink grow in terms of functionality from an
engine to one of the most complete open-source stream processing
frameworks available. The community grew from a relatively small and
geographically focused team, to a truly global, and one of the largest
big data communities in the the Apache Software Foundation.</p>
<p>We will also look at some interesting stats, including that the
busiest days for Flink are Mondays (who would have thought :-).</p>
<h1 id="community-growth">
Community growth
<a class="anchor" href="#community-growth">#</a>
</h1>
<p>Let us start with some simple statistics from <a href="https://github.com/apache/flink">Flink&rsquo;s
github repository</a>. During 2015, the
Flink community <strong>doubled</strong> in size, from about 75 contributors to
over 150. Forks of the repository more than <strong>tripled</strong> from 160 in
February 2015 to 544 in December 2015, and the number of stars of the
repository almost tripled from 289 to 813.</p>
<center>
<img src="/img/blog/community-growth.png" style="height:400px;margin:15px">
</center>
<p>Although Flink started out geographically in Berlin, Germany, the
community is by now spread all around the globe, with many
contributors from North America, Europe, and Asia. A simple search at
meetup.com for groups that mention Flink as a focus area reveals <a href="http://apache-flink.meetup.com/">16
meetups around the globe</a>:</p>
<center>
<img src="/img/blog/meetup-map.png" style="height:400px;margin:15px">
</center>
<h1 id="flink-forward-2015">
Flink Forward 2015
<a class="anchor" href="#flink-forward-2015">#</a>
</h1>
<p>One of the highlights of the year for Flink was undoubtedly the <a href="http://2015.flink-forward.org/">Flink
Forward</a> conference, the first conference
on Apache Flink that was held in October in Berlin. More than 250
participants (roughly half based outside Germany where the conference
was held) attended more than 33 technical talks from organizations
including Google, MongoDB, Bouygues Telecom, NFLabs, Euranova, RedHat,
IBM, Huawei, Intel, Ericsson, Capital One, Zalando, Amadeus, the Otto
Group, and ResearchGate. If you have not yet watched their talks,
check out the <a href="http://2015.flink-forward.org/?post_type=day">slides</a> and
<a href="https://www.youtube.com/playlist?list=PLDX4T_cnKjD31JeWR1aMOi9LXPRQ6nyHO">videos</a>
from Flink Forward.</p>
<center>
<img src="/img/blog/ff-speakers.png" style="height:400px;margin:15px">
</center>
<h1 id="media-coverage">
Media coverage
<a class="anchor" href="#media-coverage">#</a>
</h1>
<p>And of course, interest in Flink was picked up by the tech
media. During 2015, articles about Flink appeared in
<a href="http://www.infoq.com/Apache-Flink/news/">InfoQ</a>,
<a href="http://www.zdnet.com/article/five-open-source-big-data-projects-to-watch/">ZDNet</a>,
<a href="http://www.datanami.com/tag/apache-flink/">Datanami</a>,
<a href="http://www.infoworld.com/article/2919602/hadoop/flink-hadoops-new-contender-for-mapreduce-spark.html">Infoworld</a>
(including being one of the <a href="http://www.infoworld.com/article/2982429/open-source-tools/bossie-awards-2015-the-best-open-source-big-data-tools.html">best open source big data tools of
2015</a>),
the <a href="http://blogs.gartner.com/nick-heudecker/apache-flink-offers-a-challenge-to-spark/">Gartner
blog</a>,
<a href="http://dataconomy.com/tag/apache-flink/">Dataconomy</a>,
<a href="http://sdtimes.com/tag/apache-flink/">SDTimes</a>, the <a href="https://www.mapr.com/blog/apache-flink-new-way-handle-streaming-data">MapR
blog</a>,
<a href="http://www.kdnuggets.com/2015/08/apache-flink-stream-processing.html">KDnuggets</a>,
and
<a href="http://www.hadoopsphere.com/2015/02/distributed-data-processing-with-apache.html">HadoopSphere</a>.</p>
<center>
<img src="/img/blog/appeared-in.png" style="height:400px;margin:15px">
</center>
<p>It is interesting to see that Hadoop Summit EMEA 2016 had a whopping
number of 17 (!) talks submitted that are mentioning Flink in their
title and abstract:</p>
<center>
<img src="/img/blog/hadoop-summit.png" style="height:400px;margin:15px">
</center>
<h1 id="fun-with-stats-when-do-committers-commit">
Fun with stats: when do committers commit?
<a class="anchor" href="#fun-with-stats-when-do-committers-commit">#</a>
</h1>
<p>To get some deeper insight on what is happening in the Flink
community, let us do some analytics on the git log of the project :-)
The easiest thing we can do is count the number of commits at the
repository in 2015. Running</p>
<pre tabindex="0"><code>git log --pretty=oneline --after=1/1/2015 | wc -l
</code></pre><p>on the Flink repository yields a total of <strong>2203 commits</strong> in 2015.</p>
<p>To dig deeper, we will use an open source tool called gitstats that
will give us some interesting statistics on the committer
behavior. You can create these also yourself and see many more by
following four easy steps:</p>
<ol>
<li>Download gitstats from the <a href="http://gitstats.sourceforge.net/">project homepage</a>.. E.g., on OS X with homebrew, type</li>
</ol>
<pre tabindex="0"><code>brew install --HEAD homebrew/head-only/gitstats
</code></pre><ol start="2">
<li>Clone the Apache Flink git repository:</li>
</ol>
<pre tabindex="0"><code>git clone git@github.com:apache/flink.git
</code></pre><ol start="3">
<li>Generate the statistics</li>
</ol>
<pre tabindex="0"><code>gitstats flink/ flink-stats/
</code></pre><ol start="4">
<li>View all the statistics as an html page using your favorite browser (e.g., chrome):</li>
</ol>
<pre tabindex="0"><code>chrome flink-stats/index.html
</code></pre><p>First, we can see a steady growth of lines of code in Flink since the
initial Apache incubator project. During 2015, the codebase almost
<strong>doubled</strong> from 500,000 LOC to 900,000 LOC.</p>
<center>
<img src="/img/blog/code-growth.png" style="height:400px;margin:15px">
</center>
<p>It is interesting to see when committers commit. For Flink, Monday
afternoons are by far the most popular times to commit to the
repository:</p>
<center>
<img src="/img/blog/commit-stats.png" style="height:400px;margin:15px">
</center>
<h1 id="feature-timeline">
Feature timeline
<a class="anchor" href="#feature-timeline">#</a>
</h1>
<p>So, what were the major features added to Flink and the Flink
ecosystem during 2015? Here is a (non-exhaustive) chronological list:</p>
<center>
<img src="/img/blog/feature-timeline.png" style="height:400px;margin:15px">
</center>
<h1 id="roadmap-for-2016">
Roadmap for 2016
<a class="anchor" href="#roadmap-for-2016">#</a>
</h1>
<p>With 2015 coming to a close, the Flink community has already started
discussing Flink&rsquo;s roadmap for the future. Some highlights
are:</p>
<ul>
<li>
<p><strong>Runtime scaling of streaming jobs:</strong> streaming jobs are running
forever, and need to react to a changing environment. Runtime
scaling means dynamically increasing and decreasing the
parallelism of a job to sustain certain SLAs, or react to changing
input throughput.</p>
</li>
<li>
<p><strong>SQL queries for static data sets and streams:</strong> building on top of
Flink&rsquo;s Table API, users should be able to write SQL
queries for static data sets, as well as SQL queries on data
streams that continuously produce new results.</p>
</li>
<li>
<p><strong>Streaming operators backed by managed memory:</strong> currently,
streaming operators like user-defined state and windows are backed
by JVM heap objects. Moving those to Flink managed memory will add
the ability to spill to disk, GC efficiency, as well as better
control over memory utilization.</p>
</li>
<li>
<p><strong>Library for detecting temporal event patterns:</strong> a common use case
for stream processing is detecting patterns in an event stream
with timestamps. Flink makes this possible with its support for
event time, so many of these operators can be surfaced in the form
of a library.</p>
</li>
<li>
<p><strong>Support for Apache Mesos, and resource-dynamic YARN support:</strong>
support for both Mesos and YARN, including dynamic allocation and
release of resource for more resource elasticity (for both batch
and stream processing).</p>
</li>
<li>
<p><strong>Security:</strong> encrypt both the messages exchanged between
TaskManagers and JobManager, as well as the connections for data
exchange between workers.</p>
</li>
<li>
<p><strong>More streaming connectors, more runtime metrics, and continuous
DataStream API enhancements:</strong> add support for more sources and
sinks (e.g., Amazon Kinesis, Cassandra, Flume, etc), expose more
metrics to the user, and provide continuous improvements to the
DataStream API.</p>
</li>
</ul>
<p>If you are interested in these features, we highly encourage you to
take a look at the <a href="https://docs.google.com/document/d/1ExmtVpeVVT3TIhO1JoBpC5JKXm-778DAD7eqw5GANwE/edit">current
draft</a>,
and <a href="https://mail-archives.apache.org/mod_mbox/flink-dev/201512.mbox/browser">join the
discussion</a>
on the Flink mailing lists.</p>
</p>
</article>
<div class="edit-this-page">
<p>
<a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a>
</p>
<p>
<a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2015-12-18-a-year-in-review.md">
Edit This Page<i class="fa fa-edit fa-fw"></i>
</a>
</p>
</div>
</section>
<aside class="book-toc">
<nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3>
<ul>
<li><a href="#community-growth">Community growth</a></li>
<li><a href="#flink-forward-2015">Flink Forward 2015</a></li>
<li><a href="#media-coverage">Media coverage</a></li>
<li><a href="#fun-with-stats-when-do-committers-commit">Fun with stats: when do committers commit?</a></li>
<li><a href="#feature-timeline">Feature timeline</a></li>
<li><a href="#roadmap-for-2016">Roadmap for 2016</a></li>
</ul>
</nav>
</aside>
<aside class="expand-toc hidden">
<a class="toc" onclick="expandToc()" href="javascript:void(0)">
<i class="fa fa-bars" aria-hidden="true"></i>
</a>
</aside>
</main>
<footer>
<div class="separator"></div>
<div class="panels">
<div class="wrapper">
<div class="panel">
<ul>
<li>
<a href="https://flink-packages.org/">flink-packages.org</a>
</li>
<li>
<a href="https://www.apache.org/">Apache Software Foundation</a>
</li>
<li>
<a href="https://www.apache.org/licenses/">License</a>
</li>
<li>
<a href="/zh/">
<i class="fa fa-globe" aria-hidden="true"></i>&nbsp;中文版
</a>
</li>
</ul>
</div>
<div class="panel">
<ul>
<li>
<a href="/what-is-flink/security">Security</a-->
</li>
<li>
<a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
</li>
<li>
<a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
</li>
</ul>
</div>
<div class="panel icons">
<div>
<a href="/posts">
<div class="icon flink-blog-icon"></div>
<span>Flink blog</span>
</a>
</div>
<div>
<a href="https://github.com/apache/flink">
<div class="icon flink-github-icon"></div>
<span>Github</span>
</a>
</div>
<div>
<a href="https://twitter.com/apacheflink">
<div class="icon flink-twitter-icon"></div>
<span>Twitter</span>
</a>
</div>
</div>
</div>
</div>
<hr/>
<div class="container disclaimer">
<p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</footer>
</body>
</html>