blob: 28a0787d223bde5b3d1c8b3bda7a39586cdefe7f [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" dir=ZgotmplZ>
<head>
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
<script src="/bootstrap/js/bootstrap.bundle.min.js"></script>
<link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css">
<script src="/js/anchor.min.js"></script>
<script src="/js/flink.js"></script>
<link rel="canonical" href="https://flink.apache.org/2015/01/06/december-2014-in-the-flink-community/">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="This is the first blog post of a “newsletter” like series where we give a summary of the monthly activity in the Flink community. As the Flink project grows, this can serve as a &ldquo;tl;dr&rdquo; for people that are not following the Flink dev and user mailing lists, or those that are simply overwhelmed by the traffic.
Flink graduation # The biggest news is that the Apache board approved Flink as a top-level Apache project!">
<meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="December 2014 in the Flink community" />
<meta property="og:description" content="This is the first blog post of a “newsletter” like series where we give a summary of the monthly activity in the Flink community. As the Flink project grows, this can serve as a &ldquo;tl;dr&rdquo; for people that are not following the Flink dev and user mailing lists, or those that are simply overwhelmed by the traffic.
Flink graduation # The biggest news is that the Apache board approved Flink as a top-level Apache project!" />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://flink.apache.org/2015/01/06/december-2014-in-the-flink-community/" /><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2015-01-06T10:00:00+00:00" />
<meta property="article:modified_time" content="2015-01-06T10:00:00+00:00" />
<title>December 2014 in the Flink community | Apache Flink</title>
<link rel="manifest" href="/manifest.json">
<link rel="icon" href="/favicon.png" type="image/x-icon">
<link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU=">
<script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script>
<!--
Made with Book Theme
https://github.com/alex-shpak/hugo-book
-->
<meta name="generator" content="Hugo 0.124.1">
<script>
var _paq = window._paq = window._paq || [];
_paq.push(['disableCookies']);
_paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="//analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '1']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
</head>
<body dir=ZgotmplZ>
<header>
<nav class="navbar navbar-expand-xl">
<div class="container-fluid">
<a class="navbar-brand" href="/">
<img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle">
<span>Apache Flink</span>
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<i class="fa fa-bars navbar-toggler-icon"></i>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/security/">Security</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a>
</li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link" href="/posts/">Flink Blog</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/downloads/">Downloads</a>
</li>
</ul>
<div class="book-search">
<div class="book-search-spinner hidden">
<i class="fa fa-refresh fa-spin"></i>
</div>
<form class="search-bar d-flex" onsubmit="return false;"su>
<input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/">
<i class="fa fa-search search"></i>
<i class="fa fa-circle-o-notch fa-spin spinner"></i>
</form>
<div class="book-search-spinner hidden"></div>
<ul id="book-search-results"></ul>
</div>
</div>
</div>
</nav>
<div class="navbar-clearfix"></div>
</header>
<main class="flex">
<section class="container book-page">
<article class="markdown">
<h1>
<a href="/2015/01/06/december-2014-in-the-flink-community/">December 2014 in the Flink community</a>
</h1>
January 6, 2015 -
<p><p>This is the first blog post of a “newsletter” like series where we give a summary of the monthly activity in the Flink community. As the Flink project grows, this can serve as a &ldquo;tl;dr&rdquo; for people that are not following the Flink dev and user mailing lists, or those that are simply overwhelmed by the traffic.</p>
<h3 id="flink-graduation">
Flink graduation
<a class="anchor" href="#flink-graduation">#</a>
</h3>
<p>The biggest news is that the Apache board approved Flink as a top-level Apache project! The Flink team is working closely with the Apache press team for an official announcement, so stay tuned for details!</p>
<h3 id="new-flink-website">
New Flink website
<a class="anchor" href="#new-flink-website">#</a>
</h3>
<p>The <a href="http://flink.apache.org">Flink website</a> got a total make-over, both in terms of appearance and content.</p>
<h3 id="flink-irc-channel">
Flink IRC channel
<a class="anchor" href="#flink-irc-channel">#</a>
</h3>
<p>A new IRC channel called #flink was created at irc.freenode.org. An easy way to access the IRC channel is through the <a href="http://webchat.freenode.net/">web client</a>. Feel free to stop by to ask anything or share your ideas about Apache Flink!</p>
<h3 id="meetups-and-talks">
Meetups and Talks
<a class="anchor" href="#meetups-and-talks">#</a>
</h3>
<p>Apache Flink was presented in the <a href="http://www.meetup.com/Netherlands-Hadoop-User-Group/events/218635152">Amsterdam Hadoop User Group</a></p>
<h2 id="notable-code-contributions">
Notable code contributions
<a class="anchor" href="#notable-code-contributions">#</a>
</h2>
<p><strong>Note:</strong> Code contributions listed here may not be part of a release or even the current snapshot yet.</p>
<h3 id="streaming-scala-apihttpsgithubcomapacheincubator-flinkpull275">
<a href="https://github.com/apache/incubator-flink/pull/275">Streaming Scala API</a>
<a class="anchor" href="#streaming-scala-apihttpsgithubcomapacheincubator-flinkpull275">#</a>
</h3>
<p>The Flink Streaming Java API recently got its Scala counterpart. Once merged, Flink Streaming users can use both Scala and Java for their development. The Flink Streaming Scala API is built as a thin layer on top of the Java API, making sure that the APIs are kept easily in sync.</p>
<h3 id="intermediate-datasetshttpsgithubcomapacheincubator-flinkpull254">
<a href="https://github.com/apache/incubator-flink/pull/254">Intermediate datasets</a>
<a class="anchor" href="#intermediate-datasetshttpsgithubcomapacheincubator-flinkpull254">#</a>
</h3>
<p>This pull request introduces a major change in the Flink runtime. Currently, the Flink runtime is based on the notion of operators that exchange data through channels. With the PR, intermediate data sets that are produced by operators become first-class citizens in the runtime. While this does not have any user-facing impact yet, it lays the groundwork for a slew of future features such as blocking execution, fine-grained fault-tolerance, and more efficient data sharing between cluster and client.</p>
<h3 id="configurable-execution-modehttpsgithubcomapacheincubator-flinkpull259">
<a href="https://github.com/apache/incubator-flink/pull/259">Configurable execution mode</a>
<a class="anchor" href="#configurable-execution-modehttpsgithubcomapacheincubator-flinkpull259">#</a>
</h3>
<p>This pull request allows the user to change the object-reuse behaviour. Before this pull request, some operations would reuse objects passed to the user function while others would always create new objects. This introduces a system wide switch and changes all operators to either reuse objects or don’t reuse objects.</p>
<h3 id="distributed-coordination-via-akkahttpsgithubcomapacheincubator-flinkpull149">
<a href="https://github.com/apache/incubator-flink/pull/149">Distributed Coordination via Akka</a>
<a class="anchor" href="#distributed-coordination-via-akkahttpsgithubcomapacheincubator-flinkpull149">#</a>
</h3>
<p>Another major change is a complete rewrite of the JobManager / TaskManager components in Scala. In addition to that, the old RPC service was replaced by Actors, using the Akka framework.</p>
<h3 id="sorting-of-very-large-recordshttpsgithubcomapacheincubator-flinkpull249-">
<a href="https://github.com/apache/incubator-flink/pull/249">Sorting of very large records</a>
<a class="anchor" href="#sorting-of-very-large-recordshttpsgithubcomapacheincubator-flinkpull249-">#</a>
</h3>
<p>Flink&rsquo;s internal sort-algorithms were improved to better handle large records (multiple 100s of megabytes or larger). Previously, the system did in some cases hold instances of multiple large records, resulting in high memory consumption and JVM heap thrashing. Through this fix, large records are streamed through the operators, reducing the memory consumption and GC pressure. The system now requires much less memory to support algorithms that work on such large records.</p>
<h3 id="kryo-serialization-as-the-new-default-fallbackhttpsgithubcomapacheincubator-flinkpull271">
<a href="https://github.com/apache/incubator-flink/pull/271">Kryo Serialization as the new default fallback</a>
<a class="anchor" href="#kryo-serialization-as-the-new-default-fallbackhttpsgithubcomapacheincubator-flinkpull271">#</a>
</h3>
<p>Flink’s build-in type serialization framework is handles all common types very efficiently. Prior versions uses Avro to serialize types that the built-in framework could not handle.
Flink serialization system improved a lot over time and by now surpasses the capabilities of Avro in many cases. Kryo now serves as the default fallback serialization framework, supporting a much broader range of types.</p>
<h3 id="hadoop-filesystem-supporthttpsgithubcomapacheincubator-flinkpull268">
<a href="https://github.com/apache/incubator-flink/pull/268">Hadoop FileSystem support</a>
<a class="anchor" href="#hadoop-filesystem-supporthttpsgithubcomapacheincubator-flinkpull268">#</a>
</h3>
<p>This change permits users to use all file systems supported by Hadoop with Flink. In practice this means that users can use Flink with Tachyon, Google Cloud Storage (also out of the box Flink YARN support on Google Compute Cloud), FTP and all the other file system implementations for Hadoop.</p>
<h2 id="heading-to-the-080-release">
Heading to the 0.8.0 release
<a class="anchor" href="#heading-to-the-080-release">#</a>
</h2>
<p>The community is working hard together with the Apache infra team to migrate the Flink infrastructure to a top-level project. At the same time, the Flink community is working on the Flink 0.8.0 release which should be out very soon.</p>
</p>
</article>
<div class="edit-this-page">
<p>
<a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a>
</p>
<p>
<a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2015-01-06-december-in-flink.md">
Edit This Page<i class="fa fa-edit fa-fw"></i>
</a>
</p>
</div>
</section>
<aside class="book-toc">
<nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3>
<ul>
<li>
<ul>
<li>
<ul>
<li><a href="#flink-graduation">Flink graduation</a></li>
<li><a href="#new-flink-website">New Flink website</a></li>
<li><a href="#flink-irc-channel">Flink IRC channel</a></li>
<li><a href="#meetups-and-talks">Meetups and Talks</a></li>
</ul>
</li>
<li><a href="#notable-code-contributions">Notable code contributions</a>
<ul>
<li><a href="#streaming-scala-apihttpsgithubcomapacheincubator-flinkpull275"><a href="https://github.com/apache/incubator-flink/pull/275">Streaming Scala API</a></a></li>
<li><a href="#intermediate-datasetshttpsgithubcomapacheincubator-flinkpull254"><a href="https://github.com/apache/incubator-flink/pull/254">Intermediate datasets</a></a></li>
<li><a href="#configurable-execution-modehttpsgithubcomapacheincubator-flinkpull259"><a href="https://github.com/apache/incubator-flink/pull/259">Configurable execution mode</a></a></li>
<li><a href="#distributed-coordination-via-akkahttpsgithubcomapacheincubator-flinkpull149"><a href="https://github.com/apache/incubator-flink/pull/149">Distributed Coordination via Akka</a></a></li>
<li><a href="#sorting-of-very-large-recordshttpsgithubcomapacheincubator-flinkpull249-"><a href="https://github.com/apache/incubator-flink/pull/249">Sorting of very large records</a></a></li>
<li><a href="#kryo-serialization-as-the-new-default-fallbackhttpsgithubcomapacheincubator-flinkpull271"><a href="https://github.com/apache/incubator-flink/pull/271">Kryo Serialization as the new default fallback</a></a></li>
<li><a href="#hadoop-filesystem-supporthttpsgithubcomapacheincubator-flinkpull268"><a href="https://github.com/apache/incubator-flink/pull/268">Hadoop FileSystem support</a></a></li>
</ul>
</li>
<li><a href="#heading-to-the-080-release">Heading to the 0.8.0 release</a></li>
</ul>
</li>
</ul>
</nav>
</aside>
<aside class="expand-toc hidden">
<a class="toc" onclick="expandToc()" href="javascript:void(0)">
<i class="fa fa-bars" aria-hidden="true"></i>
</a>
</aside>
</main>
<footer>
<div class="separator"></div>
<div class="panels">
<div class="wrapper">
<div class="panel">
<ul>
<li>
<a href="https://flink-packages.org/">flink-packages.org</a>
</li>
<li>
<a href="https://www.apache.org/">Apache Software Foundation</a>
</li>
<li>
<a href="https://www.apache.org/licenses/">License</a>
</li>
<li>
<a href="/zh/">
<i class="fa fa-globe" aria-hidden="true"></i>&nbsp;中文版
</a>
</li>
</ul>
</div>
<div class="panel">
<ul>
<li>
<a href="/what-is-flink/security">Security</a-->
</li>
<li>
<a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
</li>
<li>
<a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
</li>
</ul>
</div>
<div class="panel icons">
<div>
<a href="/posts">
<div class="icon flink-blog-icon"></div>
<span>Flink blog</span>
</a>
</div>
<div>
<a href="https://github.com/apache/flink">
<div class="icon flink-github-icon"></div>
<span>Github</span>
</a>
</div>
<div>
<a href="https://twitter.com/apacheflink">
<div class="icon flink-twitter-icon"></div>
<span>Twitter</span>
</a>
</div>
</div>
</div>
</div>
<hr/>
<div class="container disclaimer">
<p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</footer>
</body>
</html>