blob: 9a8f77900e980e9d10de19c3572dc04077e3c196 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" dir=ZgotmplZ>
<head>
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
<script src="/bootstrap/js/bootstrap.bundle.min.js"></script>
<link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css">
<script src="/js/anchor.min.js"></script>
<script src="/js/flink.js"></script>
<link rel="canonical" href="https://flink.apache.org/2020/08/20/the-state-of-flink-on-docker/">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="With over 50 million downloads from Docker Hub, the Flink docker images are a very popular deployment option.
The Flink community recently put some effort into improving the Docker experience for our users with the goal to reduce confusion and improve usability.
Let&rsquo;s quickly break down the recent improvements:
Reduce confusion: Flink used to have 2 Dockerfiles and a 3rd file maintained outside of the official repository — all with different features and varying stability.">
<meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="The State of Flink on Docker" />
<meta property="og:description" content="With over 50 million downloads from Docker Hub, the Flink docker images are a very popular deployment option.
The Flink community recently put some effort into improving the Docker experience for our users with the goal to reduce confusion and improve usability.
Let&rsquo;s quickly break down the recent improvements:
Reduce confusion: Flink used to have 2 Dockerfiles and a 3rd file maintained outside of the official repository — all with different features and varying stability." />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://flink.apache.org/2020/08/20/the-state-of-flink-on-docker/" /><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2020-08-20T00:00:00+00:00" />
<meta property="article:modified_time" content="2020-08-20T00:00:00+00:00" />
<title>The State of Flink on Docker | Apache Flink</title>
<link rel="manifest" href="/manifest.json">
<link rel="icon" href="/favicon.png" type="image/x-icon">
<link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU=">
<script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script>
<!--
Made with Book Theme
https://github.com/alex-shpak/hugo-book
-->
<meta name="generator" content="Hugo 0.124.1">
<script>
var _paq = window._paq = window._paq || [];
_paq.push(['disableCookies']);
_paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="//analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '1']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
</head>
<body dir=ZgotmplZ>
<header>
<nav class="navbar navbar-expand-xl">
<div class="container-fluid">
<a class="navbar-brand" href="/">
<img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle">
<span>Apache Flink</span>
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<i class="fa fa-bars navbar-toggler-icon"></i>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/security/">Security</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a>
</li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link" href="/posts/">Flink Blog</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/downloads/">Downloads</a>
</li>
</ul>
<div class="book-search">
<div class="book-search-spinner hidden">
<i class="fa fa-refresh fa-spin"></i>
</div>
<form class="search-bar d-flex" onsubmit="return false;"su>
<input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/">
<i class="fa fa-search search"></i>
<i class="fa fa-circle-o-notch fa-spin spinner"></i>
</form>
<div class="book-search-spinner hidden"></div>
<ul id="book-search-results"></ul>
</div>
</div>
</div>
</nav>
<div class="navbar-clearfix"></div>
</header>
<main class="flex">
<section class="container book-page">
<article class="markdown">
<h1>
<a href="/2020/08/20/the-state-of-flink-on-docker/">The State of Flink on Docker</a>
</h1>
August 20, 2020 -
Robert Metzger
<a href="https://twitter.com/rmetzger_">(@rmetzger_)</a>
<p><p>With over 50 million downloads from Docker Hub, the Flink docker images are a very popular deployment option.</p>
<p>The Flink community recently put some effort into improving the Docker experience for our users with the goal to reduce confusion and improve usability.</p>
<p>Let&rsquo;s quickly break down the recent improvements:</p>
<ul>
<li>
<p>Reduce confusion: Flink used to have 2 Dockerfiles and a 3rd file maintained outside of the official repository — all with different features and varying stability. Now, we have one central place for all images: <a href="https://github.com/apache/flink-docker">apache/flink-docker</a>.</p>
<p>Here, we keep all the Dockerfiles for the different releases. Check out the <a href="https://github.com/apache/flink-docker/blob/master/README.md">detailed readme</a> of that repository for further explanation on the different branches, as well as the <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-111%3A&#43;Docker&#43;image&#43;unification">Flink Improvement Proposal (FLIP-111)</a> that contains the detailed planning.</p>
<p>The <code>apache/flink-docker</code> repository also seeds the <a href="https://hub.docker.com/_/flink">official Flink image on Docker Hub</a>.</p>
</li>
<li>
<p>Improve Usability: The Dockerfiles are used for various purposes: <a href="//nightlies.apache.org/flink/flink-docs-master/ops/deployment/docker.html">Native Docker deployments</a>, <a href="//nightlies.apache.org/flink/flink-docs-master/ops/deployment/native_kubernetes.html">Flink on Kubernetes</a>, the (unofficial) <a href="https://github.com/docker-flink/examples">Flink helm example</a> and the project&rsquo;s <a href="https://github.com/apache/flink/tree/master/flink-end-to-end-tests">internal end to end tests</a>. With one unified image, all these consumers of the images benefit from the same set of features, documentation and testing.</p>
<p>The new images support <a href="//nightlies.apache.org/flink/flink-docs-master/ops/deployment/docker.html#configure-options">passing configuration variables</a> via a <code>FLINK_PROPERTIES</code> environment variable. Users can <a href="//nightlies.apache.org/flink/flink-docs-master/ops/deployment/docker.html#using-plugins">enable default plugins</a> with the <code>ENABLE_BUILT_IN_PLUGINS</code> environment variable. The images also allow loading custom jar paths and configuration files.</p>
</li>
</ul>
<p>Looking into the future, there are already some interesting potential improvements lined up:</p>
<ul>
<li><a href="https://issues.apache.org/jira/browse/FLINK-16260">Java 11 Docker images</a> (already completed)</li>
<li><a href="https://issues.apache.org/jira/browse/FLINK-15793">Use vanilla docker-entrypoint with flink-kubernetes</a> (in progress)</li>
<li><a href="https://issues.apache.org/jira/browse/FLINK-17167">History server support</a></li>
<li><a href="https://issues.apache.org/jira/browse/FLINK-15587">Support for OpenShift</a></li>
</ul>
<h2 id="how-do-i-get-started">
How do I get started?
<a class="anchor" href="#how-do-i-get-started">#</a>
</h2>
<p>This is a short tutorial on <a href="//nightlies.apache.org/flink/flink-docs-master/ops/deployment/docker.html#start-a-session-cluster">how to start a Flink Session Cluster</a> with Docker.</p>
<p><em>Flink Session cluster</em> can be used to run multiple jobs. Each job needs to be submitted to the cluster after it has been deployed. To deploy a <em>Flink Session cluster</em> with Docker, you need to start a <em>JobManager</em> container. To enable communication between the containers, we first set a required Flink configuration property and create a network:</p>
<pre tabindex="0"><code>FLINK_PROPERTIES=&#34;jobmanager.rpc.address: jobmanager&#34;
docker network create flink-network
</code></pre><p>Then we launch the JobManager:</p>
<pre tabindex="0"><code>docker run \
--rm \
--name=jobmanager \
--network flink-network \
-p 8081:8081 \
--env FLINK_PROPERTIES=&#34;${FLINK_PROPERTIES}&#34; \
flink:1.11.1 jobmanager
</code></pre><p>and one or more <em>TaskManager</em> containers:</p>
<pre tabindex="0"><code>docker run \
--rm \
--name=taskmanager \
--network flink-network \
--env FLINK_PROPERTIES=&#34;${FLINK_PROPERTIES}&#34; \
flink:1.11.1 taskmanager
</code></pre><p>You now have a fully functional Flink cluster running! You can access the the web front end here: <a href="http://localhost:8081/">localhost:8081</a>.</p>
<p>Let&rsquo;s now submit one of Flink&rsquo;s example jobs:</p>
<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"><span class="line"><span class="cl"><span class="c1"># 1: (optional) Download the Flink distribution, and unpack it</span>
</span></span><span class="line"><span class="cl">wget https://archive.apache.org/dist/flink/flink-1.11.1/flink-1.11.1-bin-scala_2.12.tgz
</span></span><span class="line"><span class="cl">tar xf flink-1.11.1-bin-scala_2.12.tgz
</span></span><span class="line"><span class="cl"><span class="nb">cd</span> flink-1.11.1
</span></span><span class="line"><span class="cl">
</span></span><span class="line"><span class="cl"><span class="c1"># 2: Start the Flink job</span>
</span></span><span class="line"><span class="cl">./bin/flink run ./examples/streaming/TopSpeedWindowing.jar
</span></span></code></pre></div><p>The main steps of the tutorial are also recorded in this short screencast:</p>
<center>
<img src="/img/blog/flink-docker/flink-docker.gif" width="882px" height="730px" alt="Demo video"/>
</center>
<p><strong>Next steps</strong>: Now that you&rsquo;ve successfully completed this tutorial, we recommend you checking out the full <a href="//nightlies.apache.org/flink/flink-docs-master/ops/deployment/docker.html">Flink on Docker documentation</a> for implementing more advanced deployment scenarios, such as Job Clusters, Docker Compose or our native Kubernetes integration.</p>
<h2 id="conclusion">
Conclusion
<a class="anchor" href="#conclusion">#</a>
</h2>
<p>We encourage all readers to try out Flink on Docker to provide the community with feedback to further improve the experience.
Please refer to the <a href="mailto:user@flink.apache.org">user@flink.apache.org</a> (<a href="https://flink.apache.org/community.html#how-to-subscribe-to-a-mailing-list">remember to subscribe first</a>) for general questions and our <a href="https://issues.apache.org/jira/issues/?jql=project&#43;%3D&#43;FLINK&#43;AND&#43;component&#43;%3D&#43;flink-docker">issue tracker</a> for specific bugs or improvements, or <a href="https://flink.apache.org/contributing/how-to-contribute.html">ideas for contributions</a>!</p>
</p>
</article>
<div class="edit-this-page">
<p>
<a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a>
</p>
<p>
<a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2020-08-20-flink-docker.md">
Edit This Page<i class="fa fa-edit fa-fw"></i>
</a>
</p>
</div>
</section>
<aside class="book-toc">
<nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3>
<ul>
<li>
<ul>
<li><a href="#how-do-i-get-started">How do I get started?</a></li>
<li><a href="#conclusion">Conclusion</a></li>
</ul>
</li>
</ul>
</nav>
</aside>
<aside class="expand-toc hidden">
<a class="toc" onclick="expandToc()" href="javascript:void(0)">
<i class="fa fa-bars" aria-hidden="true"></i>
</a>
</aside>
</main>
<footer>
<div class="separator"></div>
<div class="panels">
<div class="wrapper">
<div class="panel">
<ul>
<li>
<a href="https://flink-packages.org/">flink-packages.org</a>
</li>
<li>
<a href="https://www.apache.org/">Apache Software Foundation</a>
</li>
<li>
<a href="https://www.apache.org/licenses/">License</a>
</li>
<li>
<a href="/zh/">
<i class="fa fa-globe" aria-hidden="true"></i>&nbsp;中文版
</a>
</li>
</ul>
</div>
<div class="panel">
<ul>
<li>
<a href="/what-is-flink/security">Security</a-->
</li>
<li>
<a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
</li>
<li>
<a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
</li>
</ul>
</div>
<div class="panel icons">
<div>
<a href="/posts">
<div class="icon flink-blog-icon"></div>
<span>Flink blog</span>
</a>
</div>
<div>
<a href="https://github.com/apache/flink">
<div class="icon flink-github-icon"></div>
<span>Github</span>
</a>
</div>
<div>
<a href="https://twitter.com/apacheflink">
<div class="icon flink-twitter-icon"></div>
<span>Twitter</span>
</a>
</div>
</div>
</div>
</div>
<hr/>
<div class="container disclaimer">
<p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</footer>
</body>
</html>