| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> |
| <title>Apache Flink: The State of Flink on Docker</title> |
| <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> |
| <link rel="icon" href="/favicon.ico" type="image/x-icon"> |
| |
| <!-- Bootstrap --> |
| <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css"> |
| <link rel="stylesheet" href="/css/flink.css"> |
| <link rel="stylesheet" href="/css/syntax.css"> |
| |
| <!-- Blog RSS feed --> |
| <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> |
| |
| <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> |
| <!-- We need to load Jquery in the header for custom google analytics event tracking--> |
| <script src="/js/jquery.min.js"></script> |
| |
| <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> |
| <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> |
| <!--[if lt IE 9]> |
| <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script> |
| <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script> |
| <![endif]--> |
| </head> |
| <body> |
| |
| |
| <!-- Main content. --> |
| <div class="container"> |
| <div class="row"> |
| |
| |
| <div id="sidebar" class="col-sm-3"> |
| |
| |
| <!-- Top navbar. --> |
| <nav class="navbar navbar-default"> |
| <!-- The logo. --> |
| <div class="navbar-header"> |
| <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| </button> |
| <div class="navbar-logo"> |
| <a href="/"> |
| <img alt="Apache Flink" src="/img/flink-header-logo.svg" width="147px" height="73px"> |
| </a> |
| </div> |
| </div><!-- /.navbar-header --> |
| |
| <!-- The navigation links. --> |
| <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> |
| <ul class="nav navbar-nav navbar-main"> |
| |
| <!-- First menu section explains visitors what Flink is --> |
| |
| <!-- What is Stream Processing? --> |
| <!-- |
| <li><a href="/streamprocessing1.html">What is Stream Processing?</a></li> |
| --> |
| |
| <!-- What is Flink? --> |
| <li><a href="/flink-architecture.html">What is Apache Flink?</a></li> |
| |
| |
| <ul class="nav navbar-nav navbar-subnav"> |
| <li > |
| <a href="/flink-architecture.html">Architecture</a> |
| </li> |
| <li > |
| <a href="/flink-applications.html">Applications</a> |
| </li> |
| <li > |
| <a href="/flink-operations.html">Operations</a> |
| </li> |
| </ul> |
| |
| |
| <!-- What is Stateful Functions? --> |
| |
| <li><a href="/stateful-functions.html">What is Stateful Functions?</a></li> |
| |
| <!-- Use cases --> |
| <li><a href="/usecases.html">Use Cases</a></li> |
| |
| <!-- Powered by --> |
| <li><a href="/poweredby.html">Powered By</a></li> |
| |
| |
| |
| <!-- Second menu section aims to support Flink users --> |
| |
| <!-- Downloads --> |
| <li><a href="/downloads.html">Downloads</a></li> |
| |
| <!-- Getting Started --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Getting Started<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/getting-started/index.html" target="_blank">With Flink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1/getting-started/project-setup.html" target="_blank">With Flink Stateful Functions <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="/training.html">Training Course</a></li> |
| </ul> |
| </li> |
| |
| <!-- Documentation --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11" target="_blank">Flink 1.11 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-master" target="_blank">Flink Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.1" target="_blank">Flink Stateful Functions 2.1 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-master" target="_blank">Flink Stateful Functions Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| </ul> |
| </li> |
| |
| <!-- getting help --> |
| <li><a href="/gettinghelp.html">Getting Help</a></li> |
| |
| <!-- Blog --> |
| <li class="active"><a href="/blog/"><b>Flink Blog</b></a></li> |
| |
| |
| <!-- Flink-packages --> |
| <li> |
| <a href="https://flink-packages.org" target="_blank">flink-packages.org <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| <!-- Third menu section aim to support community and contributors --> |
| |
| <!-- Community --> |
| <li><a href="/community.html">Community & Project Info</a></li> |
| |
| <!-- Roadmap --> |
| <li><a href="/roadmap.html">Roadmap</a></li> |
| |
| <!-- Contribute --> |
| <li><a href="/contributing/how-to-contribute.html">How to Contribute</a></li> |
| |
| |
| <!-- GitHub --> |
| <li> |
| <a href="https://github.com/apache/flink" target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| |
| <!-- Language Switcher --> |
| <li> |
| |
| |
| <!-- link to the Chinese home page when current is blog page --> |
| <a href="/zh">中文版</a> |
| |
| |
| </li> |
| |
| </ul> |
| |
| <ul class="nav navbar-nav navbar-bottom"> |
| <hr /> |
| |
| <!-- Twitter --> |
| <li><a href="https://twitter.com/apacheflink" target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <!-- Visualizer --> |
| <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <hr /> |
| |
| <li><a href="https://apache.org" target="_blank">Apache Software Foundation <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <li> |
| <style> |
| .smalllinks:link { |
| display: inline-block !important; background: none; padding-top: 0px; padding-bottom: 0px; padding-right: 0px; min-width: 75px; |
| } |
| </style> |
| |
| <a class="smalllinks" href="https://www.apache.org/licenses/" target="_blank">License</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/security/" target="_blank">Security</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/sponsorship.html" target="_blank">Donate</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| </li> |
| |
| </ul> |
| </div><!-- /.navbar-collapse --> |
| </nav> |
| |
| </div> |
| <div class="col-sm-9"> |
| <div class="row-fluid"> |
| <div class="col-sm-12"> |
| <div class="row"> |
| <h1>The State of Flink on Docker</h1> |
| <p><i></i></p> |
| |
| <article> |
| <p>20 Aug 2020 Robert Metzger (<a href="https://twitter.com/rmetzger_">@rmetzger_</a>)</p> |
| |
| <p>With over 50 million downloads from Docker Hub, the Flink docker images are a very popular deployment option.</p> |
| |
| <p>The Flink community recently put some effort into improving the Docker experience for our users with the goal to reduce confusion and improve usability.</p> |
| |
| <p>Let’s quickly break down the recent improvements:</p> |
| |
| <ul> |
| <li> |
| <p>Reduce confusion: Flink used to have 2 Dockerfiles and a 3rd file maintained outside of the official repository — all with different features and varying stability. Now, we have one central place for all images: <a href="https://github.com/apache/flink-docker">apache/flink-docker</a>.</p> |
| |
| <p>Here, we keep all the Dockerfiles for the different releases. Check out the <a href="https://github.com/apache/flink-docker/blob/master/README.md">detailed readme</a> of that repository for further explanation on the different branches, as well as the <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-111%3A+Docker+image+unification">Flink Improvement Proposal (FLIP-111)</a> that contains the detailed planning.</p> |
| |
| <p>The <code>apache/flink-docker</code> repository also seeds the <a href="https://hub.docker.com/_/flink">official Flink image on Docker Hub</a>.</p> |
| </li> |
| <li> |
| <p>Improve Usability: The Dockerfiles are used for various purposes: <a href="https://ci.apache.org/projects/flink/flink-docs-master/ops/deployment/docker.html">Native Docker deployments</a>, <a href="https://ci.apache.org/projects/flink/flink-docs-master/ops/deployment/native_kubernetes.html">Flink on Kubernetes</a>, the (unofficial) <a href="https://github.com/docker-flink/examples">Flink helm example</a> and the project’s <a href="https://github.com/apache/flink/tree/master/flink-end-to-end-tests">internal end to end tests</a>. With one unified image, all these consumers of the images benefit from the same set of features, documentation and testing.</p> |
| |
| <p>The new images support <a href="https://ci.apache.org/projects/flink/flink-docs-master/ops/deployment/docker.html#configure-options">passing configuration variables</a> via a <code>FLINK_PROPERTIES</code> environment variable. Users can <a href="https://ci.apache.org/projects/flink/flink-docs-master/ops/deployment/docker.html#using-plugins">enable default plugins</a> with the <code>ENABLE_BUILT_IN_PLUGINS</code> environment variable. The images also allow loading custom jar paths and configuration files.</p> |
| </li> |
| </ul> |
| |
| <p>Looking into the future, there are already some interesting potential improvements lined up:</p> |
| |
| <ul> |
| <li><a href="https://issues.apache.org/jira/browse/FLINK-16260">Java 11 Docker images</a> (already completed)</li> |
| <li><a href="https://issues.apache.org/jira/browse/FLINK-15793">Use vanilla docker-entrypoint with flink-kubernetes</a> (in progress)</li> |
| <li><a href="https://issues.apache.org/jira/browse/FLINK-17167">History server support</a></li> |
| <li><a href="https://issues.apache.org/jira/browse/FLINK-15587">Support for OpenShift</a></li> |
| </ul> |
| |
| <h2 id="how-do-i-get-started">How do I get started?</h2> |
| |
| <p>This is a short tutorial on <a href="https://ci.apache.org/projects/flink/flink-docs-master/ops/deployment/docker.html#start-a-session-cluster">how to start a Flink Session Cluster</a> with Docker.</p> |
| |
| <p>A <em>Flink Session cluster</em> can be used to run multiple jobs. Each job needs to be submitted to the cluster after it has been deployed. To deploy a <em>Flink Session cluster</em> with Docker, you need to start a <em>JobManager</em> container. To enable communication between the containers, we first set a required Flink configuration property and create a network:</p> |
| |
| <div class="highlight"><pre><code>FLINK_PROPERTIES="jobmanager.rpc.address: jobmanager" |
| docker network create flink-network |
| </code></pre></div> |
| |
| <p>Then we launch the JobManager:</p> |
| |
| <div class="highlight"><pre><code>docker run \ |
| --rm \ |
| --name=jobmanager \ |
| --network flink-network \ |
| -p 8081:8081 \ |
| --env FLINK_PROPERTIES="${FLINK_PROPERTIES}" \ |
| flink:1.11.1 jobmanager |
| </code></pre></div> |
| <p>and one or more <em>TaskManager</em> containers:</p> |
| |
| <div class="highlight"><pre><code>docker run \ |
| --rm \ |
| --name=taskmanager \ |
| --network flink-network \ |
| --env FLINK_PROPERTIES="${FLINK_PROPERTIES}" \ |
| flink:1.11.1 taskmanager |
| </code></pre></div> |
| |
| <p>You now have a fully functional Flink cluster running! You can access the the web front end here: <a href="http://localhost:8081/">localhost:8081</a>.</p> |
| |
| <p>Let’s now submit one of Flink’s example jobs:</p> |
| |
| <div class="highlight"><pre><code class="language-bash"><span class="c"># 1: (optional) Download the Flink distribution, and unpack it</span> |
| wget https://archive.apache.org/dist/flink/flink-1.11.1/flink-1.11.1-bin-scala_2.12.tgz |
| tar xf flink-1.11.1-bin-scala_2.12.tgz |
| <span class="nb">cd </span>flink-1.11.1 |
| |
| <span class="c"># 2: Start the Flink job</span> |
| ./bin/flink run ./examples/streaming/TopSpeedWindowing.jar</code></pre></div> |
| |
| <p>The main steps of the tutorial are also recorded in this short screencast:</p> |
| |
| <center> |
| <img src="/img/blog/flink-docker/flink-docker.gif" width="882px" height="730px" alt="Demo video" /> |
| </center> |
| |
| <p><strong>Next steps</strong>: Now that you’ve successfully completed this tutorial, we recommend you checking out the full <a href="https://ci.apache.org/projects/flink/flink-docs-master/ops/deployment/docker.html">Flink on Docker documentation</a> for implementing more advanced deployment scenarios, such as Job Clusters, Docker Compose or our native Kubernetes integration.</p> |
| |
| <h2 id="conclusion">Conclusion</h2> |
| |
| <p>We encourage all readers to try out Flink on Docker to provide the community with feedback to further improve the experience. |
| Please refer to the user@flink.apache.org (<a href="https://flink.apache.org/community.html#how-to-subscribe-to-a-mailing-list">remember to subscribe first</a>) for general questions and our <a href="https://issues.apache.org/jira/issues/?jql=project+%3D+FLINK+AND+component+%3D+flink-docker">issue tracker</a> for specific bugs or improvements, or <a href="https://flink.apache.org/contributing/how-to-contribute.html">ideas for contributions</a>!</p> |
| |
| </article> |
| </div> |
| |
| <div class="row"> |
| <div id="disqus_thread"></div> |
| <script type="text/javascript"> |
| /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ |
| var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname |
| |
| /* * * DON'T EDIT BELOW THIS LINE * * */ |
| (function() { |
| var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; |
| dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; |
| (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); |
| })(); |
| </script> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| <hr /> |
| |
| <div class="row"> |
| <div class="footer text-center col-sm-12"> |
| <p>Copyright © 2014-2019 <a href="http://apache.org">The Apache Software Foundation</a>. All Rights Reserved.</p> |
| <p>Apache Flink, Flink®, Apache®, the squirrel logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> |
| <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> |
| </div> |
| </div> |
| </div><!-- /.container --> |
| |
| <!-- Include all compiled plugins (below), or include individual files as needed --> |
| <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.matchHeight/0.7.0/jquery.matchHeight-min.js"></script> |
| <script src="/js/codetabs.js"></script> |
| <script src="/js/stickysidebar.js"></script> |
| |
| <!-- Google Analytics --> |
| <script> |
| (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ |
| (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), |
| m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) |
| })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); |
| |
| ga('create', 'UA-52545728-1', 'auto'); |
| ga('send', 'pageview'); |
| </script> |
| </body> |
| </html> |
