| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> |
| <title>Apache Flink: Apache Beam: How Beam Runs on Top of Flink</title> |
| <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> |
| <link rel="icon" href="/favicon.ico" type="image/x-icon"> |
| |
| <!-- Bootstrap --> |
| <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css"> |
| <link rel="stylesheet" href="/css/flink.css"> |
| <link rel="stylesheet" href="/css/syntax.css"> |
| |
| <!-- Blog RSS feed --> |
| <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> |
| |
| <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> |
| <!-- We need to load Jquery in the header for custom google analytics event tracking--> |
| <script src="/js/jquery.min.js"></script> |
| |
| <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> |
| <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> |
| <!--[if lt IE 9]> |
| <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script> |
| <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script> |
| <![endif]--> |
| </head> |
| <body> |
| |
| |
| <!-- Main content. --> |
| <div class="container"> |
| <div class="row"> |
| |
| |
| <div id="sidebar" class="col-sm-3"> |
| |
| |
| <!-- Top navbar. --> |
| <nav class="navbar navbar-default"> |
| <!-- The logo. --> |
| <div class="navbar-header"> |
| <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| </button> |
| <div class="navbar-logo"> |
| <a href="/"> |
| <img alt="Apache Flink" src="/img/flink-header-logo.svg" width="147px" height="73px"> |
| </a> |
| </div> |
| </div><!-- /.navbar-header --> |
| |
| <!-- The navigation links. --> |
| <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> |
| <ul class="nav navbar-nav navbar-main"> |
| |
| <!-- First menu section explains visitors what Flink is --> |
| |
| <!-- What is Stream Processing? --> |
| <!-- |
| <li><a href="/streamprocessing1.html">What is Stream Processing?</a></li> |
| --> |
| |
| <!-- What is Flink? --> |
| <li><a href="/flink-architecture.html">What is Apache Flink?</a></li> |
| |
| |
| |
| <!-- What is Stateful Functions? --> |
| |
| <li><a href="/stateful-functions.html">What is Stateful Functions?</a></li> |
| |
| <!-- Use cases --> |
| <li><a href="/usecases.html">Use Cases</a></li> |
| |
| <!-- Powered by --> |
| <li><a href="/poweredby.html">Powered By</a></li> |
| |
| |
| |
| <!-- Second menu section aims to support Flink users --> |
| |
| <!-- Downloads --> |
| <li><a href="/downloads.html">Downloads</a></li> |
| |
| <!-- Getting Started --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Getting Started<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.10/getting-started/index.html" target="_blank">With Flink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.0/getting-started/project-setup.html" target="_blank">With Flink Stateful Functions <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="/training.html">Training Course</a></li> |
| </ul> |
| </li> |
| |
| <!-- Documentation --> |
| <li class="dropdown"> |
| <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a> |
| <ul class="dropdown-menu"> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.10" target="_blank">Flink 1.10 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-docs-master" target="_blank">Flink Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-release-2.0" target="_blank">Flink Stateful Functions 2.0 (Latest stable release) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| <li><a href="https://ci.apache.org/projects/flink/flink-statefun-docs-master" target="_blank">Flink Stateful Functions Master (Latest Snapshot) <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| </ul> |
| </li> |
| |
| <!-- getting help --> |
| <li><a href="/gettinghelp.html">Getting Help</a></li> |
| |
| <!-- Blog --> |
| <li><a href="/blog/"><b>Flink Blog</b></a></li> |
| |
| |
| <!-- Flink-packages --> |
| <li> |
| <a href="https://flink-packages.org" target="_blank">flink-packages.org <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| <!-- Third menu section aim to support community and contributors --> |
| |
| <!-- Community --> |
| <li><a href="/community.html">Community & Project Info</a></li> |
| |
| <!-- Roadmap --> |
| <li><a href="/roadmap.html">Roadmap</a></li> |
| |
| <!-- Contribute --> |
| <li><a href="/contributing/how-to-contribute.html">How to Contribute</a></li> |
| |
| |
| <!-- GitHub --> |
| <li> |
| <a href="https://github.com/apache/flink" target="_blank">Flink on GitHub <small><span class="glyphicon glyphicon-new-window"></span></small></a> |
| </li> |
| |
| |
| |
| <!-- Language Switcher --> |
| <li> |
| |
| |
| <a href="/zh/ecosystem/2020/02/22/apache-beam-how-beam-runs-on-top-of-flink.html">中文版</a> |
| |
| |
| </li> |
| |
| </ul> |
| |
| <ul class="nav navbar-nav navbar-bottom"> |
| <hr /> |
| |
| <!-- Twitter --> |
| <li><a href="https://twitter.com/apacheflink" target="_blank">@ApacheFlink <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <!-- Visualizer --> |
| <li class=" hidden-md hidden-sm"><a href="/visualizer/" target="_blank">Plan Visualizer <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <hr /> |
| |
| <li><a href="https://apache.org" target="_blank">Apache Software Foundation <small><span class="glyphicon glyphicon-new-window"></span></small></a></li> |
| |
| <li> |
| <style> |
| .smalllinks:link { |
| display: inline-block !important; background: none; padding-top: 0px; padding-bottom: 0px; padding-right: 0px; min-width: 75px; |
| } |
| </style> |
| |
| <a class="smalllinks" href="https://www.apache.org/licenses/" target="_blank">License</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/security/" target="_blank">Security</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/sponsorship.html" target="_blank">Donate</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| |
| <a class="smalllinks" href="https://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a> <small><span class="glyphicon glyphicon-new-window"></span></small> |
| </li> |
| |
| </ul> |
| </div><!-- /.navbar-collapse --> |
| </nav> |
| |
| </div> |
| <div class="col-sm-9"> |
| <div class="row-fluid"> |
| <div class="col-sm-12"> |
| <div class="row"> |
| <h1>Apache Beam: How Beam Runs on Top of Flink</h1> |
| <p><i></i></p> |
| |
| <article> |
| <p>22 Feb 2020 Maximilian Michels (<a href="https://twitter.com/stadtlegende">@stadtlegende</a>) & Markos Sfikas (<a href="https://twitter.com/MarkSfik">@MarkSfik</a>)</p> |
| |
| <p>Note: This blog post is based on the talk <a href="https://www.youtube.com/watch?v=hxHGLrshnCY">“Beam on Flink: How Does It Actually Work?”</a>.</p> |
| |
| <p><a href="https://flink.apache.org/">Apache Flink</a> and <a href="https://beam.apache.org/">Apache Beam</a> are open-source frameworks for parallel, distributed data processing at scale. Unlike Flink, Beam does not come with a full-blown execution engine of its own but plugs into other execution engines, such as Apache Flink, Apache Spark, or Google Cloud Dataflow. In this blog post we discuss the reasons to use Flink together with Beam for your batch and stream processing needs. We also take a closer look at how Beam works with Flink to provide an idea of the technical aspects of running Beam pipelines with Flink. We hope you find some useful information on how and why the two frameworks can be utilized in combination. For more information, you can refer to the corresponding <a href="https://beam.apache.org/documentation/runners/flink/">documentation</a> on the Beam website or contact the community through the <a href="https://beam.apache.org/community/contact-us/">Beam mailing list</a>.</p> |
| |
| <h1 id="what-is-apache-beam">What is Apache Beam</h1> |
| |
| <p><a href="https://beam.apache.org/">Apache Beam</a> is an open-source, unified model for defining batch and streaming data-parallel processing pipelines. It is unified in the sense that you use a single API, in contrast to using a separate API for batch and streaming like it is the case in Flink. Beam was originally developed by Google which released it in 2014 as the Cloud Dataflow SDK. In 2016, it was donated to <a href="https://www.apache.org/">the Apache Software Foundation</a> with the name of Beam. It has been developed by the open-source community ever since. With Apache Beam, developers can write data processing jobs, also known as pipelines, in multiple languages, e.g. Java, Python, Go, SQL. A pipeline is then executed by one of Beam’s Runners. A Runner is responsible for translating Beam pipelines such that they can run on an execution engine. Every supported execution engine has a Runner. The following Runners are available: Apache Flink, Apache Spark, Apache Samza, Hazelcast Jet, Google Cloud Dataflow, and others.</p> |
| |
| <p>The execution model, as well as the API of Apache Beam, are similar to Flink’s. Both frameworks are inspired by the <a href="https://static.googleusercontent.com/media/research.google.com/en//archive/mapreduce-osdi04.pdf">MapReduce</a>, <a href="https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/41378.pdf">MillWheel</a>, and <a href="https://research.google/pubs/pub43864/">Dataflow</a> papers. Like Flink, Beam is designed for parallel, distributed data processing. Both have similar transformations, support for windowing, event/processing time, watermarks, timers, triggers, and much more. However, Beam not being a full runtime focuses on providing the framework for building portable, multi-language batch and stream processing pipelines such that they can be run across several execution engines. The idea is that you write your pipeline once and feed it with either batch or streaming data. When you run it, you just pick one of the supported backends to execute. A large integration test suite in Beam called “ValidatesRunner” ensures that the results will be the same, regardless of which backend you choose for the execution.</p> |
| |
| <p>One of the most exciting developments in the Beam technology is the framework’s support for multiple programming languages including Java, Python, Go, Scala and SQL. Essentially, developers can write their applications in a programming language of their choice. Beam, with the help of the Runners, translates the program to one of the execution engines, as shown in the diagram below.</p> |
| |
| <center> |
| <img src="/img/blog/2020-02-22-beam-on-flink/flink-runner-beam-beam-vision.png" width="600px" alt="The vision of Apache Beam" /> |
| </center> |
| |
| <h1 id="reasons-to-use-beam-with-flink">Reasons to use Beam with Flink</h1> |
| |
| <p>Why would you want to use Beam with Flink instead of directly using Flink? Ultimately, Beam and Flink complement each other and provide additional value to the user. The main reasons for using Beam with Flink are the following:</p> |
| |
| <ul> |
| <li>Beam provides a unified API for both batch and streaming scenarios.</li> |
| <li>Beam comes with native support for different programming languages, like Python or Go with all their libraries like Numpy, Pandas, Tensorflow, or TFX.</li> |
| <li>You get the power of Apache Flink like its exactly-once semantics, strong memory management and robustness.</li> |
| <li>Beam programs run on your existing Flink infrastructure or infrastructure for other supported Runners, like Spark or Google Cloud Dataflow.</li> |
| <li>You get additional features like side inputs and cross-language pipelines that are not supported natively in Flink but only supported when using Beam with Flink.</li> |
| </ul> |
| |
| <h1 id="the-flink-runner-in-beam">The Flink Runner in Beam</h1> |
| |
| <p>The Flink Runner in Beam translates Beam pipelines into Flink jobs. The translation can be parameterized using Beam’s pipeline options which are parameters for settings like configuring the job name, parallelism, checkpointing, or metrics reporting.</p> |
| |
| <p>If you are familiar with a DataSet or a DataStream, you will have no problems understanding what a PCollection is. PCollection stands for parallel collection in Beam and is exactly what DataSet/DataStream would be in Flink. Due to Beam’s unified API we only have one type of results of transformation: PCollection.</p> |
| |
| <p>Beam pipelines are composed of transforms. Transforms are like operators in Flink and come in two flavors: primitive and composite transforms. The beauty of all this is that Beam only comes with a small set of primitive transforms which are:</p> |
| |
| <ul> |
| <li><code>Source</code> (for loading data)</li> |
| <li><code>ParDo</code> (think of a flat map operator on steroids)</li> |
| <li><code>GroupByKey</code> (think of keyBy() in Flink)</li> |
| <li><code>AssignWindows</code> (windows can be assigned at any point in time in Beam)</li> |
| <li><code>Flatten</code> (like a union() operation in Flink)</li> |
| </ul> |
| |
| <p>Composite transforms are built by combining the above primitive transforms. For example, <code>Combine = GroupByKey + ParDo</code>.</p> |
| |
| <h1 id="flink-runner-internals">Flink Runner Internals</h1> |
| |
| <p>Although using the Flink Runner in Beam has no prerequisite to understanding its internals, we provide more details of how the Flink runner works in Beam to share knowledge of how the two frameworks can integrate and work together to provide state-of-the-art streaming data pipelines.</p> |
| |
| <p>The Flink Runner has two translation paths. Depending on whether we execute in batch or streaming mode, the Runner either translates into Flink’s DataSet or into Flink’s DataStream API. Since multi-language support has been added to Beam, another two translation paths have been added. To summarize the four modes:</p> |
| |
| <ol> |
| <li><strong>The Classic Flink Runner for batch jobs:</strong> Executes batch Java pipelines</li> |
| <li><strong>The Classic Flink Runner for streaming jobs:</strong> Executes streaming Java pipelines</li> |
| <li><strong>The Portable Flink Runner for batch jobs:</strong> Executes Java as well as Python, Go and other supported SDK pipelines for batch scenarios</li> |
| <li><strong>The Portable Flink Runner for streaming jobs:</strong> Executes Java as well as Python, Go and other supported SDK pipelines for streaming scenarios</li> |
| </ol> |
| |
| <center> |
| <img src="/img/blog/2020-02-22-beam-on-flink/flink-runner-beam-runner-translation-paths.png" width="300px" alt="The 4 translation paths in the Beam's Flink Runner" /> |
| </center> |
| |
| <h2 id="the-classic-flink-runner-in-beam">The “Classic” Flink Runner in Beam</h2> |
| |
| <p>The classic Flink Runner was the initial version of the Runner, hence the “classic” name. Beam pipelines are represented as a graph in Java which is composed of the aforementioned composite and primitive transforms. Beam provides translators which traverse the graph in topological order. Topological order means that we start from all the sources first as we iterate through the graph. Presented with a transform from the graph, the Flink Runner generates the API calls as you would normally when writing a Flink job.</p> |
| |
| <center> |
| <img src="/img/blog/2020-02-22-beam-on-flink/classic-flink-runner-beam.png" width="600px" alt="The Classic Flink Runner in Beam" /> |
| </center> |
| |
| <p>While Beam and Flink share very similar concepts, there are enough differences between the two frameworks that make Beam pipelines impossible to be translated 1:1 into a Flink program. In the following sections, we will present the key differences:</p> |
| |
| <h3 id="serializers-vs-coders">Serializers vs Coders</h3> |
| |
| <p>When data is transferred over the wire in Flink, it has to be turned into bytes. This is done with the help of serializers. Flink has a type system to instantiate the correct coder for a given type, e.g. <code>StringTypeSerializer</code> for a String. Apache Beam also has its own type system which is similar to Flink’s but uses slightly different interfaces. Serializers are called Coders in Beam. In order to make a Beam Coder run in Flink, we have to make the two serializer types compatible. This is done by creating a special Flink type information that looks like the one in Flink but calls the appropriate Beam coder. That way, we can use Beam’s coders although we are executing the Beam job with Flink. Flink operators expect a TypeInformation, e.g. <code>StringTypeInformation</code>, for which we use a <code>CoderTypeInformation</code> in Beam. The type information returns the serializer for which we return a <code>CoderTypeSerializer</code>, which calls the underlying Beam Coder.</p> |
| |
| <center> |
| <img src="/img/blog/2020-02-22-beam-on-flink/flink-runner-beam-serializers-coders.png" width="300px" alt="Serializers vs Coders" /> |
| </center> |
| |
| <h3 id="read">Read</h3> |
| |
| <p>The <code>Read</code> transform provides a way to read data into your pipeline in Beam. The Read transform is supported by two wrappers in Beam, the <code>SourceInputFormat</code> for batch processing and the <code>UnboundedSourceWrapper</code> for stream processing.</p> |
| |
| <h3 id="pardo">ParDo</h3> |
| |
| <p><code>ParDo</code> is the swiss army knife of Beam and can be compared to a <code>RichFlatMapFunction</code> in Flink with additional features such as <code>SideInputs</code>, <code>SideOutputs</code>, State and Timers. <code>ParDo</code> is essentially translated by the Flink runner using the <code>FlinkDoFnFunction</code> for batch processing or the <code>FlinkStatefulDoFnFunction</code>, while for streaming scenarios the translation is executed with the <code>DoFnOperator</code> that takes care of checkpointing and buffering of data during checkpoints, watermark emissions and maintenance of state and timers. This is all executed by Beam’s interface, called the <code>DoFnRunner</code>, that encapsulates Beam-specific execution logic, like retrieving state, executing state and timers, or reporting metrics.</p> |
| |
| <h3 id="side-inputs">Side Inputs</h3> |
| |
| <p>In addition to the main input, ParDo transforms can have a number of side inputs. A side input can be a static set of data that you want to have available at all parallel instances. However, it is more flexible than that. You can have keyed and even windowed side input which updates based on the window size. This is a very powerful concept which does not exist in Flink but is added on top of Flink using Beam.</p> |
| |
| <h3 id="assignwindows">AssignWindows</h3> |
| |
| <p>In Flink, windows are assigned by the <code>WindowOperator</code> when you use the <code>window()</code> in the API. In Beam, windows can be assigned at any point in time. Any element is implicitly part of a window. If no window is assigned explicitly, the element is part of the <code>GlobalWindow</code>. Window information is stored for each element in a wrapper called <code>WindowedValue</code>. The window information is only used once we issue a <code>GroupByKey</code>.</p> |
| |
| <h3 id="groupbykey">GroupByKey</h3> |
| |
| <p>Most of the time it is useful to partition the data by a key. In Flink, this is done via the <code>keyBy()</code> API call. In Beam the <code>GroupByKey</code> transform can only be applied if the input is of the form <code>KV<Key, Value></code>. Unlike Flink where the key can even be nested inside the data, Beam enforces the key to always be explicit. The <code>GroupByKey</code> transform then groups the data by key and by window which is similar to what <code>keyBy(..).window(..)</code> would give us in Flink. Beam has its own set of libraries to do that because Beam has its own set of window functions and triggers. Essentially, GroupByKey is very similar to what the WindowOperator does in Flink.</p> |
| |
| <h3 id="flatten">Flatten</h3> |
| |
| <p>The Flatten operator takes multiple DataSet/DataStreams, called P[arallel]Collections in Beam, and combines them into one collection. This is equivalent to Flink’s <code>union()</code> operation.</p> |
| |
| <h2 id="the-portable-flink-runner-in-beam">The “Portable” Flink Runner in Beam</h2> |
| |
| <p>The portable Flink Runner in Beam is the evolution of the classic Runner. Classic Runners are tied to the JVM ecosystem, but the Beam community wanted to move past this and also execute Python, Go and other languages. This adds another dimension to Beam in terms of portability because, like previously mentioned, Beam already had portability across execution engines. It was necessary to change the translation logic of the Runner to be able to support language portability.</p> |
| |
| <p>There are two important building blocks for portable Runners:</p> |
| |
| <ol> |
| <li>A common pipeline format across all the languages: The Runner API</li> |
| <li>A common interface during execution for the communication between the Runner and the code written in any language: The Fn API</li> |
| </ol> |
| |
| <p>The Runner API provides a universal representation of the pipeline as Protobuf which contains the transforms, types, and user code. Protobuf was chosen as the format because every language has libraries available for it. Similarly, for the execution part, Beam introduced the Fn API interface to handle the communication between the Runner/execution engine and the user code that may be written in a different language and executes in a different process. Fn API is pronounced “fun API”, you may guess why.</p> |
| |
| <center> |
| <img src="/img/blog/2020-02-22-beam-on-flink/flink-runner-beam-language-portability.png" width="600px" alt="Language Portability in Apache Beam" /> |
| </center> |
| |
| <h2 id="how-are-beam-programs-translated-in-language-portability">How Are Beam Programs Translated In Language Portability?</h2> |
| |
| <p>Users write their Beam pipelines in one language, but they may get executed in an environment based on a completely different language. How does that work? To explain that, let’s follow the lifecycle of a pipeline. Let’s suppose we use the Python SDK to write the pipeline. Before submitting the pipeline via the Job API to Beam’s JobServer, Beam would convert it to the Runner API, the language-agnostic format we described before. The JobServer is also a Beam component that handles the staging of the required dependencies during execution. The JobServer will then kick-off the translation which is similar to the classic Runner. However, an important change is the so-called <code>ExecutableStage</code> transform. It is essentially a ParDo transform that we already know but designed for holding language-dependent code. Beam tries to combine as many of these transforms into one “executable stage”. The result again is a Flink program which is then sent to the Flink cluster and executed there. The major difference compared to the classic Runner is that during execution we will start <em>environments</em> to execute the aforementioned <em>ExecutableStages</em>. The following environments are available:</p> |
| |
| <ul> |
| <li>Docker-based (the default)</li> |
| <li>Process-based (a simple process is started)</li> |
| <li>Externally-provided (K8s or other schedulers)</li> |
| <li>Embedded (intended for testing and only works with Java)</li> |
| </ul> |
| |
| <p>Environments hold the <em>SDK Harness</em> which is the code that handles the execution and the communication with the Runner over the Fn API. For example, when Flink executes Python code, it sends the data to the Python environment containing the Python SDK Harness. Sending data to an external process involves a minor overhead which we have measured to be 5-10% slower than the classic Java pipelines. However, Beam uses a fusion of transforms to execute as many transforms as possible in the same environment which share the same input or output. That’s why in real-world scenarios the overhead could be much lower.</p> |
| |
| <center> |
| <img src="/img/blog/2020-02-22-beam-on-flink/flink-runner-beam-language-portability-architecture.png" width="600px" alt="Language Portability Architecture in beam" /> |
| </center> |
| |
| <p>Environments can be present for many languages. This opens up an entirely new type of pipelines: cross-language pipelines. In cross-language pipelines we can combine transforms of two or more languages, e.g. a machine learning pipeline with the feature generation written in Java and the learning written in Python. All this can be run on top of Flink.</p> |
| |
| <h2 id="conclusion">Conclusion</h2> |
| |
| <p>Using Apache Beam with Apache Flink combines (a.) the power of Flink with (b.) the flexibility of Beam. All it takes to run Beam is a Flink cluster, which you may already have. Apache Beam’s fully-fledged Python API is probably the most compelling argument for using Beam with Flink, but the unified API which allows to “write-once” and “execute-anywhere” is also very appealing to Beam users. On top of this, features like side inputs and a rich connector ecosystem are also reasons why people like Beam.</p> |
| |
| <p>With the introduction of schemas, a new format for handling type information, Beam is heading in a similar direction as Flink with its type system which is essential for the Table API or SQL. Speaking of, the next Flink release will include a Python version of the Table API which is based on the language portability of Beam. Looking ahead, the Beam community plans to extend the support for interactive programs like notebooks. TFX, which is built with Beam, is a very powerful way to solve many problems around training and validating machine learning models.</p> |
| |
| <p>For many years, Beam and Flink have inspired and learned from each other. With the Python support being based on Beam in Flink, they only seem to come closer to each other. That’s all the better for the community, and also users have more options and functionality to choose from.</p> |
| |
| </article> |
| </div> |
| |
| <div class="row"> |
| <div id="disqus_thread"></div> |
| <script type="text/javascript"> |
| /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ |
| var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname |
| |
| /* * * DON'T EDIT BELOW THIS LINE * * */ |
| (function() { |
| var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; |
| dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; |
| (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); |
| })(); |
| </script> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| <hr /> |
| |
| <div class="row"> |
| <div class="footer text-center col-sm-12"> |
| <p>Copyright © 2014-2019 <a href="http://apache.org">The Apache Software Foundation</a>. All Rights Reserved.</p> |
| <p>Apache Flink, Flink®, Apache®, the squirrel logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> |
| <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> |
| </div> |
| </div> |
| </div><!-- /.container --> |
| |
| <!-- Include all compiled plugins (below), or include individual files as needed --> |
| <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery.matchHeight/0.7.0/jquery.matchHeight-min.js"></script> |
| <script src="/js/codetabs.js"></script> |
| <script src="/js/stickysidebar.js"></script> |
| |
| <!-- Google Analytics --> |
| <script> |
| (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ |
| (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), |
| m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) |
| })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); |
| |
| ga('create', 'UA-52545728-1', 'auto'); |
| ga('send', 'pageview'); |
| </script> |
| </body> |
| </html> |
