| |
| <!DOCTYPE html> |
| <html lang="en" dir=ZgotmplZ> |
| |
| <head> |
| |
| |
| |
| <link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css"> |
| <script src="/bootstrap/js/bootstrap.bundle.min.js"></script> |
| <link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css"> |
| <script src="/js/anchor.min.js"></script> |
| <script src="/js/flink.js"></script> |
| <link rel="canonical" href="https://flink.apache.org/2020/07/14/application-deployment-in-flink-current-state-and-the-new-application-mode/"> |
| |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| <meta name="description" content="With the rise of stream processing and real-time analytics as a critical tool for modern businesses, an increasing number of organizations build platforms with Apache Flink at their core and offer it internally as a service. Many talks with related topics from companies like Uber, Netflix and Alibaba in the latest editions of Flink Forward further illustrate this trend. |
| These platforms aim at simplifying application submission internally by lifting all the operational burden from the end user."> |
| <meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="Application Deployment in Flink: Current State and the new Application Mode" /> |
| <meta property="og:description" content="With the rise of stream processing and real-time analytics as a critical tool for modern businesses, an increasing number of organizations build platforms with Apache Flink at their core and offer it internally as a service. Many talks with related topics from companies like Uber, Netflix and Alibaba in the latest editions of Flink Forward further illustrate this trend. |
| These platforms aim at simplifying application submission internally by lifting all the operational burden from the end user." /> |
| <meta property="og:type" content="article" /> |
| <meta property="og:url" content="https://flink.apache.org/2020/07/14/application-deployment-in-flink-current-state-and-the-new-application-mode/" /><meta property="article:section" content="posts" /> |
| <meta property="article:published_time" content="2020-07-14T08:00:00+00:00" /> |
| <meta property="article:modified_time" content="2020-07-14T08:00:00+00:00" /> |
| <title>Application Deployment in Flink: Current State and the new Application Mode | Apache Flink</title> |
| <link rel="manifest" href="/manifest.json"> |
| <link rel="icon" href="/favicon.png" type="image/x-icon"> |
| <link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU="> |
| <script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script> |
| <!-- |
| Made with Book Theme |
| https://github.com/alex-shpak/hugo-book |
| --> |
| |
| <meta name="generator" content="Hugo 0.124.1"> |
| |
| |
| <script> |
| var _paq = window._paq = window._paq || []; |
| |
| |
| _paq.push(['disableCookies']); |
| |
| _paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="//analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '1']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| |
| </head> |
| |
| <body dir=ZgotmplZ> |
| |
| |
| |
| <header> |
| <nav class="navbar navbar-expand-xl"> |
| <div class="container-fluid"> |
| <a class="navbar-brand" href="/"> |
| <img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle"> |
| <span>Apache Flink</span> |
| </a> |
| <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation"> |
| <i class="fa fa-bars navbar-toggler-icon"></i> |
| </button> |
| <div class="collapse navbar-collapse" id="navbarSupportedContent"> |
| <ul class="navbar-nav"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="nav-item dropdown"> |
| <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a> |
| <ul class="dropdown-menu"> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/security/">Security</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a> |
| |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| <li class="nav-item dropdown"> |
| <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a> |
| <ul class="dropdown-menu"> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| <li class="nav-item dropdown"> |
| <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a> |
| <ul class="dropdown-menu"> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| <li class="nav-item dropdown"> |
| <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a> |
| <ul class="dropdown-menu"> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a> |
| |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="nav-item"> |
| |
| |
| <a class="nav-link" href="/posts/">Flink Blog</a> |
| |
| |
| </li> |
| |
| |
| |
| |
| |
| <li class="nav-item"> |
| |
| |
| <a class="nav-link" href="/downloads/">Downloads</a> |
| |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </ul> |
| <div class="book-search"> |
| <div class="book-search-spinner hidden"> |
| <i class="fa fa-refresh fa-spin"></i> |
| </div> |
| <form class="search-bar d-flex" onsubmit="return false;"su> |
| <input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/"> |
| <i class="fa fa-search search"></i> |
| <i class="fa fa-circle-o-notch fa-spin spinner"></i> |
| </form> |
| <div class="book-search-spinner hidden"></div> |
| <ul id="book-search-results"></ul> |
| </div> |
| </div> |
| </div> |
| </nav> |
| <div class="navbar-clearfix"></div> |
| </header> |
| |
| |
| <main class="flex"> |
| <section class="container book-page"> |
| |
| <article class="markdown"> |
| <h1> |
| <a href="/2020/07/14/application-deployment-in-flink-current-state-and-the-new-application-mode/">Application Deployment in Flink: Current State and the new Application Mode</a> |
| </h1> |
| |
| |
| |
| July 14, 2020 - |
| |
| |
| |
| Kostas Kloudas |
| |
| <a href="https://twitter.com/kkloudas">(@kkloudas)</a> |
| |
| |
| |
| |
| <p><p>With the rise of stream processing and real-time analytics as a critical tool for modern |
| businesses, an increasing number of organizations build platforms with Apache Flink at their |
| core and offer it internally as a service. Many talks with related topics from companies |
| like <a href="https://www.youtube.com/watch?v=VX3S9POGAdU">Uber</a>, <a href="https://www.youtube.com/watch?v=VX3S9POGAdU">Netflix</a> |
| and <a href="https://www.youtube.com/watch?v=cH9UdK0yYjc">Alibaba</a> in the latest editions of Flink Forward further |
| illustrate this trend.</p> |
| <p>These platforms aim at simplifying application submission internally by lifting all the |
| operational burden from the end user. To submit Flink applications, these platforms |
| usually expose only a centralized or low-parallelism endpoint (<em>e.g.</em> a Web frontend) |
| for application submission that we will call the <em>Deployer</em>.</p> |
| <p>One of the roadblocks that platform developers and maintainers often mention is that the |
| Deployer can be a heavy resource consumer that is difficult to provision for. Provisioning |
| for average load can lead to the Deployer service being overwhelmed with deployment |
| requests (in the worst case, for all production applications in a short period of time), |
| while planning based on top load leads to unnecessary costs. Building on this observation, |
| Flink 1.11 introduces the <a href="//nightlies.apache.org/flink/flink-docs-stable/ops/deployment/#application-mode">Application Mode</a> |
| as a deployment option, which allows for a lightweight, more scalable application |
| submission process that manages to spread more evenly the application deployment load |
| across the nodes in the cluster.</p> |
| <p>In order to understand the problem and how the Application Mode solves it, we start by |
| describing briefly the current status of application execution in Flink, before |
| describing the architectural changes introduced by the new deployment mode and how to |
| leverage them.</p> |
| <h1 id="application-execution-in-flink"> |
| Application Execution in Flink |
| <a class="anchor" href="#application-execution-in-flink">#</a> |
| </h1> |
| <p>The execution of an application in Flink mainly involves three entities: the <em>Client</em>, |
| the <em>JobManager</em> and the <em>TaskManagers</em>. The Client is responsible for submitting the application to the |
| cluster, the JobManager is responsible for the necessary bookkeeping during execution, |
| and the TaskManagers are the ones doing the actual computation. For more details please |
| refer to <a href="//nightlies.apache.org/flink/flink-docs-stable/concepts/flink-architecture.html">Flink’s Architecture</a> |
| documentation page.</p> |
| <h2 id="current-deployment-modes"> |
| Current Deployment Modes |
| <a class="anchor" href="#current-deployment-modes">#</a> |
| </h2> |
| <p>Before the introduction of the Application Mode in version 1.11, Flink allowed users to execute an application either on a |
| <em>Session</em> or a <em>Per-Job Cluster</em>. The differences between the two have to do with the cluster |
| lifecycle and the resource isolation guarantees they provide.</p> |
| <h3 id="session-mode"> |
| Session Mode |
| <a class="anchor" href="#session-mode">#</a> |
| </h3> |
| <p>Session Mode assumes an already running cluster and uses the resources of that cluster to |
| execute any submitted application. Applications executed in the same (session) cluster use, |
| and consequently compete for, the same resources. This has the advantage that you do not |
| pay the resource overhead of spinning up a full cluster for every submitted job. But, if |
| one of the jobs misbehaves or brings down a TaskManager, then all jobs running on that |
| TaskManager will be affected by the failure. Apart from a negative impact on the job that |
| caused the failure, this implies a potential massive recovery process with all the |
| restarting jobs accessing the file system concurrently and making it unavailable to other |
| services. Additionally, having a single cluster running multiple jobs implies more load |
| for the JobManager, which is responsible for the bookkeeping of all the jobs in the |
| cluster. This mode is ideal for short jobs where startup latency is of high importance, |
| <em>e.g.</em> interactive queries.</p> |
| <h3 id="per-job-mode"> |
| Per-Job Mode |
| <a class="anchor" href="#per-job-mode">#</a> |
| </h3> |
| <p>In Per-Job Mode, the available cluster manager framework (<em>e.g.</em> YARN or Kubernetes) is |
| used to spin up a Flink cluster for each submitted job, which is available to that job |
| only. When the job finishes, the cluster is shut down and any lingering resources |
| (<em>e.g.</em> files) are cleaned up. This mode allows for better resource isolation, as a |
| misbehaving job cannot affect any other job. In addition, it spreads the load of |
| bookkeeping across multiple entities, as each application has its own JobManager. |
| Given the aforementioned resource isolation concerns of the Session Mode, users often |
| opt for the Per-Job Mode for long-running jobs which are willing to accept some increase |
| in startup latency in favor of resilience.</p> |
| <p>To summarize, in Session Mode, the cluster lifecycle is independent of any job running on |
| the cluster and all jobs running on the cluster share its resources. The per-job mode |
| chooses to pay the price of spinning up a cluster for every submitted job, in order to |
| provide better resource isolation guarantees as the resources are not shared across jobs. |
| In this case, the lifecycle of the cluster is bound to that of the job.</p> |
| <h2 id="application-submission"> |
| Application Submission |
| <a class="anchor" href="#application-submission">#</a> |
| </h2> |
| <p>Flink application execution consists of two stages: <em>pre-flight</em>, when the users’ <code>main()</code> |
| method is called; and <em>runtime</em>, which is triggered as soon as the user code calls <code>execute()</code>. |
| The <code>main()</code> method constructs the user program using one of Flink’s APIs |
| (DataStream API, Table API, DataSet API). When the <code>main()</code> method calls <code>env.execute()</code>, |
| the user-defined pipeline is translated into a form that Flink’s runtime can understand, |
| called the <em>job graph</em>, and it is shipped to the cluster.</p> |
| <p>Despite their differences, both session and per-job modes execute the application’s <code>main()</code> |
| method, <em>i.e.</em> the <em>pre-flight</em> phase, on the client side.<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup></p> |
| <p>This is usually not a problem for individual users who already have all the dependencies |
| of their jobs locally, and then submit their applications through a client running on |
| their machine. But in the case of submission through a remote entity like the Deployer, |
| this process includes:</p> |
| <ul> |
| <li> |
| <p>downloading the application’s dependencies locally,</p> |
| </li> |
| <li> |
| <p>executing the main()method to extract the job graph,</p> |
| </li> |
| <li> |
| <p>ship the job graph and its dependencies to the cluster for execution and,</p> |
| </li> |
| <li> |
| <p>potentially, wait for the result.</p> |
| </li> |
| </ul> |
| <p>This makes the Client a heavy resource consumer as it may need substantial network |
| bandwidth to download dependencies and ship binaries to the cluster, and CPU cycles to |
| execute the <code>main()</code> method. This problem is even more pronounced as more users share |
| the same Client.</p> |
| <div style="line-height:60%;"> |
| <br> |
| </div> |
| <center> |
| <img src="/img/blog/2020-07-14-application-mode/session-per-job.png" width="75%" alt="Session and Per-Job Mode"/> |
| </center> |
| <div style="line-height:150%;"> |
| <br> |
| </div> |
| <p>The figure above illustrates the two deployment modes using 3 applications depicted in |
| <span style="color:red">red</span>, <span style="color:blue">blue</span> and <span style="color:green">green</span>. |
| Each one has a parallelism of 3. The black rectangles represent |
| different processes: TaskManagers, JobManagers and the Deployer; and we assume a single |
| Deployer process in all scenarios. The colored triangles represent the load of the |
| submission process, while the colored rectangles represent the load of the TaskManager |
| and JobManager processes. As shown in the figure, the Deployer in both per-job and |
| session mode share the same load. Their difference lies in the distribution of the |
| tasks and the JobManager load. In the Session Mode, there is a single JobManager for |
| all the jobs in the cluster while in the per-job mode, there is one for each job. In |
| addition, tasks in Session Mode are assigned randomly to TaskManagers while in Per-Job |
| Mode, each TaskManager can only have tasks of a single job.</p> |
| <h1 id="application-mode"> |
| Application Mode |
| <a class="anchor" href="#application-mode">#</a> |
| </h1> |
| <img style="float: right;margin-left:10px;margin-right: 15px;" src="/img/blog/2020-07-14-application-mode/application.png" width="320px" alt="Application Mode"/> |
| <p>The Application Mode builds on the above observations and tries to combine the resource |
| isolation of the per-job mode with a lightweight and scalable application submission |
| process. To achieve this, it creates a cluster <em>per submitted application</em>, but this |
| time, the <code>main()</code> method of the application is executed on the JobManager.</p> |
| <p>Creating a cluster per application can be seen as creating a session cluster shared |
| only among the jobs of a particular application and torn down when the application |
| finishes. With this architecture, the Application Mode provides the same resource |
| isolation and load balancing guarantees as the Per-Job Mode, but at the granularity of |
| a whole application. This makes sense, as jobs belonging to the same application are |
| expected to be correlated and treated as a unit.</p> |
| <p>Executing the <code>main()</code> method on the JobManager allows saving the CPU cycles required |
| for extracting the job graph, but also the bandwidth required on the client for |
| downloading the dependencies locally and shipping the job graph and its dependencies |
| to the cluster. Furthermore, it spreads the network load more evenly, as there is one |
| JobManager per application. This is illustrated in the figure above, where we have the |
| same scenario as in the session and per-job deployment mode section, but this time |
| the client load has shifted to the JobManager of each application.</p> |
| <div class="alert alert-info" markdown="1"> |
| <span class="label label-info" style="display: inline-block"><span class="glyphicon glyphicon-info-sign" aria-hidden="true"></span> Note</span> |
| In the Application Mode, the main() method is executed on the cluster and not on the Client, as in the other modes. |
| This may have implications for your code as, for example, any paths you register in your |
| environment using the registerCachedFile() must be accessible by the JobManager of |
| your application. |
| </div> |
| <p>Compared to the Per-Job Mode, the Application Mode allows the submission of applications |
| consisting of multiple jobs. The order of job execution is not affected by the deployment |
| mode but by the call used to launch the job. Using the blocking <code>execute()</code> method |
| establishes an order and will lead to the execution of the “next” job being postponed |
| until “this” job finishes. In contrast, the non-blocking <code>executeAsync()</code> method will |
| immediately continue to submit the “next” job as soon as the current job is submitted.</p> |
| <h2 id="reducing-network-requirements"> |
| Reducing Network Requirements |
| <a class="anchor" href="#reducing-network-requirements">#</a> |
| </h2> |
| <p>As described above, by executing the application’s <code>main()</code> method on the JobManager, |
| the Application Mode manages to save a lot of the resources previously required during |
| job submission. But there is still room for improvement.</p> |
| <p>Focusing on YARN, which already supports all the optimizations mentioned here<sup id="fnref:2"><a href="#fn:2" class="footnote-ref" role="doc-noteref">2</a></sup>, and |
| even with the Application Mode in place, the Client is still required to send the user |
| jar to the JobManager. In addition, <em>for each application</em>, the Client has to ship to |
| the cluster the “flink-dist” directory which contains the binaries of the framework |
| itself, including the <code>flink-dist.jar</code>, <code>lib/</code> and <code>plugin/</code> directories. These two can |
| account for a substantial amount of bandwidth on the client side. Furthermore, shipping |
| the same flink-dist binaries on every submission is both a waste of bandwidth but also |
| of storage space which can be alleviated by simply allowing applications to share the |
| same binaries.</p> |
| <p>In Flink 1.11, we introduce options that allow the user to:</p> |
| <ol> |
| <li> |
| <p>Specify a remote path to a directory where YARN can find the Flink distribution binaries, and</p> |
| </li> |
| <li> |
| <p>Specify a remote path where YARN can find the user jar.</p> |
| </li> |
| </ol> |
| <p>For 1., we leverage YARN’s distributed cache and allow applications to share these |
| binaries. So, if an application happens to find copies of Flink on the local storage |
| of its TaskManager due to a previous application that was executed on the same |
| TaskManager, it will not even have to download it internally.</p> |
| <div class="alert alert-info" markdown="1"> |
| <span class="label label-info" style="display: inline-block"><span class="glyphicon glyphicon-info-sign" aria-hidden="true"></span> Note</span> |
| Both optimizations are available to all deployment modes on YARN, and not only the Application Mode. |
| </div> |
| <h1 id="example-application-mode-on-yarn"> |
| Example: Application Mode on Yarn |
| <a class="anchor" href="#example-application-mode-on-yarn">#</a> |
| </h1> |
| <p>For a full description, please refer to the official Flink documentation and more |
| specifically to the page that refers to your cluster management framework, <em>e.g.</em> |
| <a href="//nightlies.apache.org/flink/flink-docs-stable/ops/deployment/yarn_setup.html#run-an-application-in-application-mode">YARN</a> |
| or <a href="//nightlies.apache.org/flink/flink-docs-stable/ops/deployment/native_kubernetes.html#flink-kubernetes-application">Kubernetes</a>. |
| Here we will give some examples around YARN, where all the above features are available.</p> |
| <p>To launch an application in Application Mode, you can use:</p> |
| <pre><code><b>./bin/flink run-application -t yarn-application</b> ./MyApplication.jar</code></pre> |
| <p>With this command, all configuration parameters, such as the path to a savepoint to |
| be used to bootstrap the application’s state or the required JobManager/TaskManager |
| memory sizes, can be specified by their configuration option, prefixed by <code>-D</code>. For |
| a catalog of the available configuration options, please refer to Flink’s |
| <a href="//nightlies.apache.org/flink/flink-docs-stable/ops/config.html">configuration page</a>.</p> |
| <p>As an example, the command to specify the memory sizes of the JobManager and the |
| TaskManager would look like:</p> |
| <pre><code>./bin/flink run-application -t yarn-application \ |
| <b>-Djobmanager.memory.process.size=2048m</b> \ |
| <b>-Dtaskmanager.memory.process.size=4096m</b> \ |
| ./MyApplication.jar |
| </code></pre> |
| <p>As discussed earlier, the above will make sure that your application’s <code>main()</code> method |
| will be executed on the JobManager.</p> |
| <p>To further save the bandwidth of shipping the Flink distribution to the cluster, consider |
| pre-uploading the Flink distribution to a location accessible by YARN and using the |
| <code>yarn.provided.lib.dirs</code> configuration option, as shown below:</p> |
| <pre><code>./bin/flink run-application -t yarn-application \ |
| -Djobmanager.memory.process.size=2048m \ |
| -Dtaskmanager.memory.process.size=4096m \ |
| <b>-Dyarn.provided.lib.dirs="hdfs://myhdfs/remote-flink-dist-dir"</b> \ |
| ./MyApplication.jar |
| </code></pre> |
| <p>Finally, in order to further save the bandwidth required to submit your application jar, |
| you can pre-upload it to HDFS, and specify the remote path that points to |
| <code>./MyApplication.jar</code>, as shown below:</p> |
| <pre><code>./bin/flink run-application -t yarn-application \ |
| -Djobmanager.memory.process.size=2048m \ |
| -Dtaskmanager.memory.process.size=4096m \ |
| -Dyarn.provided.lib.dirs="hdfs://myhdfs/remote-flink-dist-dir" \ |
| <b>hdfs://myhdfs/jars/MyApplication.jar</b> |
| </code></pre> |
| <p>This will make the job submission extra lightweight as the needed Flink jars and the |
| application jar are going to be picked up from the specified remote locations rather |
| than be shipped to the cluster by the Client. The only thing the Client will ship to |
| the cluster is the configuration of your application which includes all the |
| aforementioned paths.</p> |
| <h1 id="conclusion"> |
| Conclusion |
| <a class="anchor" href="#conclusion">#</a> |
| </h1> |
| <p>We hope that this discussion helped you understand the differences between the various |
| deployment modes offered by Flink and will help you to make informed decisions about |
| which one is suitable in your own setup. Feel free to play around with them and report |
| any issues you may find. If you have any questions or requests, do not hesitate to post |
| them in the <a href="https://wints.github.io/flink-web//community.html#mailing-lists">mailing lists</a> |
| and, hopefully, see you (virtually) at one of our conferences or meetups soon!</p> |
| <div class="footnotes" role="doc-endnotes"> |
| <hr> |
| <ol> |
| <li id="fn:1"> |
| <p>The only exceptions are the Web Submission and the Standalone per-job implementation. <a href="#fnref:1" class="footnote-backref" role="doc-backlink">↩︎</a></p> |
| </li> |
| <li id="fn:2"> |
| <p>Support for Kubernetes will come soon. <a href="#fnref:2" class="footnote-backref" role="doc-backlink">↩︎</a></p> |
| </li> |
| </ol> |
| </div> |
| </p> |
| </article> |
| |
| |
| |
| |
| |
| |
| |
| <div class="edit-this-page"> |
| <p> |
| <a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a> |
| </p> |
| <p> |
| <a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2020-07-14-application-mode.md"> |
| Edit This Page<i class="fa fa-edit fa-fw"></i> |
| </a> |
| </p> |
| </div> |
| |
| </section> |
| |
| <aside class="book-toc"> |
| |
| |
| |
| <nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3> |
| <ul> |
| <li><a href="#application-execution-in-flink">Application Execution in Flink</a> |
| <ul> |
| <li><a href="#current-deployment-modes">Current Deployment Modes</a> |
| <ul> |
| <li><a href="#session-mode">Session Mode</a></li> |
| <li><a href="#per-job-mode">Per-Job Mode</a></li> |
| </ul> |
| </li> |
| <li><a href="#application-submission">Application Submission</a></li> |
| </ul> |
| </li> |
| <li><a href="#application-mode">Application Mode</a> |
| <ul> |
| <li><a href="#reducing-network-requirements">Reducing Network Requirements</a></li> |
| </ul> |
| </li> |
| <li><a href="#example-application-mode-on-yarn">Example: Application Mode on Yarn</a></li> |
| <li><a href="#conclusion">Conclusion</a></li> |
| </ul> |
| </nav> |
| |
| |
| </aside> |
| <aside class="expand-toc hidden"> |
| <a class="toc" onclick="expandToc()" href="javascript:void(0)"> |
| <i class="fa fa-bars" aria-hidden="true"></i> |
| </a> |
| </aside> |
| |
| </main> |
| |
| <footer> |
| |
| |
| |
| <div class="separator"></div> |
| <div class="panels"> |
| <div class="wrapper"> |
| <div class="panel"> |
| <ul> |
| <li> |
| <a href="https://flink-packages.org/">flink-packages.org</a> |
| </li> |
| <li> |
| <a href="https://www.apache.org/">Apache Software Foundation</a> |
| </li> |
| <li> |
| <a href="https://www.apache.org/licenses/">License</a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li> |
| <a href="/zh/"> |
| <i class="fa fa-globe" aria-hidden="true"></i> 中文版 |
| </a> |
| </li> |
| |
| |
| |
| </ul> |
| </div> |
| <div class="panel"> |
| <ul> |
| <li> |
| <a href="/what-is-flink/security">Security</a--> |
| </li> |
| <li> |
| <a href="https://www.apache.org/foundation/sponsorship.html">Donate</a> |
| </li> |
| <li> |
| <a href="https://www.apache.org/foundation/thanks.html">Thanks</a> |
| </li> |
| </ul> |
| </div> |
| <div class="panel icons"> |
| <div> |
| <a href="/posts"> |
| <div class="icon flink-blog-icon"></div> |
| <span>Flink blog</span> |
| </a> |
| </div> |
| <div> |
| <a href="https://github.com/apache/flink"> |
| <div class="icon flink-github-icon"></div> |
| <span>Github</span> |
| </a> |
| </div> |
| <div> |
| <a href="https://twitter.com/apacheflink"> |
| <div class="icon flink-twitter-icon"></div> |
| <span>Twitter</span> |
| </a> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| <hr/> |
| |
| <div class="container disclaimer"> |
| <p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p> |
| </div> |
| |
| |
| |
| </footer> |
| |
| </body> |
| </html> |
| |
| |
| |
| |
| |
| |
