docs-archive/1.10.8/executor/celery.html - airflow-site - Git at Google



 <!--
 Javascript to render AIRFLOW-XXX and PR references in text
 as HTML links.

 Overrides extrahead block from sphinx_rtd_theme
 https://www.sphinx-doc.org/en/master/templating.html
 -->


 <!DOCTYPE html>
 <!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
 <!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
 <head>
   <meta charset="utf-8">

   <meta name="viewport" content="width=device-width, initial-scale=1.0">

   <title>Celery Executor &mdash; Airflow Documentation</title>


     <link rel="shortcut icon" href="../_static/pin_32.png"/>


   <script type="text/javascript" src="../_static/js/modernizr.min.js"></script>


       <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
         <script type="text/javascript" src="../_static/jquery.js"></script>
         <script type="text/javascript" src="../_static/underscore.js"></script>
         <script type="text/javascript" src="../_static/doctools.js"></script>
         <script type="text/javascript" src="../_static/language_data.js"></script>
         <script type="text/javascript" src="../_static/jira-links.js"></script>

     <script type="text/javascript" src="../_static/js/theme.js"></script>


   <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
   <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
   <link rel="stylesheet" href="../_static/graphviz.css" type="text/css" />
   <link rel="stylesheet" href="../_static/exampleinclude.css" type="text/css" />
     <link rel="index" title="Index" href="../genindex.html" />
     <link rel="search" title="Search" href="../search.html" />
     <link rel="next" title="Dask Executor" href="dask.html" />
     <link rel="prev" title="Executor" href="index.html" />

   <script>
   </script>
   <style>

   </style>

 </head>

 <body class="wy-body-for-nav">


   <div class="wy-grid-for-nav">

     <nav data-toggle="wy-nav-shift" class="wy-nav-side">
       <div class="wy-side-scroll">
         <div class="wy-side-nav-search" >


             <a href="../index.html" class="icon icon-home"> Airflow


           </a>


               <div class="version">
                 1.10.8
               </div>


 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
     <input type="text" name="q" placeholder="Search docs" />
     <input type="hidden" name="check_keywords" value="yes" />
     <input type="hidden" name="area" value="default" />
   </form>
 </div>


         </div>

         <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">


               <ul class="current">
 <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../howto/index.html">How-to Guides</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface Reference</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling &amp; Triggers</a></li>
 <li class="toctree-l1 current"><a class="reference internal" href="index.html">Executor</a><ul class="current">
 <li class="toctree-l2 current"><a class="current reference internal" href="#">Celery Executor</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="#architecture">Architecture</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="dask.html">Dask Executor</a></li>
 <li class="toctree-l2"><a class="reference internal" href="debug.html">Debug Executor</a></li>
 <li class="toctree-l2"><a class="reference internal" href="kubernetes.html">Kubernetes Executor</a></li>
 <li class="toctree-l2"><a class="reference internal" href="mesos.html">Scaling Out with Mesos (community contributed)</a></li>
 </ul>
 </li>
 <li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../timezone.html">Time zones</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../api.html">REST API Reference</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../metrics.html">Metrics</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../errors.html">Error Tracking</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../kubernetes.html">Kubernetes</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../lineage.html">Lineage</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../dag-serialization.html">DAG Serialization</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../changelog.html">Changelog</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../best-practices.html">Best Practices</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../macros.html">Macros reference</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../privacy_notice.html">Privacy Notice</a></li>
 </ul>
 <p class="caption"><span class="caption-text">References</span></p>
 <ul>
 <li class="toctree-l1"><a class="reference internal" href="../_api/index.html">Python API</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../configurations-ref.html">Configurations</a></li>
 </ul>


         </div>
       </div>
     </nav>

     <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">


       <nav class="wy-nav-top" aria-label="top navigation">

           <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
           <a href="../index.html">Airflow</a>

       </nav>


       <div class="wy-nav-content">

         <div class="rst-content">


 <div role="navigation" aria-label="breadcrumbs navigation">

   <ul class="wy-breadcrumbs">

       <li><a href="../index.html">Docs</a> &raquo;</li>

           <li><a href="index.html">Executor</a> &raquo;</li>

       <li>Celery Executor</li>


       <li class="wy-breadcrumbs-aside">


             <a href="../_sources/executor/celery.rst.txt" rel="nofollow"> View page source</a>


       </li>

   </ul>


   <hr/>
 </div>
           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
            <div itemprop="articleBody">

   <blockquote>
 <div></div></blockquote>
 <div class="section" id="celery-executor">
 <h1>Celery Executor<a class="headerlink" href="#celery-executor" title="Permalink to this headline">¶</a></h1>
 <p><code class="docutils literal notranslate"><span class="pre">CeleryExecutor</span></code> is one of the ways you can scale out the number of workers. For this
 to work, you need to setup a Celery backend (<strong>RabbitMQ</strong>, <strong>Redis</strong>, …) and
 change your <code class="docutils literal notranslate"><span class="pre">airflow.cfg</span></code> to point the executor parameter to
 <code class="docutils literal notranslate"><span class="pre">CeleryExecutor</span></code> and provide the related Celery settings.</p>
 <p>For more information about setting up a Celery broker, refer to the
 exhaustive <a class="reference external" href="http://docs.celeryproject.org/en/latest/getting-started/brokers/index.html">Celery documentation on the topic</a>.</p>
 <p>Here are a few imperative requirements for your workers:</p>
 <ul class="simple">
 <li><p><code class="docutils literal notranslate"><span class="pre">airflow</span></code> needs to be installed, and the CLI needs to be in the path</p></li>
 <li><p>Airflow configuration settings should be homogeneous across the cluster</p></li>
 <li><p>Operators that are executed on the worker need to have their dependencies
 met in that context. For example, if you use the <code class="docutils literal notranslate"><span class="pre">HiveOperator</span></code>,
 the hive CLI needs to be installed on that box, or if you use the
 <code class="docutils literal notranslate"><span class="pre">MySqlOperator</span></code>, the required Python library needs to be available in
 the <code class="docutils literal notranslate"><span class="pre">PYTHONPATH</span></code> somehow</p></li>
 <li><p>The worker needs to have access to its <code class="docutils literal notranslate"><span class="pre">DAGS_FOLDER</span></code>, and you need to
 synchronize the filesystems by your own means. A common setup would be to
 store your <code class="docutils literal notranslate"><span class="pre">DAGS_FOLDER</span></code> in a Git repository and sync it across machines using
 Chef, Puppet, Ansible, or whatever you use to configure machines in your
 environment. If all your boxes have a common mount point, having your
 pipelines files shared there should work as well</p></li>
 </ul>
 <p>To kick off a worker, you need to setup Airflow and kick off the worker
 subcommand</p>
 <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>airflow worker
 </pre></div>
 </div>
 <p>Your worker should start picking up tasks as soon as they get fired in
 its direction.</p>
 <p>Note that you can also run “Celery Flower”, a web UI built on top of Celery,
 to monitor your workers. You can use the shortcut command <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">flower</span></code>
 to start a Flower web server.</p>
 <p>Please note that you must have the <code class="docutils literal notranslate"><span class="pre">flower</span></code> python library already installed on your system. The recommend way is to install the airflow celery bundle.</p>
 <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>pip install <span class="s1">&#39;apache-airflow[celery]&#39;</span>
 </pre></div>
 </div>
 <p>Some caveats:</p>
 <ul class="simple">
 <li><p>Make sure to use a database backed result backend</p></li>
 <li><p>Make sure to set a visibility timeout in <code class="docutils literal notranslate"><span class="pre">[celery_broker_transport_options]</span></code> that exceeds the ETA of your longest running task</p></li>
 <li><p>Tasks can consume resources. Make sure your worker has enough resources to run <code class="docutils literal notranslate"><span class="pre">worker_concurrency</span></code> tasks</p></li>
 <li><p>Queue names are limited to 256 characters, but each broker backend might have its own restrictions</p></li>
 </ul>
 <div class="section" id="architecture">
 <h2>Architecture<a class="headerlink" href="#architecture" title="Permalink to this headline">¶</a></h2>
 digraph A{
     rankdir=&quot;TB&quot;
     node[shape=&quot;rectangle&quot;, style=&quot;rounded&quot;]


     subgraph cluster {
         label=&quot;Cluster&quot;;
         {rank = same; dag; database}
         {rank = same; workers; scheduler; web}

         workers[label=&quot;Workers&quot;]
         scheduler[label=&quot;Scheduler&quot;]
         web[label=&quot;Web server&quot;]
         database[label=&quot;Database&quot;]
         dag[label=&quot;DAG files&quot;]

         subgraph cluster_queue {
             label=&quot;Celery&quot;;
             {rank = same; queue_broker; queue_result_backend}
             queue_broker[label=&quot;Queue broker&quot;]
             queue_result_backend[label=&quot;Result backend&quot;]
         }

         web-&gt;workers[label=&quot;1&quot;]
         web-&gt;dag[label=&quot;2&quot;]
         web-&gt;database[label=&quot;3&quot;]

         workers-&gt;dag[label=&quot;4&quot;]
         workers-&gt;database[label=&quot;5&quot;]
         workers-&gt;queue_result_backend[label=&quot;6&quot;]
         workers-&gt;queue_broker[label=&quot;7&quot;]

         scheduler-&gt;dag[label=&quot;8&quot;]
         scheduler-&gt;database[label=&quot;9&quot;]
         scheduler-&gt;queue_result_backend[label=&quot;10&quot;]
         scheduler-&gt;queue_broker[label=&quot;11&quot;]
     }
 }<p>Airflow consist of several components:</p>
 <ul class="simple">
 <li><p><strong>Workers</strong> - Execute the assigned tasks</p></li>
 <li><p><strong>Scheduler</strong> - Responsible for adding the necessary tasks to the queue</p></li>
 <li><p><strong>Web server</strong> - HTTP Server provides access to DAG/task status information</p></li>
 <li><p><strong>Database</strong> - Contains information about the status of tasks, DAGs, Variables, connections, etc.</p></li>
 <li><p><strong>Celery</strong> - Queue mechanism</p></li>
 </ul>
 <p>Please note that the queue at Celery consists of two components:</p>
 <ul class="simple">
 <li><p><strong>Broker</strong> - Stores commands for execution</p></li>
 <li><p><strong>Result backend</strong> - Stores status of completed commands</p></li>
 </ul>
 <p>The components communicate with each other in many places</p>
 <ul class="simple">
 <li><p>[1] <strong>Web server</strong> –&gt; <strong>Workers</strong> - Fetches task execution logs</p></li>
 <li><p>[2] <strong>Web server</strong> –&gt; <strong>DAG files</strong> - Reveal the DAG structure</p></li>
 <li><p>[3] <strong>Web server</strong> –&gt; <strong>Database</strong> - Fetch the status of the tasks</p></li>
 <li><p>[4] <strong>Workers</strong> –&gt; <strong>DAG files</strong> - Reveal the DAG structure and execute the tasks</p></li>
 <li><p>[5] <strong>Workers</strong> –&gt; <strong>Database</strong> - Gets and stores information about connection configuration, variables and XCOM.</p></li>
 <li><p>[6] <strong>Workers</strong> –&gt; <strong>Celery’s result backend</strong> - Saves the status of tasks</p></li>
 <li><p>[7] <strong>Workers</strong> –&gt; <strong>Celery’s broker</strong> - Stores commands for execution</p></li>
 <li><p>[8] <strong>Scheduler</strong> –&gt; <strong>Database</strong> - Store a DAG run and related tasks</p></li>
 <li><p>[9] <strong>Scheduler</strong> –&gt; <strong>DAG files</strong> - Reveal the DAG structure and execute the tasks</p></li>
 <li><p>[10] <strong>Scheduler</strong> –&gt; <strong>Celery’s result backend</strong> - Gets information about the status of completed tasks</p></li>
 <li><p>[11] <strong>Scheduler</strong> –&gt; <strong>Celery’s broker</strong> - Put the commands to be executed</p></li>
 </ul>
 </div>
 </div>


            </div>

           </div>


 <footer>

     <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">

         <a href="dask.html" class="btn btn-neutral float-right" title="Dask Executor" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>


         <a href="index.html" class="btn btn-neutral float-left" title="Executor" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>

     </div>


   <hr/>

   <div role="contentinfo">
     <p>

     </p>
   </div>
   Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
   <div class="footer">This page uses <a href="https://analytics.google.com/">
     Google Analytics</a> to collect statistics. You can disable it by blocking
     the JavaScript coming from www.google-analytics.com. Check our
     <a href="../privacy_notice.html">Privacy Policy</a>
     for more details.
   </div>


 </footer>

         </div>
       </div>

     </section>

   </div>


   <script type="text/javascript">
       jQuery(function () {
           SphinxRtdTheme.Navigation.enable(true);
       });
   </script>


     <!-- Theme Analytics -->
     <script>
     (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
       (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
       m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
     })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

     ga('create', 'UA-140539454-1', 'auto');
     ga('send', 'pageview');
     </script>


 </body>
 </html>


	<!--
	Javascript to render AIRFLOW-XXX and PR references in text
	as HTML links.

	Overrides extrahead block from sphinx_rtd_theme
	https://www.sphinx-doc.org/en/master/templating.html
	-->


	<!DOCTYPE html>
	<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
	<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
	<head>
	<meta charset="utf-8">

	<meta name="viewport" content="width=device-width, initial-scale=1.0">

	<title>Celery Executor — Airflow Documentation</title>




	<link rel="shortcut icon" href="../_static/pin_32.png"/>





	<script type="text/javascript" src="../_static/js/modernizr.min.js"></script>


	<script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
	<script type="text/javascript" src="../_static/jquery.js"></script>
	<script type="text/javascript" src="../_static/underscore.js"></script>
	<script type="text/javascript" src="../_static/doctools.js"></script>
	<script type="text/javascript" src="../_static/language_data.js"></script>
	<script type="text/javascript" src="../_static/jira-links.js"></script>

	<script type="text/javascript" src="../_static/js/theme.js"></script>




	<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
	<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
	<link rel="stylesheet" href="../_static/graphviz.css" type="text/css" />
	<link rel="stylesheet" href="../_static/exampleinclude.css" type="text/css" />
	<link rel="index" title="Index" href="../genindex.html" />
	<link rel="search" title="Search" href="../search.html" />
	<link rel="next" title="Dask Executor" href="dask.html" />
	<link rel="prev" title="Executor" href="index.html" />

	<script>
	</script>
	<style>

	</style>

	</head>

	<body class="wy-body-for-nav">


	<div class="wy-grid-for-nav">

	<nav data-toggle="wy-nav-shift" class="wy-nav-side">
	<div class="wy-side-scroll">
	<div class="wy-side-nav-search" >



	<a href="../index.html" class="icon icon-home"> Airflow



	</a>




	<div class="version">
	1.10.8
	</div>




	<div role="search">
	<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
	<input type="text" name="q" placeholder="Search docs" />
	<input type="hidden" name="check_keywords" value="yes" />
	<input type="hidden" name="area" value="default" />
	</form>
	</div>


	</div>

	<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">






	<ul class="current">
	<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../howto/index.html">How-to Guides</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface Reference</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li>
	<li class="toctree-l1 current"><a class="reference internal" href="index.html">Executor</a><ul class="current">
	<li class="toctree-l2 current"><a class="current reference internal" href="#">Celery Executor</a><ul>
	<li class="toctree-l3"><a class="reference internal" href="#architecture">Architecture</a></li>
	</ul>
	</li>
	<li class="toctree-l2"><a class="reference internal" href="dask.html">Dask Executor</a></li>
	<li class="toctree-l2"><a class="reference internal" href="debug.html">Debug Executor</a></li>
	<li class="toctree-l2"><a class="reference internal" href="kubernetes.html">Kubernetes Executor</a></li>
	<li class="toctree-l2"><a class="reference internal" href="mesos.html">Scaling Out with Mesos (community contributed)</a></li>
	</ul>
	</li>
	<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../timezone.html">Time zones</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../api.html">REST API Reference</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../metrics.html">Metrics</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../errors.html">Error Tracking</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../kubernetes.html">Kubernetes</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../lineage.html">Lineage</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../dag-serialization.html">DAG Serialization</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../changelog.html">Changelog</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../best-practices.html">Best Practices</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../macros.html">Macros reference</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../privacy_notice.html">Privacy Notice</a></li>
	</ul>
	<p class="caption"><span class="caption-text">References</span></p>
	<ul>
	<li class="toctree-l1"><a class="reference internal" href="../_api/index.html">Python API</a></li>
	<li class="toctree-l1"><a class="reference internal" href="../configurations-ref.html">Configurations</a></li>
	</ul>



	</div>
	</div>
	</nav>

	<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">


	<nav class="wy-nav-top" aria-label="top navigation">

	<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
	<a href="../index.html">Airflow</a>

	</nav>


	<div class="wy-nav-content">

	<div class="rst-content">

















	<div role="navigation" aria-label="breadcrumbs navigation">

	<ul class="wy-breadcrumbs">

	<li><a href="../index.html">Docs</a> »</li>

	<li><a href="index.html">Executor</a> »</li>

	<li>Celery Executor</li>


	<li class="wy-breadcrumbs-aside">


	<a href="../_sources/executor/celery.rst.txt" rel="nofollow"> View page source</a>


	</li>

	</ul>


	<hr/>
	</div>
	<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
	<div itemprop="articleBody">

	<blockquote>
	<div></div></blockquote>
	<div class="section" id="celery-executor">
	<h1>Celery Executor<a class="headerlink" href="#celery-executor" title="Permalink to this headline">¶</a></h1>
	<p><code class="docutils literal notranslate"><span class="pre">CeleryExecutor</span></code> is one of the ways you can scale out the number of workers. For this
	to work, you need to setup a Celery backend (<strong>RabbitMQ</strong>, <strong>Redis</strong>, …) and
	change your <code class="docutils literal notranslate"><span class="pre">airflow.cfg</span></code> to point the executor parameter to
	<code class="docutils literal notranslate"><span class="pre">CeleryExecutor</span></code> and provide the related Celery settings.</p>
	<p>For more information about setting up a Celery broker, refer to the
	exhaustive <a class="reference external" href="http://docs.celeryproject.org/en/latest/getting-started/brokers/index.html">Celery documentation on the topic</a>.</p>
	<p>Here are a few imperative requirements for your workers:</p>
	<ul class="simple">
	<li><p><code class="docutils literal notranslate"><span class="pre">airflow</span></code> needs to be installed, and the CLI needs to be in the path</p></li>
	<li><p>Airflow configuration settings should be homogeneous across the cluster</p></li>
	<li><p>Operators that are executed on the worker need to have their dependencies
	met in that context. For example, if you use the <code class="docutils literal notranslate"><span class="pre">HiveOperator</span></code>,
	the hive CLI needs to be installed on that box, or if you use the
	<code class="docutils literal notranslate"><span class="pre">MySqlOperator</span></code>, the required Python library needs to be available in
	the <code class="docutils literal notranslate"><span class="pre">PYTHONPATH</span></code> somehow</p></li>
	<li><p>The worker needs to have access to its <code class="docutils literal notranslate"><span class="pre">DAGS_FOLDER</span></code>, and you need to
	synchronize the filesystems by your own means. A common setup would be to
	store your <code class="docutils literal notranslate"><span class="pre">DAGS_FOLDER</span></code> in a Git repository and sync it across machines using
	Chef, Puppet, Ansible, or whatever you use to configure machines in your
	environment. If all your boxes have a common mount point, having your
	pipelines files shared there should work as well</p></li>
	</ul>
	<p>To kick off a worker, you need to setup Airflow and kick off the worker
	subcommand</p>
	<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>airflow worker
	</pre></div>
	</div>
	<p>Your worker should start picking up tasks as soon as they get fired in
	its direction.</p>
	<p>Note that you can also run “Celery Flower”, a web UI built on top of Celery,
	to monitor your workers. You can use the shortcut command <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">flower</span></code>
	to start a Flower web server.</p>
	<p>Please note that you must have the <code class="docutils literal notranslate"><span class="pre">flower</span></code> python library already installed on your system. The recommend way is to install the airflow celery bundle.</p>
	<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>pip install <span class="s1">'apache-airflow[celery]'</span>
	</pre></div>
	</div>
	<p>Some caveats:</p>
	<ul class="simple">
	<li><p>Make sure to use a database backed result backend</p></li>
	<li><p>Make sure to set a visibility timeout in <code class="docutils literal notranslate"><span class="pre">[celery_broker_transport_options]</span></code> that exceeds the ETA of your longest running task</p></li>
	<li><p>Tasks can consume resources. Make sure your worker has enough resources to run <code class="docutils literal notranslate"><span class="pre">worker_concurrency</span></code> tasks</p></li>
	<li><p>Queue names are limited to 256 characters, but each broker backend might have its own restrictions</p></li>
	</ul>
	<div class="section" id="architecture">
	<h2>Architecture<a class="headerlink" href="#architecture" title="Permalink to this headline">¶</a></h2>
	digraph A{
	rankdir="TB"
	node[shape="rectangle", style="rounded"]


	subgraph cluster {
	label="Cluster";
	{rank = same; dag; database}
	{rank = same; workers; scheduler; web}

	workers[label="Workers"]
	scheduler[label="Scheduler"]
	web[label="Web server"]
	database[label="Database"]
	dag[label="DAG files"]

	subgraph cluster_queue {
	label="Celery";
	{rank = same; queue_broker; queue_result_backend}
	queue_broker[label="Queue broker"]
	queue_result_backend[label="Result backend"]
	}

	web->workers[label="1"]
	web->dag[label="2"]
	web->database[label="3"]

	workers->dag[label="4"]
	workers->database[label="5"]
	workers->queue_result_backend[label="6"]
	workers->queue_broker[label="7"]

	scheduler->dag[label="8"]
	scheduler->database[label="9"]
	scheduler->queue_result_backend[label="10"]
	scheduler->queue_broker[label="11"]
	}
	}<p>Airflow consist of several components:</p>
	<ul class="simple">
	<li><p><strong>Workers</strong> - Execute the assigned tasks</p></li>
	<li><p><strong>Scheduler</strong> - Responsible for adding the necessary tasks to the queue</p></li>
	<li><p><strong>Web server</strong> - HTTP Server provides access to DAG/task status information</p></li>
	<li><p><strong>Database</strong> - Contains information about the status of tasks, DAGs, Variables, connections, etc.</p></li>
	<li><p><strong>Celery</strong> - Queue mechanism</p></li>
	</ul>
	<p>Please note that the queue at Celery consists of two components:</p>
	<ul class="simple">
	<li><p><strong>Broker</strong> - Stores commands for execution</p></li>
	<li><p><strong>Result backend</strong> - Stores status of completed commands</p></li>
	</ul>
	<p>The components communicate with each other in many places</p>
	<ul class="simple">
	<li><p>[1] <strong>Web server</strong> –> <strong>Workers</strong> - Fetches task execution logs</p></li>
	<li><p>[2] <strong>Web server</strong> –> <strong>DAG files</strong> - Reveal the DAG structure</p></li>
	<li><p>[3] <strong>Web server</strong> –> <strong>Database</strong> - Fetch the status of the tasks</p></li>
	<li><p>[4] <strong>Workers</strong> –> <strong>DAG files</strong> - Reveal the DAG structure and execute the tasks</p></li>
	<li><p>[5] <strong>Workers</strong> –> <strong>Database</strong> - Gets and stores information about connection configuration, variables and XCOM.</p></li>
	<li><p>[6] <strong>Workers</strong> –> <strong>Celery’s result backend</strong> - Saves the status of tasks</p></li>
	<li><p>[7] <strong>Workers</strong> –> <strong>Celery’s broker</strong> - Stores commands for execution</p></li>
	<li><p>[8] <strong>Scheduler</strong> –> <strong>Database</strong> - Store a DAG run and related tasks</p></li>
	<li><p>[9] <strong>Scheduler</strong> –> <strong>DAG files</strong> - Reveal the DAG structure and execute the tasks</p></li>
	<li><p>[10] <strong>Scheduler</strong> –> <strong>Celery’s result backend</strong> - Gets information about the status of completed tasks</p></li>
	<li><p>[11] <strong>Scheduler</strong> –> <strong>Celery’s broker</strong> - Put the commands to be executed</p></li>
	</ul>
	</div>
	</div>


	</div>

	</div>


	<footer>

	<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">

	<a href="dask.html" class="btn btn-neutral float-right" title="Dask Executor" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>


	<a href="index.html" class="btn btn-neutral float-left" title="Executor" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>

	</div>


	<hr/>

	<div role="contentinfo">
	<p>

	</p>
	</div>
	Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
	<div class="footer">This page uses <a href="https://analytics.google.com/">
	Google Analytics</a> to collect statistics. You can disable it by blocking
	the JavaScript coming from www.google-analytics.com. Check our
	<a href="../privacy_notice.html">Privacy Policy</a>
	for more details.
	</div>


	</footer>

	</div>
	</div>

	</section>

	</div>



	<script type="text/javascript">
	jQuery(function () {
	SphinxRtdTheme.Navigation.enable(true);
	});
	</script>




	<!-- Theme Analytics -->
	<script>
	(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]\|\|function(){
	(i[r].q=i[r].q\|\|[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
	m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
	})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

	ga('create', 'UA-140539454-1', 'auto');
	ga('send', 'pageview');
	</script>




	</body>
	</html>