| |
| |
| <!DOCTYPE html> |
| <!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> |
| <!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> |
| <head> |
| <meta charset="utf-8"> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| |
| <title>Using Operators — Airflow Documentation</title> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> |
| <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> |
| <link rel="index" title="Index" href="../genindex.html" /> |
| <link rel="search" title="Search" href="../search.html" /> |
| <link rel="next" title="Managing Connections" href="manage-connections.html" /> |
| <link rel="prev" title="Initializing a Database Backend" href="initialize-database.html" /> |
| |
| |
| <script src="../_static/js/modernizr.min.js"></script> |
| |
| <script type="application/javascript"> |
| window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date; |
| ga("create", "UA-140539454-1", "auto"); |
| ga("send", "pageview"); |
| </script> |
| <script async src="https://www.google-analytics.com/analytics.js"></script> |
| </head> |
| |
| |
| <body class="wy-body-for-nav"> |
| |
| |
| <div class="wy-grid-for-nav"> |
| |
| |
| <nav data-toggle="wy-nav-shift" class="wy-nav-side"> |
| <div class="wy-side-scroll"> |
| <div class="wy-side-nav-search"> |
| |
| |
| |
| <a href="../index.html" class="icon icon-home"> Airflow |
| |
| |
| |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <div role="search"> |
| <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> |
| <input type="text" name="q" placeholder="Search docs" /> |
| <input type="hidden" name="check_keywords" value="yes" /> |
| <input type="hidden" name="area" value="default" /> |
| </form> |
| </div> |
| |
| |
| </div> |
| |
| <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> |
| |
| |
| |
| |
| |
| |
| <ul class="current"> |
| <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li> |
| <li class="toctree-l1 current"><a class="reference internal" href="index.html">How-to Guides</a><ul class="current"> |
| <li class="toctree-l2"><a class="reference internal" href="set-config.html">Setting Configuration Options</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="initialize-database.html">Initializing a Database Backend</a></li> |
| <li class="toctree-l2 current"><a class="current reference internal" href="#">Using Operators</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#bashoperator">BashOperator</a><ul> |
| <li class="toctree-l4"><a class="reference internal" href="#templating">Templating</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#troubleshooting">Troubleshooting</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l3"><a class="reference internal" href="#pythonoperator">PythonOperator</a><ul> |
| <li class="toctree-l4"><a class="reference internal" href="#passing-in-arguments">Passing in arguments</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#id1">Templating</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l3"><a class="reference internal" href="#google-cloud-platform-operators">Google Cloud Platform Operators</a><ul> |
| <li class="toctree-l4"><a class="reference internal" href="#googlecloudstoragetobigqueryoperator">GoogleCloudStorageToBigQueryOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#gceinstancestartoperator">GceInstanceStartOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#gceinstancestopoperator">GceInstanceStopOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#gcesetmachinetypeoperator">GceSetMachineTypeOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#gcffunctiondeleteoperator">GcfFunctionDeleteOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#gcffunctiondeployoperator">GcfFunctionDeployOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancedatabasecreateoperator">CloudSqlInstanceDatabaseCreateOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancedatabasedeleteoperator">CloudSqlInstanceDatabaseDeleteOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancedatabasepatchoperator">CloudSqlInstanceDatabasePatchOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancedeleteoperator">CloudSqlInstanceDeleteOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancecreateoperator">CloudSqlInstanceCreateOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancepatchoperator">CloudSqlInstancePatchOperator</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="manage-connections.html">Managing Connections</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="secure-connections.html">Securing Connections</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="write-logs.html">Writing Logs</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="executor/use-celery.html">Scaling Out with Celery</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="executor/use-dask.html">Scaling Out with Dask</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="executor/use-mesos.html">Scaling Out with Mesos (community contributed)</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="run-with-systemd.html">Running Airflow with systemd</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="run-with-upstart.html">Running Airflow with upstart</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="use-test-config.html">Using the Test Mode Configuration</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../timezone.html">Time zones</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../lineage.html">Lineage</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li> |
| </ul> |
| |
| |
| |
| </div> |
| </div> |
| </nav> |
| |
| <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> |
| |
| |
| <nav class="wy-nav-top" aria-label="top navigation"> |
| |
| <i data-toggle="wy-nav-top" class="fa fa-bars"></i> |
| <a href="../index.html">Airflow</a> |
| |
| </nav> |
| |
| |
| <div class="wy-nav-content"> |
| |
| <div class="rst-content"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <div role="navigation" aria-label="breadcrumbs navigation"> |
| |
| <ul class="wy-breadcrumbs"> |
| |
| <li><a href="../index.html">Docs</a> »</li> |
| |
| <li><a href="index.html">How-to Guides</a> »</li> |
| |
| <li>Using Operators</li> |
| |
| |
| <li class="wy-breadcrumbs-aside"> |
| |
| |
| <a href="../_sources/howto/operator.rst.txt" rel="nofollow"> View page source</a> |
| |
| |
| </li> |
| |
| </ul> |
| |
| |
| <hr/> |
| </div> |
| <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> |
| <div itemprop="articleBody"> |
| |
| <div class="section" id="using-operators"> |
| <h1>Using Operators<a class="headerlink" href="#using-operators" title="Permalink to this headline">¶</a></h1> |
| <p>An operator represents a single, ideally idempotent, task. Operators |
| determine what actually executes when your DAG runs.</p> |
| <p>See the <a class="reference internal" href="../concepts.html#concepts-operators"><span class="std std-ref">Operators Concepts</span></a> documentation and the |
| <a class="reference internal" href="../code.html#api-reference-operators"><span class="std std-ref">Operators API Reference</span></a> for more |
| information.</p> |
| <div class="contents local topic" id="contents"> |
| <ul class="simple"> |
| <li><a class="reference internal" href="#bashoperator" id="id28">BashOperator</a><ul> |
| <li><a class="reference internal" href="#templating" id="id29">Templating</a></li> |
| <li><a class="reference internal" href="#troubleshooting" id="id30">Troubleshooting</a><ul> |
| <li><a class="reference internal" href="#jinja-template-not-found" id="id31">Jinja template not found</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#pythonoperator" id="id32">PythonOperator</a><ul> |
| <li><a class="reference internal" href="#passing-in-arguments" id="id33">Passing in arguments</a></li> |
| <li><a class="reference internal" href="#id1" id="id34">Templating</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#google-cloud-platform-operators" id="id35">Google Cloud Platform Operators</a><ul> |
| <li><a class="reference internal" href="#googlecloudstoragetobigqueryoperator" id="id36">GoogleCloudStorageToBigQueryOperator</a></li> |
| <li><a class="reference internal" href="#gceinstancestartoperator" id="id37">GceInstanceStartOperator</a></li> |
| <li><a class="reference internal" href="#gceinstancestopoperator" id="id38">GceInstanceStopOperator</a></li> |
| <li><a class="reference internal" href="#gcesetmachinetypeoperator" id="id39">GceSetMachineTypeOperator</a></li> |
| <li><a class="reference internal" href="#gcffunctiondeleteoperator" id="id40">GcfFunctionDeleteOperator</a><ul> |
| <li><a class="reference internal" href="#id2" id="id41">Troubleshooting</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#gcffunctiondeployoperator" id="id42">GcfFunctionDeployOperator</a><ul> |
| <li><a class="reference internal" href="#id3" id="id43">Troubleshooting</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstancedatabasecreateoperator" id="id44">CloudSqlInstanceDatabaseCreateOperator</a><ul> |
| <li><a class="reference internal" href="#arguments" id="id45">Arguments</a></li> |
| <li><a class="reference internal" href="#using-the-operator" id="id46">Using the operator</a></li> |
| <li><a class="reference internal" href="#id5" id="id47">Templating</a></li> |
| <li><a class="reference internal" href="#more-information" id="id48">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstancedatabasedeleteoperator" id="id49">CloudSqlInstanceDatabaseDeleteOperator</a><ul> |
| <li><a class="reference internal" href="#id6" id="id50">Arguments</a></li> |
| <li><a class="reference internal" href="#id7" id="id51">Using the operator</a></li> |
| <li><a class="reference internal" href="#id8" id="id52">Templating</a></li> |
| <li><a class="reference internal" href="#id9" id="id53">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstancedatabasepatchoperator" id="id54">CloudSqlInstanceDatabasePatchOperator</a><ul> |
| <li><a class="reference internal" href="#id10" id="id55">Arguments</a></li> |
| <li><a class="reference internal" href="#id11" id="id56">Using the operator</a></li> |
| <li><a class="reference internal" href="#id12" id="id57">Templating</a></li> |
| <li><a class="reference internal" href="#id13" id="id58">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstancedeleteoperator" id="id59">CloudSqlInstanceDeleteOperator</a><ul> |
| <li><a class="reference internal" href="#id14" id="id60">Arguments</a></li> |
| <li><a class="reference internal" href="#id15" id="id61">Using the operator</a></li> |
| <li><a class="reference internal" href="#id16" id="id62">Templating</a></li> |
| <li><a class="reference internal" href="#id17" id="id63">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstancecreateoperator" id="id64">CloudSqlInstanceCreateOperator</a><ul> |
| <li><a class="reference internal" href="#id19" id="id65">Arguments</a></li> |
| <li><a class="reference internal" href="#id20" id="id66">Using the operator</a></li> |
| <li><a class="reference internal" href="#id21" id="id67">Templating</a></li> |
| <li><a class="reference internal" href="#id22" id="id68">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstancepatchoperator" id="id69">CloudSqlInstancePatchOperator</a><ul> |
| <li><a class="reference internal" href="#id24" id="id70">Arguments</a></li> |
| <li><a class="reference internal" href="#id25" id="id71">Using the operator</a></li> |
| <li><a class="reference internal" href="#id26" id="id72">Templating</a></li> |
| <li><a class="reference internal" href="#id27" id="id73">More information</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </div> |
| <div class="section" id="bashoperator"> |
| <h2><a class="toc-backref" href="#id28">BashOperator</a><a class="headerlink" href="#bashoperator" title="Permalink to this headline">¶</a></h2> |
| <p>Use the <a class="reference internal" href="../code.html#airflow.operators.bash_operator.BashOperator" title="airflow.operators.bash_operator.BashOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">BashOperator</span></code></a> to execute |
| commands in a <a class="reference external" href="https://www.gnu.org/software/bash/">Bash</a> shell.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">run_this</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'run_after_loop'</span><span class="p">,</span> |
| <span class="n">bash_command</span><span class="o">=</span><span class="s1">'echo 1'</span><span class="p">,</span> |
| <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="section" id="templating"> |
| <h3><a class="toc-backref" href="#id29">Templating</a><a class="headerlink" href="#templating" title="Permalink to this headline">¶</a></h3> |
| <p>You can use <a class="reference internal" href="../concepts.html#jinja-templating"><span class="std std-ref">Jinja templates</span></a> to parameterize the |
| <code class="docutils literal notranslate"><span class="pre">bash_command</span></code> argument.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">also_run_this</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'also_run_this'</span><span class="p">,</span> |
| <span class="n">bash_command</span><span class="o">=</span><span class="s1">'echo "run_id={{ run_id }} | dag_run={{ dag_run }}"'</span><span class="p">,</span> |
| <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="troubleshooting"> |
| <h3><a class="toc-backref" href="#id30">Troubleshooting</a><a class="headerlink" href="#troubleshooting" title="Permalink to this headline">¶</a></h3> |
| <div class="section" id="jinja-template-not-found"> |
| <h4><a class="toc-backref" href="#id31">Jinja template not found</a><a class="headerlink" href="#jinja-template-not-found" title="Permalink to this headline">¶</a></h4> |
| <p>Add a space after the script name when directly calling a Bash script with |
| the <code class="docutils literal notranslate"><span class="pre">bash_command</span></code> argument. This is because Airflow tries to apply a Jinja |
| template to it, which will fail.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">t2</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'bash_example'</span><span class="p">,</span> |
| |
| <span class="c1"># This fails with `Jinja template not found` error</span> |
| <span class="c1"># bash_command="/home/batcher/test.sh",</span> |
| |
| <span class="c1"># This works (has a space after)</span> |
| <span class="n">bash_command</span><span class="o">=</span><span class="s2">"/home/batcher/test.sh "</span><span class="p">,</span> |
| <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="pythonoperator"> |
| <h2><a class="toc-backref" href="#id32">PythonOperator</a><a class="headerlink" href="#pythonoperator" title="Permalink to this headline">¶</a></h2> |
| <p>Use the <a class="reference internal" href="../code.html#airflow.operators.python_operator.PythonOperator" title="airflow.operators.python_operator.PythonOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">PythonOperator</span></code></a> to execute |
| Python callables.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">print_context</span><span class="p">(</span><span class="n">ds</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> |
| <span class="n">pprint</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span> |
| <span class="k">print</span><span class="p">(</span><span class="n">ds</span><span class="p">)</span> |
| <span class="k">return</span> <span class="s1">'Whatever you return gets printed in the logs'</span> |
| |
| |
| <span class="n">run_this</span> <span class="o">=</span> <span class="n">PythonOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'print_the_context'</span><span class="p">,</span> |
| <span class="n">provide_context</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> |
| <span class="n">python_callable</span><span class="o">=</span><span class="n">print_context</span><span class="p">,</span> |
| <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="section" id="passing-in-arguments"> |
| <h3><a class="toc-backref" href="#id33">Passing in arguments</a><a class="headerlink" href="#passing-in-arguments" title="Permalink to this headline">¶</a></h3> |
| <p>Use the <code class="docutils literal notranslate"><span class="pre">op_args</span></code> and <code class="docutils literal notranslate"><span class="pre">op_kwargs</span></code> arguments to pass additional arguments |
| to the Python callable.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">my_sleeping_function</span><span class="p">(</span><span class="n">random_base</span><span class="p">):</span> |
| <span class="sd">"""This is a function that will run within the DAG execution"""</span> |
| <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">random_base</span><span class="p">)</span> |
| |
| |
| <span class="c1"># Generate 10 sleeping tasks, sleeping from 0 to 4 seconds respectively</span> |
| <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">5</span><span class="p">):</span> |
| <span class="n">task</span> <span class="o">=</span> <span class="n">PythonOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sleep_for_'</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span> |
| <span class="n">python_callable</span><span class="o">=</span><span class="n">my_sleeping_function</span><span class="p">,</span> |
| <span class="n">op_kwargs</span><span class="o">=</span><span class="p">{</span><span class="s1">'random_base'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="o">/</span> <span class="mi">10</span><span class="p">},</span> |
| <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> |
| <span class="p">)</span> |
| |
| <span class="n">run_this</span> <span class="o">>></span> <span class="n">task</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id1"> |
| <h3><a class="toc-backref" href="#id34">Templating</a><a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h3> |
| <p>When you set the <code class="docutils literal notranslate"><span class="pre">provide_context</span></code> argument to <code class="docutils literal notranslate"><span class="pre">True</span></code>, Airflow passes in |
| an additional set of keyword arguments: one for each of the <a class="reference internal" href="../code.html#macros"><span class="std std-ref">Jinja |
| template variables</span></a> and a <code class="docutils literal notranslate"><span class="pre">templates_dict</span></code> argument.</p> |
| <p>The <code class="docutils literal notranslate"><span class="pre">templates_dict</span></code> argument is templated, so each value in the dictionary |
| is evaluated as a <a class="reference internal" href="../concepts.html#jinja-templating"><span class="std std-ref">Jinja template</span></a>.</p> |
| </div> |
| </div> |
| <div class="section" id="google-cloud-platform-operators"> |
| <h2><a class="toc-backref" href="#id35">Google Cloud Platform Operators</a><a class="headerlink" href="#google-cloud-platform-operators" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="googlecloudstoragetobigqueryoperator"> |
| <h3><a class="toc-backref" href="#id36">GoogleCloudStorageToBigQueryOperator</a><a class="headerlink" href="#googlecloudstoragetobigqueryoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator" title="airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GoogleCloudStorageToBigQueryOperator</span></code></a> |
| to execute a BigQuery load job.</p> |
| </div> |
| <div class="section" id="gceinstancestartoperator"> |
| <h3><a class="toc-backref" href="#id37">GceInstanceStartOperator</a><a class="headerlink" href="#gceinstancestartoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Allows to start an existing Google Compute Engine instance.</p> |
| <p>In this example parameter values are extracted from Airflow variables. |
| Moreover, the <code class="docutils literal notranslate"><span class="pre">default_args</span></code> dict is used to pass common arguments to all operators in a single DAG.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">LOCATION</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'LOCATION'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">INSTANCE</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">SHORT_MACHINE_TYPE_NAME</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SHORT_MACHINE_TYPE_NAME'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">SET_MACHINE_TYPE_BODY</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s1">'machineType'</span><span class="p">:</span> <span class="s1">'zones/{}/machineTypes/{}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">LOCATION</span><span class="p">,</span> <span class="n">SHORT_MACHINE_TYPE_NAME</span><span class="p">)</span> |
| <span class="p">}</span> |
| |
| <span class="n">default_args</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s1">'start_date'</span><span class="p">:</span> <span class="n">airflow</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">dates</span><span class="o">.</span><span class="n">days_ago</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| <p>Define the <code class="xref py py-class docutils literal notranslate"><span class="pre">GceInstanceStartOperator</span></code> by passing the required arguments to the constructor.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_instance_start</span> <span class="o">=</span> <span class="n">GceInstanceStartOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> |
| <span class="n">zone</span><span class="o">=</span><span class="n">LOCATION</span><span class="p">,</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">INSTANCE</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_start_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="gceinstancestopoperator"> |
| <h3><a class="toc-backref" href="#id38">GceInstanceStopOperator</a><a class="headerlink" href="#gceinstancestopoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Allows to stop an existing Google Compute Engine instance.</p> |
| <p>For parameter definition take a look at <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator" title="airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GceInstanceStartOperator</span></code></a> above.</p> |
| <p>Define the <code class="xref py py-class docutils literal notranslate"><span class="pre">GceInstanceStopOperator</span></code> by passing the required arguments to the constructor.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_instance_stop</span> <span class="o">=</span> <span class="n">GceInstanceStopOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> |
| <span class="n">zone</span><span class="o">=</span><span class="n">LOCATION</span><span class="p">,</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">INSTANCE</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_stop_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="gcesetmachinetypeoperator"> |
| <h3><a class="toc-backref" href="#id39">GceSetMachineTypeOperator</a><a class="headerlink" href="#gcesetmachinetypeoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Allows to change the machine type for a stopped instance to the specified machine type.</p> |
| <p>For parameter definition take a look at <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator" title="airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GceInstanceStartOperator</span></code></a> above.</p> |
| <p>Define the <code class="xref py py-class docutils literal notranslate"><span class="pre">GceSetMachineTypeOperator</span></code> by passing the required arguments to the constructor.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_set_machine_type</span> <span class="o">=</span> <span class="n">GceSetMachineTypeOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> |
| <span class="n">zone</span><span class="o">=</span><span class="n">LOCATION</span><span class="p">,</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">INSTANCE</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">SET_MACHINE_TYPE_BODY</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_set_machine_type'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="gcffunctiondeleteoperator"> |
| <h3><a class="toc-backref" href="#id40">GcfFunctionDeleteOperator</a><a class="headerlink" href="#gcffunctiondeleteoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the <code class="docutils literal notranslate"><span class="pre">default_args</span></code> dict to pass arguments to the operator.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">LOCATION</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'LOCATION'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">ENTRYPOINT</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ENTRYPOINT'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="c1"># A fully-qualified name of the function to delete</span> |
| |
| <span class="n">FUNCTION_NAME</span> <span class="o">=</span> <span class="s1">'projects/{}/locations/{}/functions/{}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">PROJECT_ID</span><span class="p">,</span> <span class="n">LOCATION</span><span class="p">,</span> |
| <span class="n">ENTRYPOINT</span><span class="p">)</span> |
| <span class="n">default_args</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s1">'start_date'</span><span class="p">:</span> <span class="n">airflow</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">dates</span><span class="o">.</span><span class="n">days_ago</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| <p>Use the <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_function_operator.GcfFunctionDeleteOperator" title="airflow.contrib.operators.gcp_function_operator.GcfFunctionDeleteOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GcfFunctionDeleteOperator</span></code></a> |
| to delete a function from Google Cloud Functions.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span> <span class="n">t1</span> <span class="o">=</span> <span class="n">GcfFunctionDeleteOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s2">"gcf_delete_task"</span><span class="p">,</span> |
| <span class="n">name</span><span class="o">=</span><span class="n">FUNCTION_NAME</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="section" id="id2"> |
| <h4><a class="toc-backref" href="#id41">Troubleshooting</a><a class="headerlink" href="#id2" title="Permalink to this headline">¶</a></h4> |
| <p>If you want to run or deploy an operator using a service account and get “forbidden 403” |
| errors, it means that your service account does not have the correct |
| Cloud IAM permissions.</p> |
| <ol class="arabic simple"> |
| <li>Assign your Service Account the Cloud Functions Developer role.</li> |
| <li>Grant the user the Cloud IAM Service Account User role on the Cloud Functions runtime |
| service account.</li> |
| </ol> |
| <p>The typical way of assigning Cloud IAM permissions with <cite>gcloud</cite> is |
| shown below. Just replace PROJECT_ID with ID of your Google Cloud Platform project |
| and SERVICE_ACCOUNT_EMAIL with the email ID of your service account.</p> |
| <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>gcloud iam service-accounts add-iam-policy-binding <span class="se">\</span> |
| PROJECT_ID@appspot.gserviceaccount.com <span class="se">\</span> |
| --member<span class="o">=</span><span class="s2">"serviceAccount:[SERVICE_ACCOUNT_EMAIL]"</span> <span class="se">\</span> |
| --role<span class="o">=</span><span class="s2">"roles/iam.serviceAccountUser"</span> |
| </pre></div> |
| </div> |
| <p>See <a class="reference external" href="https://cloud.google.com/functions/docs/reference/iam/roles#adding_the_iam_service_agent_user_role_to_the_runtime_service_account">Adding the IAM service agent user role to the runtime service</a> for details</p> |
| </div> |
| </div> |
| <div class="section" id="gcffunctiondeployoperator"> |
| <h3><a class="toc-backref" href="#id42">GcfFunctionDeployOperator</a><a class="headerlink" href="#gcffunctiondeployoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_function_operator.GcfFunctionDeployOperator" title="airflow.contrib.operators.gcp_function_operator.GcfFunctionDeployOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GcfFunctionDeployOperator</span></code></a> |
| to deploy a function from Google Cloud Functions.</p> |
| <p>The following examples of Airflow variables show various variants and combinations |
| of default_args that you can use. The variables are defined as follows:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">LOCATION</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'LOCATION'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">SOURCE_ARCHIVE_URL</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SOURCE_ARCHIVE_URL'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">SOURCE_UPLOAD_URL</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SOURCE_UPLOAD_URL'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">SOURCE_REPOSITORY</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SOURCE_REPOSITORY'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">ZIP_PATH</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ZIP_PATH'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">ENTRYPOINT</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ENTRYPOINT'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">FUNCTION_NAME</span> <span class="o">=</span> <span class="s1">'projects/{}/locations/{}/functions/{}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">PROJECT_ID</span><span class="p">,</span> <span class="n">LOCATION</span><span class="p">,</span> |
| <span class="n">ENTRYPOINT</span><span class="p">)</span> |
| <span class="n">RUNTIME</span> <span class="o">=</span> <span class="s1">'nodejs6'</span> |
| <span class="n">VALIDATE_BODY</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'VALIDATE_BODY'</span><span class="p">,</span> <span class="bp">True</span><span class="p">)</span> |
| |
| </pre></div> |
| </div> |
| <p>With those variables you can define the body of the request:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">body</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"name"</span><span class="p">:</span> <span class="n">FUNCTION_NAME</span><span class="p">,</span> |
| <span class="s2">"entryPoint"</span><span class="p">:</span> <span class="n">ENTRYPOINT</span><span class="p">,</span> |
| <span class="s2">"runtime"</span><span class="p">:</span> <span class="n">RUNTIME</span><span class="p">,</span> |
| <span class="s2">"httpsTrigger"</span><span class="p">:</span> <span class="p">{}</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| <p>When you create a DAG, the default_args dictionary can be used to pass the body and |
| other arguments:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">default_args</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s1">'start_date'</span><span class="p">:</span> <span class="n">dates</span><span class="o">.</span><span class="n">days_ago</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> |
| <span class="s1">'project_id'</span><span class="p">:</span> <span class="n">PROJECT_ID</span><span class="p">,</span> |
| <span class="s1">'location'</span><span class="p">:</span> <span class="n">LOCATION</span><span class="p">,</span> |
| <span class="s1">'body'</span><span class="p">:</span> <span class="n">body</span><span class="p">,</span> |
| <span class="s1">'validate_body'</span><span class="p">:</span> <span class="n">VALIDATE_BODY</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| <p>Note that the neither the body nor the default args are complete in the above examples. |
| Depending on the set variables, there might be different variants on how to pass source |
| code related fields. Currently, you can pass either sourceArchiveUrl, sourceRepository |
| or sourceUploadUrl as described in the |
| <a class="reference external" href="https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions#CloudFunction">CloudFunction API specification</a>. |
| Additionally, default_args might contain zip_path parameter to run the extra step of |
| uploading the source code before deploying it. In the last case, you also need to |
| provide an empty <cite>sourceUploadUrl</cite> parameter in the body.</p> |
| <p>Based on the variables defined above, example logic of setting the source code |
| related fields is shown here:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">if</span> <span class="n">SOURCE_ARCHIVE_URL</span><span class="p">:</span> |
| <span class="n">body</span><span class="p">[</span><span class="s1">'sourceArchiveUrl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">SOURCE_ARCHIVE_URL</span> |
| <span class="k">elif</span> <span class="n">SOURCE_REPOSITORY</span><span class="p">:</span> |
| <span class="n">body</span><span class="p">[</span><span class="s1">'sourceRepository'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s1">'url'</span><span class="p">:</span> <span class="n">SOURCE_REPOSITORY</span> |
| <span class="p">}</span> |
| <span class="k">elif</span> <span class="n">ZIP_PATH</span><span class="p">:</span> |
| <span class="n">body</span><span class="p">[</span><span class="s1">'sourceUploadUrl'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">''</span> |
| <span class="n">default_args</span><span class="p">[</span><span class="s1">'zip_path'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ZIP_PATH</span> |
| <span class="k">elif</span> <span class="n">SOURCE_UPLOAD_URL</span><span class="p">:</span> |
| <span class="n">body</span><span class="p">[</span><span class="s1">'sourceUploadUrl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">SOURCE_UPLOAD_URL</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"Please provide one of the source_code parameters"</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>The code to create the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span> <span class="n">deploy_task</span> <span class="o">=</span> <span class="n">GcfFunctionDeployOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s2">"gcf_deploy_task"</span><span class="p">,</span> |
| <span class="n">name</span><span class="o">=</span><span class="n">FUNCTION_NAME</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="section" id="id3"> |
| <h4><a class="toc-backref" href="#id43">Troubleshooting</a><a class="headerlink" href="#id3" title="Permalink to this headline">¶</a></h4> |
| <p>If you want to run or deploy an operator using a service account and get “forbidden 403” |
| errors, it means that your service account does not have the correct |
| Cloud IAM permissions.</p> |
| <ol class="arabic simple"> |
| <li>Assign your Service Account the Cloud Functions Developer role.</li> |
| <li>Grant the user the Cloud IAM Service Account User role on the Cloud Functions runtime |
| service account.</li> |
| </ol> |
| <p>The typical way of assigning Cloud IAM permissions with <cite>gcloud</cite> is |
| shown below. Just replace PROJECT_ID with ID of your Google Cloud Platform project |
| and SERVICE_ACCOUNT_EMAIL with the email ID of your service account.</p> |
| <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>gcloud iam service-accounts add-iam-policy-binding <span class="se">\</span> |
| PROJECT_ID@appspot.gserviceaccount.com <span class="se">\</span> |
| --member<span class="o">=</span><span class="s2">"serviceAccount:[SERVICE_ACCOUNT_EMAIL]"</span> <span class="se">\</span> |
| --role<span class="o">=</span><span class="s2">"roles/iam.serviceAccountUser"</span> |
| </pre></div> |
| </div> |
| <p>See <a class="reference external" href="https://cloud.google.com/functions/docs/reference/iam/roles#adding_the_iam_service_agent_user_role_to_the_runtime_service_account">Adding the IAM service agent user role to the runtime service</a> for details</p> |
| <p>If the source code for your function is in Google Source Repository, make sure that |
| your service account has the Source Repository Viewer role so that the source code |
| can be downloaded if necessary.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstancedatabasecreateoperator"> |
| <h3><a class="toc-backref" href="#id44">CloudSqlInstanceDatabaseCreateOperator</a><a class="headerlink" href="#cloudsqlinstancedatabasecreateoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Creates a new database inside a Cloud SQL instance.</p> |
| <p>For parameter definition take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseCreateOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseCreateOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceDatabaseCreateOperator</span></code></a>.</p> |
| <div class="section" id="arguments"> |
| <h4><a class="toc-backref" href="#id45">Arguments</a><a class="headerlink" href="#arguments" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DB_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="using-the-operator"> |
| <h4><a class="toc-backref" href="#id46">Using the operator</a><a class="headerlink" href="#using-the-operator" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_db_create_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDatabaseCreateOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">db_create_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_db_create_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Example request body:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">db_create_body</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"instance"</span><span class="p">:</span> <span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="s2">"name"</span><span class="p">:</span> <span class="n">DB_NAME</span><span class="p">,</span> |
| <span class="s2">"project"</span><span class="p">:</span> <span class="n">PROJECT_ID</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id5"> |
| <h4><a class="toc-backref" href="#id47">Templating</a><a class="headerlink" href="#id5" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="more-information"> |
| <h4><a class="toc-backref" href="#id48">More information</a><a class="headerlink" href="#more-information" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/insert">Google Cloud SQL API documentation for database insert</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstancedatabasedeleteoperator"> |
| <h3><a class="toc-backref" href="#id49">CloudSqlInstanceDatabaseDeleteOperator</a><a class="headerlink" href="#cloudsqlinstancedatabasedeleteoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Deletes a database from a Cloud SQL instance.</p> |
| <p>For parameter definition take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseDeleteOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseDeleteOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceDatabaseDeleteOperator</span></code></a>.</p> |
| <div class="section" id="id6"> |
| <h4><a class="toc-backref" href="#id50">Arguments</a><a class="headerlink" href="#id6" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DB_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id7"> |
| <h4><a class="toc-backref" href="#id51">Using the operator</a><a class="headerlink" href="#id7" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_db_delete_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDatabaseDeleteOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">database</span><span class="o">=</span><span class="n">DB_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_db_delete_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id8"> |
| <h4><a class="toc-backref" href="#id52">Templating</a><a class="headerlink" href="#id8" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'database'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> |
| <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id9"> |
| <h4><a class="toc-backref" href="#id53">More information</a><a class="headerlink" href="#id9" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/delete">Google Cloud SQL API documentation for database delete</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstancedatabasepatchoperator"> |
| <h3><a class="toc-backref" href="#id54">CloudSqlInstanceDatabasePatchOperator</a><a class="headerlink" href="#cloudsqlinstancedatabasepatchoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Updates a resource containing information about a database inside a Cloud SQL instance |
| using patch semantics. |
| See: <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch">https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch</a></p> |
| <p>For parameter definition take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabasePatchOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabasePatchOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceDatabasePatchOperator</span></code></a>.</p> |
| <div class="section" id="id10"> |
| <h4><a class="toc-backref" href="#id55">Arguments</a><a class="headerlink" href="#id10" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DB_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id11"> |
| <h4><a class="toc-backref" href="#id56">Using the operator</a><a class="headerlink" href="#id11" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_db_patch_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDatabasePatchOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">db_patch_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">database</span><span class="o">=</span><span class="n">DB_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_db_patch_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Example request body:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">db_patch_body</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"charset"</span><span class="p">:</span> <span class="s2">"utf16"</span><span class="p">,</span> |
| <span class="s2">"collation"</span><span class="p">:</span> <span class="s2">"utf16_general_ci"</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id12"> |
| <h4><a class="toc-backref" href="#id57">Templating</a><a class="headerlink" href="#id12" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'database'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> |
| <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id13"> |
| <h4><a class="toc-backref" href="#id58">More information</a><a class="headerlink" href="#id13" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/patch">Google Cloud SQL API documentation for database patch</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstancedeleteoperator"> |
| <h3><a class="toc-backref" href="#id59">CloudSqlInstanceDeleteOperator</a><a class="headerlink" href="#cloudsqlinstancedeleteoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Deletes a Cloud SQL instance in Google Cloud Platform.</p> |
| <p>For parameter definition take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDeleteOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDeleteOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceDeleteOperator</span></code></a>.</p> |
| <div class="section" id="id14"> |
| <h4><a class="toc-backref" href="#id60">Arguments</a><a class="headerlink" href="#id14" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DB_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id15"> |
| <h4><a class="toc-backref" href="#id61">Using the operator</a><a class="headerlink" href="#id15" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_instance_delete_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDeleteOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_instance_delete_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id16"> |
| <h4><a class="toc-backref" href="#id62">Templating</a><a class="headerlink" href="#id16" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id17"> |
| <h4><a class="toc-backref" href="#id63">More information</a><a class="headerlink" href="#id17" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/delete">Google Cloud SQL API documentation for delete</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstancecreateoperator"> |
| <span id="id18"></span><h3><a class="toc-backref" href="#id64">CloudSqlInstanceCreateOperator</a><a class="headerlink" href="#cloudsqlinstancecreateoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Creates a new Cloud SQL instance in Google Cloud Platform.</p> |
| <p>For parameter definition take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceCreateOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceCreateOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceCreateOperator</span></code></a>.</p> |
| <p>If an instance with the same name exists, no action will be taken and the operator |
| will succeed.</p> |
| <div class="section" id="id19"> |
| <h4><a class="toc-backref" href="#id65">Arguments</a><a class="headerlink" href="#id19" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DB_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Example body defining the instance:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">body</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"name"</span><span class="p">:</span> <span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="s2">"settings"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"tier"</span><span class="p">:</span> <span class="s2">"db-n1-standard-1"</span><span class="p">,</span> |
| <span class="s2">"backupConfiguration"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"binaryLogEnabled"</span><span class="p">:</span> <span class="bp">True</span><span class="p">,</span> |
| <span class="s2">"enabled"</span><span class="p">:</span> <span class="bp">True</span><span class="p">,</span> |
| <span class="s2">"startTime"</span><span class="p">:</span> <span class="s2">"05:00"</span> |
| <span class="p">},</span> |
| <span class="s2">"activationPolicy"</span><span class="p">:</span> <span class="s2">"ALWAYS"</span><span class="p">,</span> |
| <span class="s2">"dataDiskSizeGb"</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span> |
| <span class="s2">"dataDiskType"</span><span class="p">:</span> <span class="s2">"PD_SSD"</span><span class="p">,</span> |
| <span class="s2">"databaseFlags"</span><span class="p">:</span> <span class="p">[],</span> |
| <span class="s2">"ipConfiguration"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"ipv4Enabled"</span><span class="p">:</span> <span class="bp">True</span><span class="p">,</span> |
| <span class="s2">"requireSsl"</span><span class="p">:</span> <span class="bp">True</span><span class="p">,</span> |
| <span class="p">},</span> |
| <span class="s2">"locationPreference"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"zone"</span><span class="p">:</span> <span class="s2">"europe-west4-a"</span> |
| <span class="p">},</span> |
| <span class="s2">"maintenanceWindow"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"hour"</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> |
| <span class="s2">"day"</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span> |
| <span class="s2">"updateTrack"</span><span class="p">:</span> <span class="s2">"canary"</span> |
| <span class="p">},</span> |
| <span class="s2">"pricingPlan"</span><span class="p">:</span> <span class="s2">"PER_USE"</span><span class="p">,</span> |
| <span class="s2">"replicationType"</span><span class="p">:</span> <span class="s2">"ASYNCHRONOUS"</span><span class="p">,</span> |
| <span class="s2">"storageAutoResize"</span><span class="p">:</span> <span class="bp">False</span><span class="p">,</span> |
| <span class="s2">"storageAutoResizeLimit"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> |
| <span class="s2">"userLabels"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"my-key"</span><span class="p">:</span> <span class="s2">"my-value"</span> |
| <span class="p">}</span> |
| <span class="p">},</span> |
| <span class="s2">"databaseVersion"</span><span class="p">:</span> <span class="s2">"MYSQL_5_7"</span><span class="p">,</span> |
| <span class="s2">"region"</span><span class="p">:</span> <span class="s2">"europe-west4"</span><span class="p">,</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id20"> |
| <h4><a class="toc-backref" href="#id66">Using the operator</a><a class="headerlink" href="#id20" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_instance_create_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceCreateOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_instance_create_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id21"> |
| <h4><a class="toc-backref" href="#id67">Templating</a><a class="headerlink" href="#id21" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id22"> |
| <h4><a class="toc-backref" href="#id68">More information</a><a class="headerlink" href="#id22" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/insert">Google Cloud SQL API documentation for insert</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstancepatchoperator"> |
| <span id="id23"></span><h3><a class="toc-backref" href="#id69">CloudSqlInstancePatchOperator</a><a class="headerlink" href="#cloudsqlinstancepatchoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Updates settings of a Cloud SQL instance in Google Cloud Platform (partial update).</p> |
| <p>For parameter definition take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstancePatchOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstancePatchOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstancePatchOperator</span></code></a>.</p> |
| <p>This is a partial update, so only values for the settings specified in the body |
| will be set / updated. The rest of the existing instance’s configuration will remain |
| unchanged.</p> |
| <div class="section" id="id24"> |
| <h4><a class="toc-backref" href="#id70">Arguments</a><a class="headerlink" href="#id24" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DB_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Example body defining the instance:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">patch_body</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"name"</span><span class="p">:</span> <span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="s2">"settings"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"dataDiskSizeGb"</span><span class="p">:</span> <span class="mi">35</span><span class="p">,</span> |
| <span class="s2">"maintenanceWindow"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"hour"</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> |
| <span class="s2">"day"</span><span class="p">:</span> <span class="mi">6</span><span class="p">,</span> |
| <span class="s2">"updateTrack"</span><span class="p">:</span> <span class="s2">"canary"</span> |
| <span class="p">},</span> |
| <span class="s2">"userLabels"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"my-key-patch"</span><span class="p">:</span> <span class="s2">"my-value-patch"</span> |
| <span class="p">}</span> |
| <span class="p">}</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id25"> |
| <h4><a class="toc-backref" href="#id71">Using the operator</a><a class="headerlink" href="#id25" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_instance_patch_task</span> <span class="o">=</span> <span class="n">CloudSqlInstancePatchOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">patch_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_instance_patch_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id26"> |
| <h4><a class="toc-backref" href="#id72">Templating</a><a class="headerlink" href="#id26" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id27"> |
| <h4><a class="toc-backref" href="#id73">More information</a><a class="headerlink" href="#id27" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/patch">Google Cloud SQL API documentation for patch</a>.</p> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| </div> |
| <footer> |
| |
| <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> |
| |
| <a href="manage-connections.html" class="btn btn-neutral float-right" title="Managing Connections" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> |
| |
| |
| <a href="initialize-database.html" class="btn btn-neutral" title="Initializing a Database Backend" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> |
| |
| </div> |
| |
| |
| <hr/> |
| |
| <div role="contentinfo"> |
| <p> |
| |
| </p> |
| </div> |
| Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. |
| |
| </footer> |
| |
| </div> |
| </div> |
| |
| </section> |
| |
| </div> |
| |
| |
| |
| |
| |
| |
| |
| <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> |
| <script type="text/javascript" src="../_static/jquery.js"></script> |
| <script type="text/javascript" src="../_static/underscore.js"></script> |
| <script type="text/javascript" src="../_static/doctools.js"></script> |
| |
| |
| |
| |
| <script type="text/javascript" src="../_static/js/theme.js"></script> |
| |
| <script type="text/javascript"> |
| jQuery(function () { |
| SphinxRtdTheme.Navigation.enable(true); |
| }); |
| </script> |
| |
| </body> |
| </html> |
