| |
| |
| <!DOCTYPE html> |
| <!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> |
| <!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> |
| <head> |
| <meta charset="utf-8"> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| |
| <title>Using Operators — Airflow Documentation</title> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> |
| <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> |
| <link rel="index" title="Index" href="../genindex.html" /> |
| <link rel="search" title="Search" href="../search.html" /> |
| <link rel="next" title="Managing Connections" href="manage-connections.html" /> |
| <link rel="prev" title="Initializing a Database Backend" href="initialize-database.html" /> |
| |
| |
| <script src="../_static/js/modernizr.min.js"></script> |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '13']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo --> |
| </head> |
| |
| |
| <body class="wy-body-for-nav"> |
| |
| |
| <div class="wy-grid-for-nav"> |
| |
| |
| <nav data-toggle="wy-nav-shift" class="wy-nav-side"> |
| <div class="wy-side-scroll"> |
| <div class="wy-side-nav-search"> |
| |
| |
| |
| <a href="../index.html" class="icon icon-home"> Airflow |
| |
| |
| |
| </a> |
| |
| |
| |
| |
| <div class="version"> |
| 1.10.2 |
| </div> |
| |
| |
| |
| |
| <div role="search"> |
| <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> |
| <input type="text" name="q" placeholder="Search docs" /> |
| <input type="hidden" name="check_keywords" value="yes" /> |
| <input type="hidden" name="area" value="default" /> |
| </form> |
| </div> |
| |
| |
| </div> |
| |
| <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> |
| |
| |
| |
| |
| |
| |
| <ul class="current"> |
| <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li> |
| <li class="toctree-l1 current"><a class="reference internal" href="index.html">How-to Guides</a><ul class="current"> |
| <li class="toctree-l2"><a class="reference internal" href="set-config.html">Setting Configuration Options</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="initialize-database.html">Initializing a Database Backend</a></li> |
| <li class="toctree-l2 current"><a class="current reference internal" href="#">Using Operators</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#bashoperator">BashOperator</a><ul> |
| <li class="toctree-l4"><a class="reference internal" href="#templating">Templating</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#troubleshooting">Troubleshooting</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l3"><a class="reference internal" href="#pythonoperator">PythonOperator</a><ul> |
| <li class="toctree-l4"><a class="reference internal" href="#passing-in-arguments">Passing in arguments</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#id1">Templating</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l3"><a class="reference internal" href="#google-cloud-storage-operators">Google Cloud Storage Operators</a><ul> |
| <li class="toctree-l4"><a class="reference internal" href="#googlecloudstoragetobigqueryoperator">GoogleCloudStorageToBigQueryOperator</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l3"><a class="reference internal" href="#google-compute-engine-operators">Google Compute Engine Operators</a><ul> |
| <li class="toctree-l4"><a class="reference internal" href="#gceinstancestartoperator">GceInstanceStartOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#gceinstancestopoperator">GceInstanceStopOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#gcesetmachinetypeoperator">GceSetMachineTypeOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#gceinstancetemplatecopyoperator">GceInstanceTemplateCopyOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#gceinstancegroupmanagerupdatetemplateoperator">GceInstanceGroupManagerUpdateTemplateOperator</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l3"><a class="reference internal" href="#google-cloud-bigtable-operators">Google Cloud Bigtable Operators</a><ul> |
| <li class="toctree-l4"><a class="reference internal" href="#bigtableinstancecreateoperator">BigtableInstanceCreateOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#bigtableinstancedeleteoperator">BigtableInstanceDeleteOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#bigtableclusterupdateoperator">BigtableClusterUpdateOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#bigtabletablecreateoperator">BigtableTableCreateOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#bigtabletabledeleteoperator">BigtableTableDeleteOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#bigtabletablewaitforreplicationsensor">BigtableTableWaitForReplicationSensor</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l3"><a class="reference internal" href="#google-cloud-functions-operators">Google Cloud Functions Operators</a><ul> |
| <li class="toctree-l4"><a class="reference internal" href="#gcffunctiondeleteoperator">GcfFunctionDeleteOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#gcffunctiondeployoperator">GcfFunctionDeployOperator</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l3"><a class="reference internal" href="#google-cloud-spanner-operators">Google Cloud Spanner Operators</a><ul> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudspannerinstancedatabasedeleteoperator">CloudSpannerInstanceDatabaseDeleteOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudspannerinstancedatabasedeployoperator">CloudSpannerInstanceDatabaseDeployOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudspannerinstancedatabaseupdateoperator">CloudSpannerInstanceDatabaseUpdateOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudspannerinstancedatabasequeryoperator">CloudSpannerInstanceDatabaseQueryOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudspannerinstancedeleteoperator">CloudSpannerInstanceDeleteOperator</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l3"><a class="reference internal" href="#google-cloud-sql-operators">Google Cloud Sql Operators</a><ul> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancedatabasecreateoperator">CloudSqlInstanceDatabaseCreateOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancedatabasedeleteoperator">CloudSqlInstanceDatabaseDeleteOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancedatabasepatchoperator">CloudSqlInstanceDatabasePatchOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancedeleteoperator">CloudSqlInstanceDeleteOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstanceexportoperator">CloudSqlInstanceExportOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstanceimportoperator">CloudSqlInstanceImportOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancecreateoperator">CloudSqlInstanceCreateOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancepatchoperator">CloudSqlInstancePatchOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#cloudsqlqueryoperator">CloudSqlQueryOperator</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l3"><a class="reference internal" href="#id100">Google Cloud Storage Operators</a><ul> |
| <li class="toctree-l4"><a class="reference internal" href="#googlecloudstoragebucketcreateaclentryoperator">GoogleCloudStorageBucketCreateAclEntryOperator</a></li> |
| <li class="toctree-l4"><a class="reference internal" href="#googlecloudstorageobjectcreateaclentryoperator">GoogleCloudStorageObjectCreateAclEntryOperator</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="manage-connections.html">Managing Connections</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="secure-connections.html">Securing Connections</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="write-logs.html">Writing Logs</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="executor/use-celery.html">Scaling Out with Celery</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="executor/use-dask.html">Scaling Out with Dask</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="executor/use-mesos.html">Scaling Out with Mesos (community contributed)</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="run-with-systemd.html">Running Airflow with systemd</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="run-with-upstart.html">Running Airflow with upstart</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="use-test-config.html">Using the Test Mode Configuration</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="check-health.html">Checking Airflow Health Status</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../timezone.html">Time zones</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../metrics.html">Metrics</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../kubernetes.html">Kubernetes</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../lineage.html">Lineage</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../changelog.html">Changelog</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li> |
| </ul> |
| |
| |
| |
| </div> |
| </div> |
| </nav> |
| |
| <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> |
| |
| |
| <nav class="wy-nav-top" aria-label="top navigation"> |
| |
| <i data-toggle="wy-nav-top" class="fa fa-bars"></i> |
| <a href="../index.html">Airflow</a> |
| |
| </nav> |
| |
| |
| <div class="wy-nav-content"> |
| |
| <div class="rst-content"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <div role="navigation" aria-label="breadcrumbs navigation"> |
| |
| <ul class="wy-breadcrumbs"> |
| |
| <li><a href="../index.html">Docs</a> »</li> |
| |
| <li><a href="index.html">How-to Guides</a> »</li> |
| |
| <li>Using Operators</li> |
| |
| |
| <li class="wy-breadcrumbs-aside"> |
| |
| |
| <a href="../_sources/howto/operator.rst.txt" rel="nofollow"> View page source</a> |
| |
| |
| </li> |
| |
| </ul> |
| |
| |
| <hr/> |
| </div> |
| <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> |
| <div itemprop="articleBody"> |
| |
| <div class="section" id="using-operators"> |
| <h1>Using Operators<a class="headerlink" href="#using-operators" title="Permalink to this headline">¶</a></h1> |
| <p>An operator represents a single, ideally idempotent, task. Operators |
| determine what actually executes when your DAG runs.</p> |
| <p>See the <a class="reference internal" href="../concepts.html#concepts-operators"><span class="std std-ref">Operators Concepts</span></a> documentation and the |
| <a class="reference internal" href="../code.html#api-reference-operators"><span class="std std-ref">Operators API Reference</span></a> for more |
| information.</p> |
| <div class="contents local topic" id="contents"> |
| <ul class="simple"> |
| <li><a class="reference internal" href="#bashoperator" id="id109">BashOperator</a><ul> |
| <li><a class="reference internal" href="#templating" id="id110">Templating</a></li> |
| <li><a class="reference internal" href="#troubleshooting" id="id111">Troubleshooting</a><ul> |
| <li><a class="reference internal" href="#jinja-template-not-found" id="id112">Jinja template not found</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#pythonoperator" id="id113">PythonOperator</a><ul> |
| <li><a class="reference internal" href="#passing-in-arguments" id="id114">Passing in arguments</a></li> |
| <li><a class="reference internal" href="#id1" id="id115">Templating</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#google-cloud-storage-operators" id="id116">Google Cloud Storage Operators</a><ul> |
| <li><a class="reference internal" href="#googlecloudstoragetobigqueryoperator" id="id117">GoogleCloudStorageToBigQueryOperator</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#google-compute-engine-operators" id="id118">Google Compute Engine Operators</a><ul> |
| <li><a class="reference internal" href="#gceinstancestartoperator" id="id119">GceInstanceStartOperator</a><ul> |
| <li><a class="reference internal" href="#arguments" id="id120">Arguments</a></li> |
| <li><a class="reference internal" href="#using-the-operator" id="id121">Using the operator</a></li> |
| <li><a class="reference internal" href="#id2" id="id122">Templating</a></li> |
| <li><a class="reference internal" href="#more-information" id="id123">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#gceinstancestopoperator" id="id124">GceInstanceStopOperator</a><ul> |
| <li><a class="reference internal" href="#id3" id="id125">Arguments</a></li> |
| <li><a class="reference internal" href="#id4" id="id126">Using the operator</a></li> |
| <li><a class="reference internal" href="#id5" id="id127">Templating</a></li> |
| <li><a class="reference internal" href="#id6" id="id128">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#gcesetmachinetypeoperator" id="id129">GceSetMachineTypeOperator</a><ul> |
| <li><a class="reference internal" href="#id8" id="id130">Arguments</a></li> |
| <li><a class="reference internal" href="#id9" id="id131">Using the operator</a></li> |
| <li><a class="reference internal" href="#id10" id="id132">Templating</a></li> |
| <li><a class="reference internal" href="#id11" id="id133">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#gceinstancetemplatecopyoperator" id="id134">GceInstanceTemplateCopyOperator</a><ul> |
| <li><a class="reference internal" href="#id13" id="id135">Arguments</a></li> |
| <li><a class="reference internal" href="#id14" id="id136">Using the operator</a></li> |
| <li><a class="reference internal" href="#id15" id="id137">Templating</a></li> |
| <li><a class="reference internal" href="#id16" id="id138">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#gceinstancegroupmanagerupdatetemplateoperator" id="id139">GceInstanceGroupManagerUpdateTemplateOperator</a><ul> |
| <li><a class="reference internal" href="#id18" id="id140">Arguments</a></li> |
| <li><a class="reference internal" href="#id19" id="id141">Using the operator</a></li> |
| <li><a class="reference internal" href="#id20" id="id142">Templating</a></li> |
| <li><a class="reference internal" href="#id21" id="id143">Troubleshooting</a></li> |
| <li><a class="reference internal" href="#id22" id="id144">More information</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#google-cloud-bigtable-operators" id="id145">Google Cloud Bigtable Operators</a><ul> |
| <li><a class="reference internal" href="#bigtableinstancecreateoperator" id="id146">BigtableInstanceCreateOperator</a><ul> |
| <li><a class="reference internal" href="#id24" id="id147">Using the operator</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#bigtableinstancedeleteoperator" id="id148">BigtableInstanceDeleteOperator</a><ul> |
| <li><a class="reference internal" href="#id25" id="id149">Using the operator</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#bigtableclusterupdateoperator" id="id150">BigtableClusterUpdateOperator</a><ul> |
| <li><a class="reference internal" href="#id26" id="id151">Using the operator</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#bigtabletablecreateoperator" id="id152">BigtableTableCreateOperator</a><ul> |
| <li><a class="reference internal" href="#id27" id="id153">Using the operator</a></li> |
| <li><a class="reference internal" href="#advanced" id="id154">Advanced</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#bigtabletabledeleteoperator" id="id155">BigtableTableDeleteOperator</a><ul> |
| <li><a class="reference internal" href="#id28" id="id156">Using the operator</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#bigtabletablewaitforreplicationsensor" id="id157">BigtableTableWaitForReplicationSensor</a><ul> |
| <li><a class="reference internal" href="#id29" id="id158">Using the operator</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#google-cloud-functions-operators" id="id159">Google Cloud Functions Operators</a><ul> |
| <li><a class="reference internal" href="#gcffunctiondeleteoperator" id="id160">GcfFunctionDeleteOperator</a><ul> |
| <li><a class="reference internal" href="#id30" id="id161">Arguments</a></li> |
| <li><a class="reference internal" href="#id31" id="id162">Using the operator</a></li> |
| <li><a class="reference internal" href="#id32" id="id163">Templating</a></li> |
| <li><a class="reference internal" href="#id33" id="id164">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#gcffunctiondeployoperator" id="id165">GcfFunctionDeployOperator</a><ul> |
| <li><a class="reference internal" href="#id34" id="id166">Arguments</a></li> |
| <li><a class="reference internal" href="#id35" id="id167">Using the operator</a></li> |
| <li><a class="reference internal" href="#id36" id="id168">Templating</a></li> |
| <li><a class="reference internal" href="#id37" id="id169">Troubleshooting</a></li> |
| <li><a class="reference internal" href="#id38" id="id170">More information</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#google-cloud-spanner-operators" id="id171">Google Cloud Spanner Operators</a><ul> |
| <li><a class="reference internal" href="#cloudspannerinstancedatabasedeleteoperator" id="id172">CloudSpannerInstanceDatabaseDeleteOperator</a><ul> |
| <li><a class="reference internal" href="#id40" id="id173">Arguments</a></li> |
| <li><a class="reference internal" href="#id41" id="id174">Using the operator</a></li> |
| <li><a class="reference internal" href="#id42" id="id175">Templating</a></li> |
| <li><a class="reference internal" href="#id43" id="id176">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudspannerinstancedatabasedeployoperator" id="id177">CloudSpannerInstanceDatabaseDeployOperator</a><ul> |
| <li><a class="reference internal" href="#id44" id="id178">Arguments</a></li> |
| <li><a class="reference internal" href="#id45" id="id179">Using the operator</a></li> |
| <li><a class="reference internal" href="#id46" id="id180">Templating</a></li> |
| <li><a class="reference internal" href="#id47" id="id181">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudspannerinstancedatabaseupdateoperator" id="id182">CloudSpannerInstanceDatabaseUpdateOperator</a><ul> |
| <li><a class="reference internal" href="#id48" id="id183">Arguments</a></li> |
| <li><a class="reference internal" href="#id49" id="id184">Using the operator</a></li> |
| <li><a class="reference internal" href="#id50" id="id185">Templating</a></li> |
| <li><a class="reference internal" href="#id51" id="id186">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudspannerinstancedatabasequeryoperator" id="id187">CloudSpannerInstanceDatabaseQueryOperator</a><ul> |
| <li><a class="reference internal" href="#id52" id="id188">Arguments</a></li> |
| <li><a class="reference internal" href="#id53" id="id189">Using the operator</a></li> |
| <li><a class="reference internal" href="#id54" id="id190">Templating</a></li> |
| <li><a class="reference internal" href="#id55" id="id191">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudspannerinstancedeleteoperator" id="id192">CloudSpannerInstanceDeleteOperator</a><ul> |
| <li><a class="reference internal" href="#id56" id="id193">Arguments</a></li> |
| <li><a class="reference internal" href="#id57" id="id194">Using the operator</a></li> |
| <li><a class="reference internal" href="#id58" id="id195">Templating</a></li> |
| <li><a class="reference internal" href="#id59" id="id196">More information</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#google-cloud-sql-operators" id="id197">Google Cloud Sql Operators</a><ul> |
| <li><a class="reference internal" href="#cloudsqlinstancedatabasecreateoperator" id="id198">CloudSqlInstanceDatabaseCreateOperator</a><ul> |
| <li><a class="reference internal" href="#id60" id="id199">Arguments</a></li> |
| <li><a class="reference internal" href="#id61" id="id200">Using the operator</a></li> |
| <li><a class="reference internal" href="#id62" id="id201">Templating</a></li> |
| <li><a class="reference internal" href="#id63" id="id202">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstancedatabasedeleteoperator" id="id203">CloudSqlInstanceDatabaseDeleteOperator</a><ul> |
| <li><a class="reference internal" href="#id64" id="id204">Arguments</a></li> |
| <li><a class="reference internal" href="#id65" id="id205">Using the operator</a></li> |
| <li><a class="reference internal" href="#id66" id="id206">Templating</a></li> |
| <li><a class="reference internal" href="#id67" id="id207">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstancedatabasepatchoperator" id="id208">CloudSqlInstanceDatabasePatchOperator</a><ul> |
| <li><a class="reference internal" href="#id68" id="id209">Arguments</a></li> |
| <li><a class="reference internal" href="#id69" id="id210">Using the operator</a></li> |
| <li><a class="reference internal" href="#id70" id="id211">Templating</a></li> |
| <li><a class="reference internal" href="#id71" id="id212">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstancedeleteoperator" id="id213">CloudSqlInstanceDeleteOperator</a><ul> |
| <li><a class="reference internal" href="#id72" id="id214">Arguments</a></li> |
| <li><a class="reference internal" href="#id73" id="id215">Using the operator</a></li> |
| <li><a class="reference internal" href="#id74" id="id216">Templating</a></li> |
| <li><a class="reference internal" href="#id75" id="id217">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstanceexportoperator" id="id218">CloudSqlInstanceExportOperator</a><ul> |
| <li><a class="reference internal" href="#id76" id="id219">Arguments</a></li> |
| <li><a class="reference internal" href="#id77" id="id220">Using the operator</a></li> |
| <li><a class="reference internal" href="#id78" id="id221">Templating</a></li> |
| <li><a class="reference internal" href="#id79" id="id222">More information</a></li> |
| <li><a class="reference internal" href="#id80" id="id223">Troubleshooting</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstanceimportoperator" id="id224">CloudSqlInstanceImportOperator</a><ul> |
| <li><a class="reference internal" href="#csv-import" id="id225">CSV import:</a></li> |
| <li><a class="reference internal" href="#sql-import" id="id226">SQL import:</a></li> |
| <li><a class="reference internal" href="#id81" id="id227">Arguments</a></li> |
| <li><a class="reference internal" href="#id82" id="id228">Using the operator</a></li> |
| <li><a class="reference internal" href="#id83" id="id229">Templating</a></li> |
| <li><a class="reference internal" href="#id84" id="id230">More information</a></li> |
| <li><a class="reference internal" href="#id85" id="id231">Troubleshooting</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstancecreateoperator" id="id232">CloudSqlInstanceCreateOperator</a><ul> |
| <li><a class="reference internal" href="#id87" id="id233">Arguments</a></li> |
| <li><a class="reference internal" href="#id88" id="id234">Using the operator</a></li> |
| <li><a class="reference internal" href="#id89" id="id235">Templating</a></li> |
| <li><a class="reference internal" href="#id90" id="id236">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlinstancepatchoperator" id="id237">CloudSqlInstancePatchOperator</a><ul> |
| <li><a class="reference internal" href="#id92" id="id238">Arguments</a></li> |
| <li><a class="reference internal" href="#id93" id="id239">Using the operator</a></li> |
| <li><a class="reference internal" href="#id94" id="id240">Templating</a></li> |
| <li><a class="reference internal" href="#id95" id="id241">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#cloudsqlqueryoperator" id="id242">CloudSqlQueryOperator</a><ul> |
| <li><a class="reference internal" href="#id96" id="id243">Arguments</a></li> |
| <li><a class="reference internal" href="#id97" id="id244">Using the operator</a></li> |
| <li><a class="reference internal" href="#id98" id="id245">Templating</a></li> |
| <li><a class="reference internal" href="#id99" id="id246">More information</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#id100" id="id247">Google Cloud Storage Operators</a><ul> |
| <li><a class="reference internal" href="#googlecloudstoragebucketcreateaclentryoperator" id="id248">GoogleCloudStorageBucketCreateAclEntryOperator</a><ul> |
| <li><a class="reference internal" href="#id101" id="id249">Arguments</a></li> |
| <li><a class="reference internal" href="#id102" id="id250">Using the operator</a></li> |
| <li><a class="reference internal" href="#id103" id="id251">Templating</a></li> |
| <li><a class="reference internal" href="#id104" id="id252">More information</a></li> |
| </ul> |
| </li> |
| <li><a class="reference internal" href="#googlecloudstorageobjectcreateaclentryoperator" id="id253">GoogleCloudStorageObjectCreateAclEntryOperator</a><ul> |
| <li><a class="reference internal" href="#id105" id="id254">Arguments</a></li> |
| <li><a class="reference internal" href="#id106" id="id255">Using the operator</a></li> |
| <li><a class="reference internal" href="#id107" id="id256">Templating</a></li> |
| <li><a class="reference internal" href="#id108" id="id257">More information</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </div> |
| <div class="section" id="bashoperator"> |
| <h2><a class="toc-backref" href="#id109">BashOperator</a><a class="headerlink" href="#bashoperator" title="Permalink to this headline">¶</a></h2> |
| <p>Use the <a class="reference internal" href="../code.html#airflow.operators.bash_operator.BashOperator" title="airflow.operators.bash_operator.BashOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">BashOperator</span></code></a> to execute |
| commands in a <a class="reference external" href="https://www.gnu.org/software/bash/">Bash</a> shell.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">run_this</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'run_after_loop'</span><span class="p">,</span> |
| <span class="n">bash_command</span><span class="o">=</span><span class="s1">'echo 1'</span><span class="p">,</span> |
| <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="section" id="templating"> |
| <h3><a class="toc-backref" href="#id110">Templating</a><a class="headerlink" href="#templating" title="Permalink to this headline">¶</a></h3> |
| <p>You can use <a class="reference internal" href="../concepts.html#jinja-templating"><span class="std std-ref">Jinja templates</span></a> to parameterize the |
| <code class="docutils literal notranslate"><span class="pre">bash_command</span></code> argument.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">also_run_this</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'also_run_this'</span><span class="p">,</span> |
| <span class="n">bash_command</span><span class="o">=</span><span class="s1">'echo "run_id={{ run_id }} | dag_run={{ dag_run }}"'</span><span class="p">,</span> |
| <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="troubleshooting"> |
| <h3><a class="toc-backref" href="#id111">Troubleshooting</a><a class="headerlink" href="#troubleshooting" title="Permalink to this headline">¶</a></h3> |
| <div class="section" id="jinja-template-not-found"> |
| <h4><a class="toc-backref" href="#id112">Jinja template not found</a><a class="headerlink" href="#jinja-template-not-found" title="Permalink to this headline">¶</a></h4> |
| <p>Add a space after the script name when directly calling a Bash script with |
| the <code class="docutils literal notranslate"><span class="pre">bash_command</span></code> argument. This is because Airflow tries to apply a Jinja |
| template to it, which will fail.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">t2</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'bash_example'</span><span class="p">,</span> |
| |
| <span class="c1"># This fails with `Jinja template not found` error</span> |
| <span class="c1"># bash_command="/home/batcher/test.sh",</span> |
| |
| <span class="c1"># This works (has a space after)</span> |
| <span class="n">bash_command</span><span class="o">=</span><span class="s2">"/home/batcher/test.sh "</span><span class="p">,</span> |
| <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="pythonoperator"> |
| <h2><a class="toc-backref" href="#id113">PythonOperator</a><a class="headerlink" href="#pythonoperator" title="Permalink to this headline">¶</a></h2> |
| <p>Use the <a class="reference internal" href="../code.html#airflow.operators.python_operator.PythonOperator" title="airflow.operators.python_operator.PythonOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">PythonOperator</span></code></a> to execute |
| Python callables.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">print_context</span><span class="p">(</span><span class="n">ds</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> |
| <span class="n">pprint</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span> |
| <span class="k">print</span><span class="p">(</span><span class="n">ds</span><span class="p">)</span> |
| <span class="k">return</span> <span class="s1">'Whatever you return gets printed in the logs'</span> |
| |
| |
| <span class="n">run_this</span> <span class="o">=</span> <span class="n">PythonOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'print_the_context'</span><span class="p">,</span> |
| <span class="n">provide_context</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> |
| <span class="n">python_callable</span><span class="o">=</span><span class="n">print_context</span><span class="p">,</span> |
| <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="section" id="passing-in-arguments"> |
| <h3><a class="toc-backref" href="#id114">Passing in arguments</a><a class="headerlink" href="#passing-in-arguments" title="Permalink to this headline">¶</a></h3> |
| <p>Use the <code class="docutils literal notranslate"><span class="pre">op_args</span></code> and <code class="docutils literal notranslate"><span class="pre">op_kwargs</span></code> arguments to pass additional arguments |
| to the Python callable.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">my_sleeping_function</span><span class="p">(</span><span class="n">random_base</span><span class="p">):</span> |
| <span class="sd">"""This is a function that will run within the DAG execution"""</span> |
| <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">random_base</span><span class="p">)</span> |
| |
| |
| <span class="c1"># Generate 5 sleeping tasks, sleeping from 0.0 to 0.4 seconds respectively</span> |
| <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">5</span><span class="p">):</span> |
| <span class="n">task</span> <span class="o">=</span> <span class="n">PythonOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sleep_for_'</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span> |
| <span class="n">python_callable</span><span class="o">=</span><span class="n">my_sleeping_function</span><span class="p">,</span> |
| <span class="n">op_kwargs</span><span class="o">=</span><span class="p">{</span><span class="s1">'random_base'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="o">/</span> <span class="mi">10</span><span class="p">},</span> |
| <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> |
| <span class="p">)</span> |
| |
| <span class="n">run_this</span> <span class="o">>></span> <span class="n">task</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id1"> |
| <h3><a class="toc-backref" href="#id115">Templating</a><a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h3> |
| <p>When you set the <code class="docutils literal notranslate"><span class="pre">provide_context</span></code> argument to <code class="docutils literal notranslate"><span class="pre">True</span></code>, Airflow passes in |
| an additional set of keyword arguments: one for each of the <a class="reference internal" href="../code.html#macros"><span class="std std-ref">Jinja |
| template variables</span></a> and a <code class="docutils literal notranslate"><span class="pre">templates_dict</span></code> argument.</p> |
| <p>The <code class="docutils literal notranslate"><span class="pre">templates_dict</span></code> argument is templated, so each value in the dictionary |
| is evaluated as a <a class="reference internal" href="../concepts.html#jinja-templating"><span class="std std-ref">Jinja template</span></a>.</p> |
| </div> |
| </div> |
| <div class="section" id="google-cloud-storage-operators"> |
| <h2><a class="toc-backref" href="#id116">Google Cloud Storage Operators</a><a class="headerlink" href="#google-cloud-storage-operators" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="googlecloudstoragetobigqueryoperator"> |
| <h3><a class="toc-backref" href="#id117">GoogleCloudStorageToBigQueryOperator</a><a class="headerlink" href="#googlecloudstoragetobigqueryoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator" title="airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GoogleCloudStorageToBigQueryOperator</span></code></a> |
| to execute a BigQuery load job.</p> |
| </div> |
| </div> |
| <div class="section" id="google-compute-engine-operators"> |
| <h2><a class="toc-backref" href="#id118">Google Compute Engine Operators</a><a class="headerlink" href="#google-compute-engine-operators" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="gceinstancestartoperator"> |
| <h3><a class="toc-backref" href="#id119">GceInstanceStartOperator</a><a class="headerlink" href="#gceinstancestartoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator" title="airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GceInstanceStartOperator</span></code></a> |
| to start an existing Google Compute Engine instance.</p> |
| <div class="section" id="arguments"> |
| <h4><a class="toc-backref" href="#id120">Arguments</a><a class="headerlink" href="#arguments" title="Permalink to this headline">¶</a></h4> |
| <p>The following examples of OS environment variables used to pass arguments to the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCE_ZONE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_ZONE'</span><span class="p">,</span> <span class="s1">'europe-west1-b'</span><span class="p">)</span> |
| <span class="n">GCE_INSTANCE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_INSTANCE'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="using-the-operator"> |
| <h4><a class="toc-backref" href="#id121">Using the operator</a><a class="headerlink" href="#using-the-operator" title="Permalink to this headline">¶</a></h4> |
| <p>The code to create the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_instance_start</span> <span class="o">=</span> <span class="n">GceInstanceStartOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">zone</span><span class="o">=</span><span class="n">GCE_ZONE</span><span class="p">,</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">GCE_INSTANCE</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_start_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>You can also create the operator without project id - project id will be retrieved |
| from the GCP connection id used:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_instance_start2</span> <span class="o">=</span> <span class="n">GceInstanceStartOperator</span><span class="p">(</span> |
| <span class="n">zone</span><span class="o">=</span><span class="n">GCE_ZONE</span><span class="p">,</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">GCE_INSTANCE</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_start_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id2"> |
| <h4><a class="toc-backref" href="#id122">Templating</a><a class="headerlink" href="#id2" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'zone'</span><span class="p">,</span> <span class="s1">'resource_id'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="more-information"> |
| <h4><a class="toc-backref" href="#id123">More information</a><a class="headerlink" href="#more-information" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/compute/docs/reference/rest/v1/instances/start">Google Compute Engine API documentation</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="gceinstancestopoperator"> |
| <h3><a class="toc-backref" href="#id124">GceInstanceStopOperator</a><a class="headerlink" href="#gceinstancestopoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the operator to stop Google Compute Engine instance.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_compute_operator.GceInstanceStopOperator" title="airflow.contrib.operators.gcp_compute_operator.GceInstanceStopOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GceInstanceStopOperator</span></code></a></p> |
| <div class="section" id="id3"> |
| <h4><a class="toc-backref" href="#id125">Arguments</a><a class="headerlink" href="#id3" title="Permalink to this headline">¶</a></h4> |
| <p>The following examples of OS environment variables used to pass arguments to the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCE_ZONE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_ZONE'</span><span class="p">,</span> <span class="s1">'europe-west1-b'</span><span class="p">)</span> |
| <span class="n">GCE_INSTANCE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_INSTANCE'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id4"> |
| <h4><a class="toc-backref" href="#id126">Using the operator</a><a class="headerlink" href="#id4" title="Permalink to this headline">¶</a></h4> |
| <p>The code to create the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_instance_stop</span> <span class="o">=</span> <span class="n">GceInstanceStopOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">zone</span><span class="o">=</span><span class="n">GCE_ZONE</span><span class="p">,</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">GCE_INSTANCE</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_stop_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>You can also create the operator without project id - project id will be retrieved |
| from the GCP connection used:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_instance_stop2</span> <span class="o">=</span> <span class="n">GceInstanceStopOperator</span><span class="p">(</span> |
| <span class="n">zone</span><span class="o">=</span><span class="n">GCE_ZONE</span><span class="p">,</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">GCE_INSTANCE</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_stop_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id5"> |
| <h4><a class="toc-backref" href="#id127">Templating</a><a class="headerlink" href="#id5" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'zone'</span><span class="p">,</span> <span class="s1">'resource_id'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id6"> |
| <h4><a class="toc-backref" href="#id128">More information</a><a class="headerlink" href="#id6" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/compute/docs/reference/rest/v1/instances/stop">Google Compute Engine API documentation</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="gcesetmachinetypeoperator"> |
| <h3><a class="toc-backref" href="#id129">GceSetMachineTypeOperator</a><a class="headerlink" href="#gcesetmachinetypeoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the operator to change machine type of a Google Compute Engine instance.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_compute_operator.GceSetMachineTypeOperator" title="airflow.contrib.operators.gcp_compute_operator.GceSetMachineTypeOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GceSetMachineTypeOperator</span></code></a>.</p> |
| <div class="section" id="id8"> |
| <h4><a class="toc-backref" href="#id130">Arguments</a><a class="headerlink" href="#id8" title="Permalink to this headline">¶</a></h4> |
| <p>The following examples of OS environment variables used to pass arguments to the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCE_ZONE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_ZONE'</span><span class="p">,</span> <span class="s1">'europe-west1-b'</span><span class="p">)</span> |
| <span class="n">GCE_INSTANCE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_INSTANCE'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCE_SHORT_MACHINE_TYPE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_SHORT_MACHINE_TYPE_NAME'</span><span class="p">,</span> <span class="s1">'n1-standard-1'</span><span class="p">)</span> |
| <span class="n">SET_MACHINE_TYPE_BODY</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s1">'machineType'</span><span class="p">:</span> <span class="s1">'zones/{}/machineTypes/{}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">GCE_ZONE</span><span class="p">,</span> <span class="n">GCE_SHORT_MACHINE_TYPE_NAME</span><span class="p">)</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id9"> |
| <h4><a class="toc-backref" href="#id131">Using the operator</a><a class="headerlink" href="#id9" title="Permalink to this headline">¶</a></h4> |
| <p>The code to create the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_set_machine_type</span> <span class="o">=</span> <span class="n">GceSetMachineTypeOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">zone</span><span class="o">=</span><span class="n">GCE_ZONE</span><span class="p">,</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">GCE_INSTANCE</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">SET_MACHINE_TYPE_BODY</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_set_machine_type'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>You can also create the operator without project id - project id will be retrieved |
| from the GCP connection used:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_set_machine_type2</span> <span class="o">=</span> <span class="n">GceSetMachineTypeOperator</span><span class="p">(</span> |
| <span class="n">zone</span><span class="o">=</span><span class="n">GCE_ZONE</span><span class="p">,</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">GCE_INSTANCE</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">SET_MACHINE_TYPE_BODY</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_set_machine_type2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id10"> |
| <h4><a class="toc-backref" href="#id132">Templating</a><a class="headerlink" href="#id10" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'zone'</span><span class="p">,</span> <span class="s1">'resource_id'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id11"> |
| <h4><a class="toc-backref" href="#id133">More information</a><a class="headerlink" href="#id11" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/compute/docs/reference/rest/v1/instances/setMachineType">Google Compute Engine API documentation</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="gceinstancetemplatecopyoperator"> |
| <h3><a class="toc-backref" href="#id134">GceInstanceTemplateCopyOperator</a><a class="headerlink" href="#gceinstancetemplatecopyoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the operator to copy an existing Google Compute Engine instance template |
| applying a patch to it.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_compute_operator.GceInstanceTemplateCopyOperator" title="airflow.contrib.operators.gcp_compute_operator.GceInstanceTemplateCopyOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GceInstanceTemplateCopyOperator</span></code></a>.</p> |
| <div class="section" id="id13"> |
| <h4><a class="toc-backref" href="#id135">Arguments</a><a class="headerlink" href="#id13" title="Permalink to this headline">¶</a></h4> |
| <p>The following examples of OS environment variables used to pass arguments to the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCE_ZONE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_ZONE'</span><span class="p">,</span> <span class="s1">'europe-west1-b'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCE_TEMPLATE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_TEMPLATE_NAME'</span><span class="p">,</span> <span class="s1">'instance-template-test'</span><span class="p">)</span> |
| <span class="n">GCE_NEW_TEMPLATE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_NEW_TEMPLATE_NAME'</span><span class="p">,</span> |
| <span class="s1">'instance-template-test-new'</span><span class="p">)</span> |
| <span class="n">GCE_NEW_DESCRIPTION</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_NEW_DESCRIPTION'</span><span class="p">,</span> <span class="s1">'Test new description'</span><span class="p">)</span> |
| <span class="n">GCE_INSTANCE_TEMPLATE_BODY_UPDATE</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"name"</span><span class="p">:</span> <span class="n">GCE_NEW_TEMPLATE_NAME</span><span class="p">,</span> |
| <span class="s2">"description"</span><span class="p">:</span> <span class="n">GCE_NEW_DESCRIPTION</span><span class="p">,</span> |
| <span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"machineType"</span><span class="p">:</span> <span class="s2">"n1-standard-2"</span> |
| <span class="p">}</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id14"> |
| <h4><a class="toc-backref" href="#id136">Using the operator</a><a class="headerlink" href="#id14" title="Permalink to this headline">¶</a></h4> |
| <p>The code to create the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_instance_template_copy</span> <span class="o">=</span> <span class="n">GceInstanceTemplateCopyOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">GCE_TEMPLATE_NAME</span><span class="p">,</span> |
| <span class="n">body_patch</span><span class="o">=</span><span class="n">GCE_INSTANCE_TEMPLATE_BODY_UPDATE</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_igm_copy_template_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>You can also create the operator without project id - project id will be retrieved |
| from the GCP connection used:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_instance_template_copy2</span> <span class="o">=</span> <span class="n">GceInstanceTemplateCopyOperator</span><span class="p">(</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">GCE_TEMPLATE_NAME</span><span class="p">,</span> |
| <span class="n">body_patch</span><span class="o">=</span><span class="n">GCE_INSTANCE_TEMPLATE_BODY_UPDATE</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_igm_copy_template_task_2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id15"> |
| <h4><a class="toc-backref" href="#id137">Templating</a><a class="headerlink" href="#id15" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'resource_id'</span><span class="p">,</span> <span class="s1">'request_id'</span><span class="p">,</span> |
| <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id16"> |
| <h4><a class="toc-backref" href="#id138">More information</a><a class="headerlink" href="#id16" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/compute/docs/reference/rest/v1/instanceTemplates">Google Compute Engine API documentation</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="gceinstancegroupmanagerupdatetemplateoperator"> |
| <h3><a class="toc-backref" href="#id139">GceInstanceGroupManagerUpdateTemplateOperator</a><a class="headerlink" href="#gceinstancegroupmanagerupdatetemplateoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the operator to update template in Google Compute Engine Instance Group Manager.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_compute_operator.GceInstanceGroupManagerUpdateTemplateOperator" title="airflow.contrib.operators.gcp_compute_operator.GceInstanceGroupManagerUpdateTemplateOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GceInstanceGroupManagerUpdateTemplateOperator</span></code></a>.</p> |
| <div class="section" id="id18"> |
| <h4><a class="toc-backref" href="#id140">Arguments</a><a class="headerlink" href="#id18" title="Permalink to this headline">¶</a></h4> |
| <p>The following examples of OS environment variables used to pass arguments to the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCE_ZONE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_ZONE'</span><span class="p">,</span> <span class="s1">'europe-west1-b'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCE_INSTANCE_GROUP_MANAGER_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCE_INSTANCE_GROUP_MANAGER_NAME'</span><span class="p">,</span> |
| <span class="s1">'instance-group-test'</span><span class="p">)</span> |
| |
| <span class="n">SOURCE_TEMPLATE_URL</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span> |
| <span class="s1">'SOURCE_TEMPLATE_URL'</span><span class="p">,</span> |
| <span class="s2">"https://www.googleapis.com/compute/beta/projects/"</span> <span class="o">+</span> <span class="n">GCP_PROJECT_ID</span> <span class="o">+</span> |
| <span class="s2">"/global/instanceTemplates/instance-template-test"</span><span class="p">)</span> |
| |
| <span class="n">DESTINATION_TEMPLATE_URL</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span> |
| <span class="s1">'DESTINATION_TEMPLATE_URL'</span><span class="p">,</span> |
| <span class="s2">"https://www.googleapis.com/compute/beta/projects/"</span> <span class="o">+</span> <span class="n">GCP_PROJECT_ID</span> <span class="o">+</span> |
| <span class="s2">"/global/instanceTemplates/"</span> <span class="o">+</span> <span class="n">GCE_NEW_TEMPLATE_NAME</span><span class="p">)</span> |
| |
| <span class="n">UPDATE_POLICY</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"OPPORTUNISTIC"</span><span class="p">,</span> |
| <span class="s2">"minimalAction"</span><span class="p">:</span> <span class="s2">"RESTART"</span><span class="p">,</span> |
| <span class="s2">"maxSurge"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"fixed"</span><span class="p">:</span> <span class="mi">1</span> |
| <span class="p">},</span> |
| <span class="s2">"minReadySec"</span><span class="p">:</span> <span class="mi">1800</span> |
| <span class="p">}</span> |
| |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id19"> |
| <h4><a class="toc-backref" href="#id141">Using the operator</a><a class="headerlink" href="#id19" title="Permalink to this headline">¶</a></h4> |
| <p>The code to create the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_instance_group_manager_update_template</span> <span class="o">=</span> \ |
| <span class="n">GceInstanceGroupManagerUpdateTemplateOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">GCE_INSTANCE_GROUP_MANAGER_NAME</span><span class="p">,</span> |
| <span class="n">zone</span><span class="o">=</span><span class="n">GCE_ZONE</span><span class="p">,</span> |
| <span class="n">source_template</span><span class="o">=</span><span class="n">SOURCE_TEMPLATE_URL</span><span class="p">,</span> |
| <span class="n">destination_template</span><span class="o">=</span><span class="n">DESTINATION_TEMPLATE_URL</span><span class="p">,</span> |
| <span class="n">update_policy</span><span class="o">=</span><span class="n">UPDATE_POLICY</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_igm_group_manager_update_template'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>You can also create the operator without project id - project id will be retrieved |
| from the GCP connection used:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_instance_group_manager_update_template2</span> <span class="o">=</span> \ |
| <span class="n">GceInstanceGroupManagerUpdateTemplateOperator</span><span class="p">(</span> |
| <span class="n">resource_id</span><span class="o">=</span><span class="n">GCE_INSTANCE_GROUP_MANAGER_NAME</span><span class="p">,</span> |
| <span class="n">zone</span><span class="o">=</span><span class="n">GCE_ZONE</span><span class="p">,</span> |
| <span class="n">source_template</span><span class="o">=</span><span class="n">SOURCE_TEMPLATE_URL</span><span class="p">,</span> |
| <span class="n">destination_template</span><span class="o">=</span><span class="n">DESTINATION_TEMPLATE_URL</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_igm_group_manager_update_template_2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id20"> |
| <h4><a class="toc-backref" href="#id142">Templating</a><a class="headerlink" href="#id20" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'resource_id'</span><span class="p">,</span> <span class="s1">'zone'</span><span class="p">,</span> <span class="s1">'request_id'</span><span class="p">,</span> |
| <span class="s1">'source_template'</span><span class="p">,</span> <span class="s1">'destination_template'</span><span class="p">,</span> |
| <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id21"> |
| <h4><a class="toc-backref" href="#id143">Troubleshooting</a><a class="headerlink" href="#id21" title="Permalink to this headline">¶</a></h4> |
| <p>You might find that your GceInstanceGroupManagerUpdateTemplateOperator fails with |
| missing permissions. To execute the operation, the service account requires |
| the permissions that theService Account User role provides |
| (assigned via Google Cloud IAM).</p> |
| </div> |
| <div class="section" id="id22"> |
| <h4><a class="toc-backref" href="#id144">More information</a><a class="headerlink" href="#id22" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers">Google Compute Engine API documentation</a>.</p> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="google-cloud-bigtable-operators"> |
| <h2><a class="toc-backref" href="#id145">Google Cloud Bigtable Operators</a><a class="headerlink" href="#google-cloud-bigtable-operators" title="Permalink to this headline">¶</a></h2> |
| <p>All examples below rely on the following variables, which can be passed via environment variables.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">CBT_INSTANCE_ID</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">'CBT_INSTANCE_ID'</span><span class="p">,</span> <span class="s1">'some-instance-id'</span><span class="p">)</span> |
| <span class="n">CBT_INSTANCE_DISPLAY_NAME</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">'CBT_INSTANCE_DISPLAY_NAME'</span><span class="p">,</span> <span class="s1">'Human-readable name'</span><span class="p">)</span> |
| <span class="n">CBT_INSTANCE_TYPE</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">'CBT_INSTANCE_TYPE'</span><span class="p">,</span> <span class="s1">'2'</span><span class="p">)</span> |
| <span class="n">CBT_INSTANCE_LABELS</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">'CBT_INSTANCE_LABELS'</span><span class="p">,</span> <span class="s1">'{}'</span><span class="p">)</span> |
| <span class="n">CBT_CLUSTER_ID</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">'CBT_CLUSTER_ID'</span><span class="p">,</span> <span class="s1">'some-cluster-id'</span><span class="p">)</span> |
| <span class="n">CBT_CLUSTER_ZONE</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">'CBT_CLUSTER_ZONE'</span><span class="p">,</span> <span class="s1">'europe-west1-b'</span><span class="p">)</span> |
| <span class="n">CBT_CLUSTER_NODES</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">'CBT_CLUSTER_NODES'</span><span class="p">,</span> <span class="s1">'3'</span><span class="p">)</span> |
| <span class="n">CBT_CLUSTER_NODES_UPDATED</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">'CBT_CLUSTER_NODES_UPDATED'</span><span class="p">,</span> <span class="s1">'5'</span><span class="p">)</span> |
| <span class="n">CBT_CLUSTER_STORAGE_TYPE</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">'CBT_CLUSTER_STORAGE_TYPE'</span><span class="p">,</span> <span class="s1">'2'</span><span class="p">)</span> |
| <span class="n">CBT_TABLE_ID</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">'CBT_TABLE_ID'</span><span class="p">,</span> <span class="s1">'some-table-id'</span><span class="p">)</span> |
| <span class="n">CBT_POKE_INTERVAL</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">'CBT_POKE_INTERVAL'</span><span class="p">,</span> <span class="s1">'60'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="section" id="bigtableinstancecreateoperator"> |
| <h3><a class="toc-backref" href="#id146">BigtableInstanceCreateOperator</a><a class="headerlink" href="#bigtableinstancecreateoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the <code class="xref py py-class docutils literal notranslate"><span class="pre">BigtableInstanceCreateOperator</span></code> |
| to create a Google Cloud Bigtable instance.</p> |
| <p>If the Cloud Bigtable instance with the given ID exists, the operator does not compare its configuration |
| and immediately succeeds. No changes are made to the existing instance.</p> |
| <div class="section" id="id24"> |
| <h4><a class="toc-backref" href="#id147">Using the operator</a><a class="headerlink" href="#id24" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">create_instance_task</span> <span class="o">=</span> <span class="n">BigtableInstanceCreateOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">main_cluster_id</span><span class="o">=</span><span class="n">CBT_CLUSTER_ID</span><span class="p">,</span> |
| <span class="n">main_cluster_zone</span><span class="o">=</span><span class="n">CBT_CLUSTER_ZONE</span><span class="p">,</span> |
| <span class="n">instance_display_name</span><span class="o">=</span><span class="n">CBT_INSTANCE_DISPLAY_NAME</span><span class="p">,</span> |
| <span class="n">instance_type</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_INSTANCE_TYPE</span><span class="p">),</span> |
| <span class="n">instance_labels</span><span class="o">=</span><span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">CBT_INSTANCE_LABELS</span><span class="p">),</span> |
| <span class="n">cluster_nodes</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_CLUSTER_NODES</span><span class="p">),</span> |
| <span class="n">cluster_storage_type</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_CLUSTER_STORAGE_TYPE</span><span class="p">),</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'create_instance_task'</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">create_instance_task2</span> <span class="o">=</span> <span class="n">BigtableInstanceCreateOperator</span><span class="p">(</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">main_cluster_id</span><span class="o">=</span><span class="n">CBT_CLUSTER_ID</span><span class="p">,</span> |
| <span class="n">main_cluster_zone</span><span class="o">=</span><span class="n">CBT_CLUSTER_ZONE</span><span class="p">,</span> |
| <span class="n">instance_display_name</span><span class="o">=</span><span class="n">CBT_INSTANCE_DISPLAY_NAME</span><span class="p">,</span> |
| <span class="n">instance_type</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_INSTANCE_TYPE</span><span class="p">),</span> |
| <span class="n">instance_labels</span><span class="o">=</span><span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">CBT_INSTANCE_LABELS</span><span class="p">),</span> |
| <span class="n">cluster_nodes</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_CLUSTER_NODES</span><span class="p">),</span> |
| <span class="n">cluster_storage_type</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_CLUSTER_STORAGE_TYPE</span><span class="p">),</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'create_instance_task2'</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">create_instance_task</span> <span class="o">>></span> <span class="n">create_instance_task2</span> |
| </pre></div> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="bigtableinstancedeleteoperator"> |
| <h3><a class="toc-backref" href="#id148">BigtableInstanceDeleteOperator</a><a class="headerlink" href="#bigtableinstancedeleteoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the <code class="xref py py-class docutils literal notranslate"><span class="pre">BigtableInstanceDeleteOperator</span></code> |
| to delete a Google Cloud Bigtable instance.</p> |
| <div class="section" id="id25"> |
| <h4><a class="toc-backref" href="#id149">Using the operator</a><a class="headerlink" href="#id25" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">delete_instance_task</span> <span class="o">=</span> <span class="n">BigtableInstanceDeleteOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'delete_instance_task'</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">delete_instance_task2</span> <span class="o">=</span> <span class="n">BigtableInstanceDeleteOperator</span><span class="p">(</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'delete_instance_task2'</span><span class="p">,</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="bigtableclusterupdateoperator"> |
| <h3><a class="toc-backref" href="#id150">BigtableClusterUpdateOperator</a><a class="headerlink" href="#bigtableclusterupdateoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the <code class="xref py py-class docutils literal notranslate"><span class="pre">BigtableClusterUpdateOperator</span></code> |
| to modify number of nodes in a Cloud Bigtable cluster.</p> |
| <div class="section" id="id26"> |
| <h4><a class="toc-backref" href="#id151">Using the operator</a><a class="headerlink" href="#id26" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">cluster_update_task</span> <span class="o">=</span> <span class="n">BigtableClusterUpdateOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">cluster_id</span><span class="o">=</span><span class="n">CBT_CLUSTER_ID</span><span class="p">,</span> |
| <span class="n">nodes</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_CLUSTER_NODES_UPDATED</span><span class="p">),</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'update_cluster_task'</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">cluster_update_task2</span> <span class="o">=</span> <span class="n">BigtableClusterUpdateOperator</span><span class="p">(</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">cluster_id</span><span class="o">=</span><span class="n">CBT_CLUSTER_ID</span><span class="p">,</span> |
| <span class="n">nodes</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_CLUSTER_NODES_UPDATED</span><span class="p">),</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'update_cluster_task2'</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">cluster_update_task</span> <span class="o">>></span> <span class="n">cluster_update_task2</span> |
| </pre></div> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="bigtabletablecreateoperator"> |
| <h3><a class="toc-backref" href="#id152">BigtableTableCreateOperator</a><a class="headerlink" href="#bigtabletablecreateoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Creates a table in a Cloud Bigtable instance.</p> |
| <p>If the table with given ID exists in the Cloud Bigtable instance, the operator compares the Column Families. |
| If the Column Families are identical operator succeeds. Otherwise, the operator fails with the appropriate |
| error message.</p> |
| <div class="section" id="id27"> |
| <h4><a class="toc-backref" href="#id153">Using the operator</a><a class="headerlink" href="#id27" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">create_table_task</span> <span class="o">=</span> <span class="n">BigtableTableCreateOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">table_id</span><span class="o">=</span><span class="n">CBT_TABLE_ID</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'create_table'</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">create_table_task2</span> <span class="o">=</span> <span class="n">BigtableTableCreateOperator</span><span class="p">(</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">table_id</span><span class="o">=</span><span class="n">CBT_TABLE_ID</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'create_table_task2'</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">create_table_task</span> <span class="o">>></span> <span class="n">create_table_task2</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="advanced"> |
| <h4><a class="toc-backref" href="#id154">Advanced</a><a class="headerlink" href="#advanced" title="Permalink to this headline">¶</a></h4> |
| <p>When creating a table, you can specify the optional <code class="docutils literal notranslate"><span class="pre">initial_split_keys</span></code> and <code class="docutils literal notranslate"><span class="pre">column_familes</span></code>. |
| Please refer to the Python Client for Google Cloud Bigtable documentation |
| <a class="reference external" href="https://googleapis.github.io/google-cloud-python/latest/bigtable/table.html">for Table</a> and <a class="reference external" href="https://googleapis.github.io/google-cloud-python/latest/bigtable/column-family.html">for Column |
| Families</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="bigtabletabledeleteoperator"> |
| <h3><a class="toc-backref" href="#id155">BigtableTableDeleteOperator</a><a class="headerlink" href="#bigtabletabledeleteoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the <code class="xref py py-class docutils literal notranslate"><span class="pre">BigtableTableDeleteOperator</span></code> |
| to delete a table in Google Cloud Bigtable.</p> |
| <div class="section" id="id28"> |
| <h4><a class="toc-backref" href="#id156">Using the operator</a><a class="headerlink" href="#id28" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">delete_table_task</span> <span class="o">=</span> <span class="n">BigtableTableDeleteOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">table_id</span><span class="o">=</span><span class="n">CBT_TABLE_ID</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'delete_table_task'</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">delete_table_task2</span> <span class="o">=</span> <span class="n">BigtableTableDeleteOperator</span><span class="p">(</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">table_id</span><span class="o">=</span><span class="n">CBT_TABLE_ID</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'delete_table_task2'</span><span class="p">,</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="bigtabletablewaitforreplicationsensor"> |
| <h3><a class="toc-backref" href="#id157">BigtableTableWaitForReplicationSensor</a><a class="headerlink" href="#bigtabletablewaitforreplicationsensor" title="Permalink to this headline">¶</a></h3> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <p>Use the <code class="xref py py-class docutils literal notranslate"><span class="pre">BigtableTableWaitForReplicationSensor</span></code> |
| to wait for the table to replicate fully.</p> |
| <p>The same arguments apply to this sensor as the <a class="reference internal" href="#bigtabletablecreateoperator">BigtableTableCreateOperator</a>.</p> |
| <p><strong>Note:</strong> If the table or the Cloud Bigtable instance does not exist, this sensor waits for the table until |
| timeout hits and does not raise any exception.</p> |
| <div class="section" id="id29"> |
| <h4><a class="toc-backref" href="#id158">Using the operator</a><a class="headerlink" href="#id29" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">wait_for_table_replication_task</span> <span class="o">=</span> <span class="n">BigtableTableWaitForReplicationSensor</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">table_id</span><span class="o">=</span><span class="n">CBT_TABLE_ID</span><span class="p">,</span> |
| <span class="n">poke_interval</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_POKE_INTERVAL</span><span class="p">),</span> |
| <span class="n">timeout</span><span class="o">=</span><span class="mi">180</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'wait_for_table_replication_task'</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">wait_for_table_replication_task2</span> <span class="o">=</span> <span class="n">BigtableTableWaitForReplicationSensor</span><span class="p">(</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">table_id</span><span class="o">=</span><span class="n">CBT_TABLE_ID</span><span class="p">,</span> |
| <span class="n">poke_interval</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_POKE_INTERVAL</span><span class="p">),</span> |
| <span class="n">timeout</span><span class="o">=</span><span class="mi">180</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'wait_for_table_replication_task2'</span><span class="p">,</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="google-cloud-functions-operators"> |
| <h2><a class="toc-backref" href="#id159">Google Cloud Functions Operators</a><a class="headerlink" href="#google-cloud-functions-operators" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="gcffunctiondeleteoperator"> |
| <h3><a class="toc-backref" href="#id160">GcfFunctionDeleteOperator</a><a class="headerlink" href="#gcffunctiondeleteoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the operator to delete a function from Google Cloud Functions.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_function_operator.GcfFunctionDeleteOperator" title="airflow.contrib.operators.gcp_function_operator.GcfFunctionDeleteOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GcfFunctionDeleteOperator</span></code></a>.</p> |
| <div class="section" id="id30"> |
| <h4><a class="toc-backref" href="#id161">Arguments</a><a class="headerlink" href="#id30" title="Permalink to this headline">¶</a></h4> |
| <p>The following examples of OS environment variables show how you can build function name |
| to use in the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCP_LOCATION</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_LOCATION'</span><span class="p">,</span> <span class="s1">'europe-west1'</span><span class="p">)</span> |
| <span class="n">GCF_SHORT_FUNCTION_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCF_SHORT_FUNCTION_NAME'</span><span class="p">,</span> <span class="s1">'hello'</span><span class="p">)</span><span class="o">.</span>\ |
| <span class="n">replace</span><span class="p">(</span><span class="s2">"-"</span><span class="p">,</span> <span class="s2">"_"</span><span class="p">)</span> <span class="c1"># make sure there are no dashes in function name (!)</span> |
| <span class="n">FUNCTION_NAME</span> <span class="o">=</span> <span class="s1">'projects/{}/locations/{}/functions/{}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">GCP_LOCATION</span><span class="p">,</span> |
| <span class="n">GCF_SHORT_FUNCTION_NAME</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id31"> |
| <h4><a class="toc-backref" href="#id162">Using the operator</a><a class="headerlink" href="#id31" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">delete_task</span> <span class="o">=</span> <span class="n">GcfFunctionDeleteOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s2">"gcf_delete_task"</span><span class="p">,</span> |
| <span class="n">name</span><span class="o">=</span><span class="n">FUNCTION_NAME</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id32"> |
| <h4><a class="toc-backref" href="#id163">Templating</a><a class="headerlink" href="#id32" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'name'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id33"> |
| <h4><a class="toc-backref" href="#id164">More information</a><a class="headerlink" href="#id33" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions/delete">Google Cloud Functions API documentation</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="gcffunctiondeployoperator"> |
| <h3><a class="toc-backref" href="#id165">GcfFunctionDeployOperator</a><a class="headerlink" href="#gcffunctiondeployoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Use the operator to deploy a function to Google Cloud Functions. |
| If a function with this name already exists, it will be updated.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_function_operator.GcfFunctionDeployOperator" title="airflow.contrib.operators.gcp_function_operator.GcfFunctionDeployOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GcfFunctionDeployOperator</span></code></a>.</p> |
| <div class="section" id="id34"> |
| <h4><a class="toc-backref" href="#id166">Arguments</a><a class="headerlink" href="#id34" title="Permalink to this headline">¶</a></h4> |
| <p>In the example DAG the following environment variables are used to parameterize the |
| operator’s definition:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCP_LOCATION</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_LOCATION'</span><span class="p">,</span> <span class="s1">'europe-west1'</span><span class="p">)</span> |
| <span class="n">GCF_SHORT_FUNCTION_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCF_SHORT_FUNCTION_NAME'</span><span class="p">,</span> <span class="s1">'hello'</span><span class="p">)</span><span class="o">.</span>\ |
| <span class="n">replace</span><span class="p">(</span><span class="s2">"-"</span><span class="p">,</span> <span class="s2">"_"</span><span class="p">)</span> <span class="c1"># make sure there are no dashes in function name (!)</span> |
| <span class="n">FUNCTION_NAME</span> <span class="o">=</span> <span class="s1">'projects/{}/locations/{}/functions/{}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">GCP_LOCATION</span><span class="p">,</span> |
| <span class="n">GCF_SHORT_FUNCTION_NAME</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCF_SOURCE_ARCHIVE_URL</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCF_SOURCE_ARCHIVE_URL'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">GCF_SOURCE_UPLOAD_URL</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCF_SOURCE_UPLOAD_URL'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">GCF_SOURCE_REPOSITORY</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span> |
| <span class="s1">'GCF_SOURCE_REPOSITORY'</span><span class="p">,</span> |
| <span class="s1">'https://source.developers.google.com/'</span> |
| <span class="s1">'projects/{}/repos/hello-world/moveable-aliases/master'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">GCP_PROJECT_ID</span><span class="p">))</span> |
| <span class="n">GCF_ZIP_PATH</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCF_ZIP_PATH'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> |
| <span class="n">GCF_ENTRYPOINT</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCF_ENTRYPOINT'</span><span class="p">,</span> <span class="s1">'helloWorld'</span><span class="p">)</span> |
| <span class="n">GCF_RUNTIME</span> <span class="o">=</span> <span class="s1">'nodejs6'</span> |
| <span class="n">GCP_VALIDATE_BODY</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_VALIDATE_BODY'</span><span class="p">,</span> <span class="bp">True</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Some of those variables are used to create the request’s body:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">body</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"name"</span><span class="p">:</span> <span class="n">FUNCTION_NAME</span><span class="p">,</span> |
| <span class="s2">"entryPoint"</span><span class="p">:</span> <span class="n">GCF_ENTRYPOINT</span><span class="p">,</span> |
| <span class="s2">"runtime"</span><span class="p">:</span> <span class="n">GCF_RUNTIME</span><span class="p">,</span> |
| <span class="s2">"httpsTrigger"</span><span class="p">:</span> <span class="p">{}</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| <p>When a DAG is created, the default_args dictionary can be used to pass |
| arguments common with other tasks:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">default_args</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s1">'start_date'</span><span class="p">:</span> <span class="n">dates</span><span class="o">.</span><span class="n">days_ago</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| <p>Note that the neither the body nor the default args are complete in the above examples. |
| Depending on the variables set, there might be different variants on how to pass source |
| code related fields. Currently, you can pass either <code class="docutils literal notranslate"><span class="pre">sourceArchiveUrl</span></code>, |
| <code class="docutils literal notranslate"><span class="pre">sourceRepository</span></code> or <code class="docutils literal notranslate"><span class="pre">sourceUploadUrl</span></code> as described in the |
| <a class="reference external" href="https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions#CloudFunction">Cloud Functions API specification</a>.</p> |
| <p>Additionally, <code class="docutils literal notranslate"><span class="pre">default_args</span></code> or direct operator args might contain <code class="docutils literal notranslate"><span class="pre">zip_path</span></code> |
| parameter |
| to run the extra step of uploading the source code before deploying it. |
| In this case, you also need to provide an empty <code class="docutils literal notranslate"><span class="pre">sourceUploadUrl</span></code> |
| parameter in the body.</p> |
| </div> |
| <div class="section" id="id35"> |
| <h4><a class="toc-backref" href="#id167">Using the operator</a><a class="headerlink" href="#id35" title="Permalink to this headline">¶</a></h4> |
| <p>Depending on the combination of parameters, the Function’s source code can be obtained |
| from different sources:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">if</span> <span class="n">GCF_SOURCE_ARCHIVE_URL</span><span class="p">:</span> |
| <span class="n">body</span><span class="p">[</span><span class="s1">'sourceArchiveUrl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">GCF_SOURCE_ARCHIVE_URL</span> |
| <span class="k">elif</span> <span class="n">GCF_SOURCE_REPOSITORY</span><span class="p">:</span> |
| <span class="n">body</span><span class="p">[</span><span class="s1">'sourceRepository'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s1">'url'</span><span class="p">:</span> <span class="n">GCF_SOURCE_REPOSITORY</span> |
| <span class="p">}</span> |
| <span class="k">elif</span> <span class="n">GCF_ZIP_PATH</span><span class="p">:</span> |
| <span class="n">body</span><span class="p">[</span><span class="s1">'sourceUploadUrl'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">''</span> |
| <span class="n">default_args</span><span class="p">[</span><span class="s1">'zip_path'</span><span class="p">]</span> <span class="o">=</span> <span class="n">GCF_ZIP_PATH</span> |
| <span class="k">elif</span> <span class="n">GCF_SOURCE_UPLOAD_URL</span><span class="p">:</span> |
| <span class="n">body</span><span class="p">[</span><span class="s1">'sourceUploadUrl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">GCF_SOURCE_UPLOAD_URL</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"Please provide one of the source_code parameters"</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>The code to create the operator:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">deploy_task</span> <span class="o">=</span> <span class="n">GcfFunctionDeployOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s2">"gcf_deploy_task"</span><span class="p">,</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">location</span><span class="o">=</span><span class="n">GCP_LOCATION</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">,</span> |
| <span class="n">validate_body</span><span class="o">=</span><span class="n">GCP_VALIDATE_BODY</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>You can also create the operator without project id - project id will be retrieved |
| from the GCP connection used:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">deploy2_task</span> <span class="o">=</span> <span class="n">GcfFunctionDeployOperator</span><span class="p">(</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s2">"gcf_deploy2_task"</span><span class="p">,</span> |
| <span class="n">location</span><span class="o">=</span><span class="n">GCP_LOCATION</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">,</span> |
| <span class="n">validate_body</span><span class="o">=</span><span class="n">GCP_VALIDATE_BODY</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id36"> |
| <h4><a class="toc-backref" href="#id168">Templating</a><a class="headerlink" href="#id36" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'location'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id37"> |
| <h4><a class="toc-backref" href="#id169">Troubleshooting</a><a class="headerlink" href="#id37" title="Permalink to this headline">¶</a></h4> |
| <p>If during the deploy you see an error similar to:</p> |
| <p><cite>“HttpError 403: Missing necessary permission iam.serviceAccounts.actAs for on resource |
| project-name@appspot.gserviceaccount.com. Please grant the |
| roles/iam.serviceAccountUser role.”</cite></p> |
| <p>it means that your service account does not have the correct Cloud IAM permissions.</p> |
| <ol class="arabic simple"> |
| <li>Assign your Service Account the Cloud Functions Developer role.</li> |
| <li>Grant the user the Cloud IAM Service Account User role on the Cloud Functions runtime |
| service account.</li> |
| </ol> |
| <p>The typical way of assigning Cloud IAM permissions with <cite>gcloud</cite> is |
| shown below. Just replace PROJECT_ID with ID of your Google Cloud Platform project |
| and SERVICE_ACCOUNT_EMAIL with the email ID of your service account.</p> |
| <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>gcloud iam service-accounts add-iam-policy-binding <span class="se">\</span> |
| PROJECT_ID@appspot.gserviceaccount.com <span class="se">\</span> |
| --member<span class="o">=</span><span class="s2">"serviceAccount:[SERVICE_ACCOUNT_EMAIL]"</span> <span class="se">\</span> |
| --role<span class="o">=</span><span class="s2">"roles/iam.serviceAccountUser"</span> |
| </pre></div> |
| </div> |
| <p>You can also do that via the GCP Web console.</p> |
| <p>See <a class="reference external" href="https://cloud.google.com/functions/docs/reference/iam/roles#adding_the_iam_service_agent_user_role_to_the_runtime_service_account">Adding the IAM service agent user role to the runtime service</a> for details.</p> |
| <p>If the source code for your function is in Google Source Repository, make sure that |
| your service account has the Source Repository Viewer role so that the source code |
| can be downloaded if necessary.</p> |
| </div> |
| <div class="section" id="id38"> |
| <h4><a class="toc-backref" href="#id170">More information</a><a class="headerlink" href="#id38" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions/create">Google Cloud Functions API documentation</a>.</p> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="google-cloud-spanner-operators"> |
| <h2><a class="toc-backref" href="#id171">Google Cloud Spanner Operators</a><a class="headerlink" href="#google-cloud-spanner-operators" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="cloudspannerinstancedatabasedeleteoperator"> |
| <h3><a class="toc-backref" href="#id172">CloudSpannerInstanceDatabaseDeleteOperator</a><a class="headerlink" href="#cloudspannerinstancedatabasedeleteoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Deletes a database from the specified Cloud Spanner instance. If the database does not |
| exist, no action is taken, and the operator succeeds.</p> |
| <p>For parameter definition, take a look at |
| <code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSpannerInstanceDatabaseDeleteOperator</span></code>.</p> |
| <div class="section" id="id40"> |
| <h4><a class="toc-backref" href="#id173">Arguments</a><a class="headerlink" href="#id40" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_INSTANCE_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_INSTANCE_ID'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_DATABASE_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_DATABASE_ID'</span><span class="p">,</span> <span class="s1">'testdatabase'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_CONFIG_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_CONFIG_NAME'</span><span class="p">,</span> |
| <span class="s1">'projects/example-project/instanceConfigs/eur3'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_NODE_COUNT</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_NODE_COUNT'</span><span class="p">,</span> <span class="s1">'1'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_DISPLAY_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_DISPLAY_NAME'</span><span class="p">,</span> <span class="s1">'Test Instance'</span><span class="p">)</span> |
| <span class="c1"># OPERATION_ID should be unique per operation</span> |
| <span class="n">OPERATION_ID</span> <span class="o">=</span> <span class="s1">'unique_operation_id'</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id41"> |
| <h4><a class="toc-backref" href="#id174">Using the operator</a><a class="headerlink" href="#id41" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">spanner_database_delete_task</span> <span class="o">=</span> <span class="n">CloudSpannerInstanceDatabaseDeleteOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">GCP_SPANNER_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">database_id</span><span class="o">=</span><span class="n">GCP_SPANNER_DATABASE_ID</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'spanner_database_delete_task'</span> |
| <span class="p">)</span> |
| <span class="n">spanner_database_delete_task2</span> <span class="o">=</span> <span class="n">CloudSpannerInstanceDatabaseDeleteOperator</span><span class="p">(</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">GCP_SPANNER_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">database_id</span><span class="o">=</span><span class="n">GCP_SPANNER_DATABASE_ID</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'spanner_database_delete_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id42"> |
| <h4><a class="toc-backref" href="#id175">Templating</a><a class="headerlink" href="#id42" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance_id'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id43"> |
| <h4><a class="toc-backref" href="#id176">More information</a><a class="headerlink" href="#id43" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances.databases/dropDatabase">Google Cloud Spanner API documentation for database drop call</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudspannerinstancedatabasedeployoperator"> |
| <h3><a class="toc-backref" href="#id177">CloudSpannerInstanceDatabaseDeployOperator</a><a class="headerlink" href="#cloudspannerinstancedatabasedeployoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Creates a new Cloud Spanner database in the specified instance, or if the |
| desired database exists, assumes success with no changes applied to database |
| configuration. No structure of the database is verified - it’s enough if the database exists |
| with the same name.</p> |
| <p>For parameter definition, take a look at |
| <code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSpannerInstanceDatabaseDeployOperator</span></code>.</p> |
| <div class="section" id="id44"> |
| <h4><a class="toc-backref" href="#id178">Arguments</a><a class="headerlink" href="#id44" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_INSTANCE_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_INSTANCE_ID'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_DATABASE_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_DATABASE_ID'</span><span class="p">,</span> <span class="s1">'testdatabase'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_CONFIG_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_CONFIG_NAME'</span><span class="p">,</span> |
| <span class="s1">'projects/example-project/instanceConfigs/eur3'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_NODE_COUNT</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_NODE_COUNT'</span><span class="p">,</span> <span class="s1">'1'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_DISPLAY_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_DISPLAY_NAME'</span><span class="p">,</span> <span class="s1">'Test Instance'</span><span class="p">)</span> |
| <span class="c1"># OPERATION_ID should be unique per operation</span> |
| <span class="n">OPERATION_ID</span> <span class="o">=</span> <span class="s1">'unique_operation_id'</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id45"> |
| <h4><a class="toc-backref" href="#id179">Using the operator</a><a class="headerlink" href="#id45" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">spanner_database_deploy_task</span> <span class="o">=</span> <span class="n">CloudSpannerInstanceDatabaseDeployOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">GCP_SPANNER_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">database_id</span><span class="o">=</span><span class="n">GCP_SPANNER_DATABASE_ID</span><span class="p">,</span> |
| <span class="n">ddl_statements</span><span class="o">=</span><span class="p">[</span> |
| <span class="s2">"CREATE TABLE my_table1 (id INT64, name STRING(MAX)) PRIMARY KEY (id)"</span><span class="p">,</span> |
| <span class="s2">"CREATE TABLE my_table2 (id INT64, name STRING(MAX)) PRIMARY KEY (id)"</span><span class="p">,</span> |
| <span class="p">],</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'spanner_database_deploy_task'</span> |
| <span class="p">)</span> |
| <span class="n">spanner_database_deploy_task2</span> <span class="o">=</span> <span class="n">CloudSpannerInstanceDatabaseDeployOperator</span><span class="p">(</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">GCP_SPANNER_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">database_id</span><span class="o">=</span><span class="n">GCP_SPANNER_DATABASE_ID</span><span class="p">,</span> |
| <span class="n">ddl_statements</span><span class="o">=</span><span class="p">[</span> |
| <span class="s2">"CREATE TABLE my_table1 (id INT64, name STRING(MAX)) PRIMARY KEY (id)"</span><span class="p">,</span> |
| <span class="s2">"CREATE TABLE my_table2 (id INT64, name STRING(MAX)) PRIMARY KEY (id)"</span><span class="p">,</span> |
| <span class="p">],</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'spanner_database_deploy_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id46"> |
| <h4><a class="toc-backref" href="#id180">Templating</a><a class="headerlink" href="#id46" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance_id'</span><span class="p">,</span> <span class="s1">'database_id'</span><span class="p">,</span> <span class="s1">'ddl_statements'</span><span class="p">,</span> |
| <span class="s1">'gcp_conn_id'</span><span class="p">)</span> |
| <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,</span> <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id47"> |
| <h4><a class="toc-backref" href="#id181">More information</a><a class="headerlink" href="#id47" title="Permalink to this headline">¶</a></h4> |
| <p>See Google Cloud Spanner API documentation for <a class="reference external" href="https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances.databases/create">database create</a></p> |
| </div> |
| </div> |
| <div class="section" id="cloudspannerinstancedatabaseupdateoperator"> |
| <h3><a class="toc-backref" href="#id182">CloudSpannerInstanceDatabaseUpdateOperator</a><a class="headerlink" href="#cloudspannerinstancedatabaseupdateoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Runs a DDL query in a Cloud Spanner database and allows you to modify the structure of an |
| existing database.</p> |
| <p>You can optionally specify an operation_id parameter which simplifies determining whether |
| the statements were executed in case the update_database call is replayed |
| (idempotency check). The operation_id should be unique within the database, and must be |
| a valid identifier: <cite>[a-z][a-z0-9_]*</cite>. More information can be found in |
| <a class="reference external" href="https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances.databases/updateDdl">the documentation of updateDdl API</a></p> |
| <p>For parameter definition take a look at |
| <code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSpannerInstanceDatabaseUpdateOperator</span></code>.</p> |
| <div class="section" id="id48"> |
| <h4><a class="toc-backref" href="#id183">Arguments</a><a class="headerlink" href="#id48" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_INSTANCE_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_INSTANCE_ID'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_DATABASE_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_DATABASE_ID'</span><span class="p">,</span> <span class="s1">'testdatabase'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_CONFIG_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_CONFIG_NAME'</span><span class="p">,</span> |
| <span class="s1">'projects/example-project/instanceConfigs/eur3'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_NODE_COUNT</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_NODE_COUNT'</span><span class="p">,</span> <span class="s1">'1'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_DISPLAY_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_DISPLAY_NAME'</span><span class="p">,</span> <span class="s1">'Test Instance'</span><span class="p">)</span> |
| <span class="c1"># OPERATION_ID should be unique per operation</span> |
| <span class="n">OPERATION_ID</span> <span class="o">=</span> <span class="s1">'unique_operation_id'</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id49"> |
| <h4><a class="toc-backref" href="#id184">Using the operator</a><a class="headerlink" href="#id49" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">spanner_database_update_task</span> <span class="o">=</span> <span class="n">CloudSpannerInstanceDatabaseUpdateOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">GCP_SPANNER_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">database_id</span><span class="o">=</span><span class="n">GCP_SPANNER_DATABASE_ID</span><span class="p">,</span> |
| <span class="n">ddl_statements</span><span class="o">=</span><span class="p">[</span> |
| <span class="s2">"CREATE TABLE my_table3 (id INT64, name STRING(MAX)) PRIMARY KEY (id)"</span><span class="p">,</span> |
| <span class="p">],</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'spanner_database_update_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">spanner_database_update_idempotent1_task</span> <span class="o">=</span> <span class="n">CloudSpannerInstanceDatabaseUpdateOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">GCP_SPANNER_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">database_id</span><span class="o">=</span><span class="n">GCP_SPANNER_DATABASE_ID</span><span class="p">,</span> |
| <span class="n">operation_id</span><span class="o">=</span><span class="n">OPERATION_ID</span><span class="p">,</span> |
| <span class="n">ddl_statements</span><span class="o">=</span><span class="p">[</span> |
| <span class="s2">"CREATE TABLE my_table_unique (id INT64, name STRING(MAX)) PRIMARY KEY (id)"</span><span class="p">,</span> |
| <span class="p">],</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'spanner_database_update_idempotent1_task'</span> |
| <span class="p">)</span> |
| <span class="n">spanner_database_update_idempotent2_task</span> <span class="o">=</span> <span class="n">CloudSpannerInstanceDatabaseUpdateOperator</span><span class="p">(</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">GCP_SPANNER_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">database_id</span><span class="o">=</span><span class="n">GCP_SPANNER_DATABASE_ID</span><span class="p">,</span> |
| <span class="n">operation_id</span><span class="o">=</span><span class="n">OPERATION_ID</span><span class="p">,</span> |
| <span class="n">ddl_statements</span><span class="o">=</span><span class="p">[</span> |
| <span class="s2">"CREATE TABLE my_table_unique (id INT64, name STRING(MAX)) PRIMARY KEY (id)"</span><span class="p">,</span> |
| <span class="p">],</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'spanner_database_update_idempotent2_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id50"> |
| <h4><a class="toc-backref" href="#id185">Templating</a><a class="headerlink" href="#id50" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance_id'</span><span class="p">,</span> <span class="s1">'database_id'</span><span class="p">,</span> <span class="s1">'ddl_statements'</span><span class="p">,</span> |
| <span class="s1">'gcp_conn_id'</span><span class="p">)</span> |
| <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,</span> <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id51"> |
| <h4><a class="toc-backref" href="#id186">More information</a><a class="headerlink" href="#id51" title="Permalink to this headline">¶</a></h4> |
| <p>See Google Cloud Spanner API documentation for <a class="reference external" href="https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances.databases/updateDdl">database update_ddl</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudspannerinstancedatabasequeryoperator"> |
| <h3><a class="toc-backref" href="#id187">CloudSpannerInstanceDatabaseQueryOperator</a><a class="headerlink" href="#cloudspannerinstancedatabasequeryoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Executes an arbitrary DML query (INSERT, UPDATE, DELETE).</p> |
| <p>For parameter definition take a look at |
| <code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSpannerInstanceDatabaseQueryOperator</span></code>.</p> |
| <div class="section" id="id52"> |
| <h4><a class="toc-backref" href="#id188">Arguments</a><a class="headerlink" href="#id52" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables.</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_INSTANCE_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_INSTANCE_ID'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_DATABASE_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_DATABASE_ID'</span><span class="p">,</span> <span class="s1">'testdatabase'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_CONFIG_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_CONFIG_NAME'</span><span class="p">,</span> |
| <span class="s1">'projects/example-project/instanceConfigs/eur3'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_NODE_COUNT</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_NODE_COUNT'</span><span class="p">,</span> <span class="s1">'1'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_DISPLAY_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_DISPLAY_NAME'</span><span class="p">,</span> <span class="s1">'Test Instance'</span><span class="p">)</span> |
| <span class="c1"># OPERATION_ID should be unique per operation</span> |
| <span class="n">OPERATION_ID</span> <span class="o">=</span> <span class="s1">'unique_operation_id'</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id53"> |
| <h4><a class="toc-backref" href="#id189">Using the operator</a><a class="headerlink" href="#id53" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">spanner_instance_query_task</span> <span class="o">=</span> <span class="n">CloudSpannerInstanceDatabaseQueryOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">GCP_SPANNER_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">database_id</span><span class="o">=</span><span class="n">GCP_SPANNER_DATABASE_ID</span><span class="p">,</span> |
| <span class="n">query</span><span class="o">=</span><span class="p">[</span><span class="s2">"DELETE FROM my_table2 WHERE true"</span><span class="p">],</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'spanner_instance_query_task'</span> |
| <span class="p">)</span> |
| <span class="n">spanner_instance_query_task2</span> <span class="o">=</span> <span class="n">CloudSpannerInstanceDatabaseQueryOperator</span><span class="p">(</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">GCP_SPANNER_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">database_id</span><span class="o">=</span><span class="n">GCP_SPANNER_DATABASE_ID</span><span class="p">,</span> |
| <span class="n">query</span><span class="o">=</span><span class="p">[</span><span class="s2">"DELETE FROM my_table2 WHERE true"</span><span class="p">],</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'spanner_instance_query_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id54"> |
| <h4><a class="toc-backref" href="#id190">Templating</a><a class="headerlink" href="#id54" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance_id'</span><span class="p">,</span> <span class="s1">'database_id'</span><span class="p">,</span> <span class="s1">'query'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">)</span> |
| <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id55"> |
| <h4><a class="toc-backref" href="#id191">More information</a><a class="headerlink" href="#id55" title="Permalink to this headline">¶</a></h4> |
| <p>See Google Cloud Spanner API documentation for <a class="reference external" href="https://cloud.google.com/spanner/docs/dml-syntax">the DML syntax</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudspannerinstancedeleteoperator"> |
| <h3><a class="toc-backref" href="#id192">CloudSpannerInstanceDeleteOperator</a><a class="headerlink" href="#cloudspannerinstancedeleteoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Deletes a Cloud Spanner instance. If an instance does not exist, no action is taken, |
| and the operator succeeds.</p> |
| <p>For parameter definition take a look at |
| <code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSpannerInstanceDeleteOperator</span></code>.</p> |
| <div class="section" id="id56"> |
| <h4><a class="toc-backref" href="#id193">Arguments</a><a class="headerlink" href="#id56" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_INSTANCE_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_INSTANCE_ID'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_DATABASE_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_DATABASE_ID'</span><span class="p">,</span> <span class="s1">'testdatabase'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_CONFIG_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_CONFIG_NAME'</span><span class="p">,</span> |
| <span class="s1">'projects/example-project/instanceConfigs/eur3'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_NODE_COUNT</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_NODE_COUNT'</span><span class="p">,</span> <span class="s1">'1'</span><span class="p">)</span> |
| <span class="n">GCP_SPANNER_DISPLAY_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_SPANNER_DISPLAY_NAME'</span><span class="p">,</span> <span class="s1">'Test Instance'</span><span class="p">)</span> |
| <span class="c1"># OPERATION_ID should be unique per operation</span> |
| <span class="n">OPERATION_ID</span> <span class="o">=</span> <span class="s1">'unique_operation_id'</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id57"> |
| <h4><a class="toc-backref" href="#id194">Using the operator</a><a class="headerlink" href="#id57" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">spanner_instance_delete_task</span> <span class="o">=</span> <span class="n">CloudSpannerInstanceDeleteOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">GCP_SPANNER_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'spanner_instance_delete_task'</span> |
| <span class="p">)</span> |
| <span class="n">spanner_instance_delete_task2</span> <span class="o">=</span> <span class="n">CloudSpannerInstanceDeleteOperator</span><span class="p">(</span> |
| <span class="n">instance_id</span><span class="o">=</span><span class="n">GCP_SPANNER_INSTANCE_ID</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'spanner_instance_delete_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id58"> |
| <h4><a class="toc-backref" href="#id195">Templating</a><a class="headerlink" href="#id58" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance_id'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id59"> |
| <h4><a class="toc-backref" href="#id196">More information</a><a class="headerlink" href="#id59" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances/delete">Google Cloud Spanner API documentation for instance delete</a>.</p> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="google-cloud-sql-operators"> |
| <h2><a class="toc-backref" href="#id197">Google Cloud Sql Operators</a><a class="headerlink" href="#google-cloud-sql-operators" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="cloudsqlinstancedatabasecreateoperator"> |
| <h3><a class="toc-backref" href="#id198">CloudSqlInstanceDatabaseCreateOperator</a><a class="headerlink" href="#cloudsqlinstancedatabasecreateoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Creates a new database inside a Cloud SQL instance.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseCreateOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseCreateOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceDatabaseCreateOperator</span></code></a>.</p> |
| <div class="section" id="id60"> |
| <h4><a class="toc-backref" href="#id199">Arguments</a><a class="headerlink" href="#id60" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'test-mysql'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME2</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME2'</span><span class="p">,</span> <span class="s1">'test-mysql2'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_DATABASE_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id61"> |
| <h4><a class="toc-backref" href="#id200">Using the operator</a><a class="headerlink" href="#id61" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_db_create_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDatabaseCreateOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">db_create_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_db_create_task'</span> |
| <span class="p">)</span> |
| <span class="n">sql_db_create_task2</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDatabaseCreateOperator</span><span class="p">(</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">db_create_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_db_create_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Example request body:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">db_create_body</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"instance"</span><span class="p">:</span> <span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="s2">"name"</span><span class="p">:</span> <span class="n">DB_NAME</span><span class="p">,</span> |
| <span class="s2">"project"</span><span class="p">:</span> <span class="n">GCP_PROJECT_ID</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id62"> |
| <h4><a class="toc-backref" href="#id201">Templating</a><a class="headerlink" href="#id62" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id63"> |
| <h4><a class="toc-backref" href="#id202">More information</a><a class="headerlink" href="#id63" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/insert">Google Cloud SQL API documentation for database insert</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstancedatabasedeleteoperator"> |
| <h3><a class="toc-backref" href="#id203">CloudSqlInstanceDatabaseDeleteOperator</a><a class="headerlink" href="#cloudsqlinstancedatabasedeleteoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Deletes a database from a Cloud SQL instance.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseDeleteOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseDeleteOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceDatabaseDeleteOperator</span></code></a>.</p> |
| <div class="section" id="id64"> |
| <h4><a class="toc-backref" href="#id204">Arguments</a><a class="headerlink" href="#id64" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'test-mysql'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME2</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME2'</span><span class="p">,</span> <span class="s1">'test-mysql2'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_DATABASE_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id65"> |
| <h4><a class="toc-backref" href="#id205">Using the operator</a><a class="headerlink" href="#id65" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_db_delete_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDatabaseDeleteOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">database</span><span class="o">=</span><span class="n">DB_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_db_delete_task'</span> |
| <span class="p">)</span> |
| <span class="n">sql_db_delete_task2</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDatabaseDeleteOperator</span><span class="p">(</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">database</span><span class="o">=</span><span class="n">DB_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_db_delete_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id66"> |
| <h4><a class="toc-backref" href="#id206">Templating</a><a class="headerlink" href="#id66" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'database'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> |
| <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id67"> |
| <h4><a class="toc-backref" href="#id207">More information</a><a class="headerlink" href="#id67" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/delete">Google Cloud SQL API documentation for database delete</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstancedatabasepatchoperator"> |
| <h3><a class="toc-backref" href="#id208">CloudSqlInstanceDatabasePatchOperator</a><a class="headerlink" href="#cloudsqlinstancedatabasepatchoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Updates a resource containing information about a database inside a Cloud SQL instance |
| using patch semantics. |
| See: <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch">https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch</a></p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabasePatchOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabasePatchOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceDatabasePatchOperator</span></code></a>.</p> |
| <div class="section" id="id68"> |
| <h4><a class="toc-backref" href="#id209">Arguments</a><a class="headerlink" href="#id68" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'test-mysql'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME2</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME2'</span><span class="p">,</span> <span class="s1">'test-mysql2'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_DATABASE_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id69"> |
| <h4><a class="toc-backref" href="#id210">Using the operator</a><a class="headerlink" href="#id69" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_db_patch_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDatabasePatchOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">db_patch_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">database</span><span class="o">=</span><span class="n">DB_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_db_patch_task'</span> |
| <span class="p">)</span> |
| <span class="n">sql_db_patch_task2</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDatabasePatchOperator</span><span class="p">(</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">db_patch_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">database</span><span class="o">=</span><span class="n">DB_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_db_patch_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Example request body:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">db_patch_body</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"charset"</span><span class="p">:</span> <span class="s2">"utf16"</span><span class="p">,</span> |
| <span class="s2">"collation"</span><span class="p">:</span> <span class="s2">"utf16_general_ci"</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id70"> |
| <h4><a class="toc-backref" href="#id211">Templating</a><a class="headerlink" href="#id70" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'database'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> |
| <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id71"> |
| <h4><a class="toc-backref" href="#id212">More information</a><a class="headerlink" href="#id71" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/patch">Google Cloud SQL API documentation for database patch</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstancedeleteoperator"> |
| <h3><a class="toc-backref" href="#id213">CloudSqlInstanceDeleteOperator</a><a class="headerlink" href="#cloudsqlinstancedeleteoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Deletes a Cloud SQL instance in Google Cloud Platform.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDeleteOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDeleteOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceDeleteOperator</span></code></a>.</p> |
| <div class="section" id="id72"> |
| <h4><a class="toc-backref" href="#id214">Arguments</a><a class="headerlink" href="#id72" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from OS environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'test-mysql'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME2</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME2'</span><span class="p">,</span> <span class="s1">'test-mysql2'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_DATABASE_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id73"> |
| <h4><a class="toc-backref" href="#id215">Using the operator</a><a class="headerlink" href="#id73" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_instance_delete_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDeleteOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_instance_delete_task'</span> |
| <span class="p">)</span> |
| <span class="n">sql_instance_delete_task2</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDeleteOperator</span><span class="p">(</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME2</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_instance_delete_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id74"> |
| <h4><a class="toc-backref" href="#id216">Templating</a><a class="headerlink" href="#id74" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id75"> |
| <h4><a class="toc-backref" href="#id217">More information</a><a class="headerlink" href="#id75" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/delete">Google Cloud SQL API documentation for delete</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstanceexportoperator"> |
| <h3><a class="toc-backref" href="#id218">CloudSqlInstanceExportOperator</a><a class="headerlink" href="#cloudsqlinstanceexportoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Exports data from a Cloud SQL instance to a Cloud Storage bucket as a SQL dump |
| or CSV file.</p> |
| <p>Note: This operator is idempotent. If executed multiple times with the same |
| export file URI, the export file in GCS will simply be overridden.</p> |
| <p>For parameter definition take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceExportOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceExportOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceExportOperator</span></code></a>.</p> |
| <div class="section" id="id76"> |
| <h4><a class="toc-backref" href="#id219">Arguments</a><a class="headerlink" href="#id76" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from Airflow variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'test-mysql'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME2</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME2'</span><span class="p">,</span> <span class="s1">'test-mysql2'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_DATABASE_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">EXPORT_URI</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_EXPORT_URI'</span><span class="p">,</span> <span class="s1">'gs://bucketName/fileName'</span><span class="p">)</span> |
| <span class="n">IMPORT_URI</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_IMPORT_URI'</span><span class="p">,</span> <span class="s1">'gs://bucketName/fileName'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Example body defining the export operation:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">export_body</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"exportContext"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"fileType"</span><span class="p">:</span> <span class="s2">"sql"</span><span class="p">,</span> |
| <span class="s2">"uri"</span><span class="p">:</span> <span class="n">EXPORT_URI</span><span class="p">,</span> |
| <span class="s2">"sqlExportOptions"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"schemaOnly"</span><span class="p">:</span> <span class="bp">False</span> |
| <span class="p">}</span> |
| <span class="p">}</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id77"> |
| <h4><a class="toc-backref" href="#id220">Using the operator</a><a class="headerlink" href="#id77" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_export_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceExportOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">export_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_export_task'</span> |
| <span class="p">)</span> |
| <span class="n">sql_export_task2</span> <span class="o">=</span> <span class="n">CloudSqlInstanceExportOperator</span><span class="p">(</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">export_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_export_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id78"> |
| <h4><a class="toc-backref" href="#id221">Templating</a><a class="headerlink" href="#id78" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id79"> |
| <h4><a class="toc-backref" href="#id222">More information</a><a class="headerlink" href="#id79" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/export">Google Cloud SQL API documentation for export</a>.</p> |
| </div> |
| <div class="section" id="id80"> |
| <h4><a class="toc-backref" href="#id223">Troubleshooting</a><a class="headerlink" href="#id80" title="Permalink to this headline">¶</a></h4> |
| <p>If you receive an “Unauthorized” error in GCP, make sure that the service account |
| of the Cloud SQL instance is authorized to write to the selected GCS bucket.</p> |
| <p>It is not the service account configured in Airflow that communicates with GCS, |
| but rather the service account of the particular Cloud SQL instance.</p> |
| <p>To grant the service account with the appropriate WRITE permissions for the GCS bucket |
| you can use the <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageBucketCreateAclEntryOperator" title="airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageBucketCreateAclEntryOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GoogleCloudStorageBucketCreateAclEntryOperator</span></code></a>, |
| as shown in the example:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_gcp_add_bucket_permission_task</span> <span class="o">=</span> <span class="n">GoogleCloudStorageBucketCreateAclEntryOperator</span><span class="p">(</span> |
| <span class="n">entity</span><span class="o">=</span><span class="s2">"user-{{ task_instance.xcom_pull("</span> |
| <span class="s2">"'sql_instance_create_task', key='service_account_email') "</span> |
| <span class="s2">"}}"</span><span class="p">,</span> |
| <span class="n">role</span><span class="o">=</span><span class="s2">"WRITER"</span><span class="p">,</span> |
| <span class="n">bucket</span><span class="o">=</span><span class="n">export_url_split</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="c1"># netloc (bucket)</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_gcp_add_bucket_permission_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstanceimportoperator"> |
| <h3><a class="toc-backref" href="#id224">CloudSqlInstanceImportOperator</a><a class="headerlink" href="#cloudsqlinstanceimportoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Imports data into a Cloud SQL instance from a SQL dump or CSV file in Cloud Storage.</p> |
| <div class="section" id="csv-import"> |
| <h4><a class="toc-backref" href="#id225">CSV import:</a><a class="headerlink" href="#csv-import" title="Permalink to this headline">¶</a></h4> |
| <p>This operator is NOT idempotent for a CSV import. If the same file is imported |
| multiple times, the imported data will be duplicated in the database. |
| Moreover, if there are any unique constraints the duplicate import may result in an |
| error.</p> |
| </div> |
| <div class="section" id="sql-import"> |
| <h4><a class="toc-backref" href="#id226">SQL import:</a><a class="headerlink" href="#sql-import" title="Permalink to this headline">¶</a></h4> |
| <p>This operator is idempotent for a SQL import if it was also exported by Cloud SQL. |
| The exported SQL contains ‘DROP TABLE IF EXISTS’ statements for all tables |
| to be imported.</p> |
| <p>If the import file was generated in a different way, idempotence is not guaranteed. |
| It has to be ensured on the SQL file level.</p> |
| <p>For parameter definition take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceImportOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceImportOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceImportOperator</span></code></a>.</p> |
| </div> |
| <div class="section" id="id81"> |
| <h4><a class="toc-backref" href="#id227">Arguments</a><a class="headerlink" href="#id81" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from Airflow variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'test-mysql'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME2</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME2'</span><span class="p">,</span> <span class="s1">'test-mysql2'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_DATABASE_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">EXPORT_URI</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_EXPORT_URI'</span><span class="p">,</span> <span class="s1">'gs://bucketName/fileName'</span><span class="p">)</span> |
| <span class="n">IMPORT_URI</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_IMPORT_URI'</span><span class="p">,</span> <span class="s1">'gs://bucketName/fileName'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Example body defining the import operation:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">import_body</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"importContext"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"fileType"</span><span class="p">:</span> <span class="s2">"sql"</span><span class="p">,</span> |
| <span class="s2">"uri"</span><span class="p">:</span> <span class="n">IMPORT_URI</span> |
| <span class="p">}</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id82"> |
| <h4><a class="toc-backref" href="#id228">Using the operator</a><a class="headerlink" href="#id82" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_import_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceImportOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">import_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME2</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_import_task'</span> |
| <span class="p">)</span> |
| <span class="n">sql_import_task2</span> <span class="o">=</span> <span class="n">CloudSqlInstanceImportOperator</span><span class="p">(</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">import_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME2</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_import_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id83"> |
| <h4><a class="toc-backref" href="#id229">Templating</a><a class="headerlink" href="#id83" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id84"> |
| <h4><a class="toc-backref" href="#id230">More information</a><a class="headerlink" href="#id84" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/import">Google Cloud SQL API documentation for import</a>.</p> |
| </div> |
| <div class="section" id="id85"> |
| <h4><a class="toc-backref" href="#id231">Troubleshooting</a><a class="headerlink" href="#id85" title="Permalink to this headline">¶</a></h4> |
| <p>If you receive an “Unauthorized” error in GCP, make sure that the service account |
| of the Cloud SQL instance is authorized to read from the selected GCS object.</p> |
| <p>It is not the service account configured in Airflow that communicates with GCS, |
| but rather the service account of the particular Cloud SQL instance.</p> |
| <p>To grant the service account with the appropriate READ permissions for the GCS object |
| you can use the <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageObjectCreateAclEntryOperator" title="airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageObjectCreateAclEntryOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GoogleCloudStorageObjectCreateAclEntryOperator</span></code></a>, |
| as shown in the example:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_gcp_add_object_permission_task</span> <span class="o">=</span> <span class="n">GoogleCloudStorageObjectCreateAclEntryOperator</span><span class="p">(</span> |
| <span class="n">entity</span><span class="o">=</span><span class="s2">"user-{{ task_instance.xcom_pull("</span> |
| <span class="s2">"'sql_instance_create_task2', key='service_account_email')"</span> |
| <span class="s2">" }}"</span><span class="p">,</span> |
| <span class="n">role</span><span class="o">=</span><span class="s2">"READER"</span><span class="p">,</span> |
| <span class="n">bucket</span><span class="o">=</span><span class="n">import_url_split</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="c1"># netloc (bucket)</span> |
| <span class="n">object_name</span><span class="o">=</span><span class="n">import_url_split</span><span class="p">[</span><span class="mi">2</span><span class="p">][</span><span class="mi">1</span><span class="p">:],</span> <span class="c1"># path (strip first '/')</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_gcp_add_object_permission_task'</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">prev_task</span> <span class="o">=</span> <span class="n">next_dep</span><span class="p">(</span><span class="n">sql_gcp_add_object_permission_task</span><span class="p">,</span> <span class="n">prev_task</span><span class="p">)</span> |
| |
| <span class="c1"># For import to work we also need to add the Cloud SQL instance's Service Account</span> |
| <span class="c1"># write access to the whole bucket!.</span> |
| <span class="n">sql_gcp_add_bucket_permission_2_task</span> <span class="o">=</span> <span class="n">GoogleCloudStorageBucketCreateAclEntryOperator</span><span class="p">(</span> |
| <span class="n">entity</span><span class="o">=</span><span class="s2">"user-{{ task_instance.xcom_pull("</span> |
| <span class="s2">"'sql_instance_create_task2', key='service_account_email') "</span> |
| <span class="s2">"}}"</span><span class="p">,</span> |
| <span class="n">role</span><span class="o">=</span><span class="s2">"WRITER"</span><span class="p">,</span> |
| <span class="n">bucket</span><span class="o">=</span><span class="n">import_url_split</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="c1"># netloc</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_gcp_add_bucket_permission_2_task'</span><span class="p">,</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstancecreateoperator"> |
| <span id="id86"></span><h3><a class="toc-backref" href="#id232">CloudSqlInstanceCreateOperator</a><a class="headerlink" href="#cloudsqlinstancecreateoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Creates a new Cloud SQL instance in Google Cloud Platform.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceCreateOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceCreateOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceCreateOperator</span></code></a>.</p> |
| <p>If an instance with the same name exists, no action will be taken and the operator |
| will succeed.</p> |
| <div class="section" id="id87"> |
| <h4><a class="toc-backref" href="#id233">Arguments</a><a class="headerlink" href="#id87" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from OS environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'test-mysql'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME2</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME2'</span><span class="p">,</span> <span class="s1">'test-mysql2'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_DATABASE_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Example body defining the instance:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">body</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"name"</span><span class="p">:</span> <span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="s2">"settings"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"tier"</span><span class="p">:</span> <span class="s2">"db-n1-standard-1"</span><span class="p">,</span> |
| <span class="s2">"backupConfiguration"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"binaryLogEnabled"</span><span class="p">:</span> <span class="bp">True</span><span class="p">,</span> |
| <span class="s2">"enabled"</span><span class="p">:</span> <span class="bp">True</span><span class="p">,</span> |
| <span class="s2">"startTime"</span><span class="p">:</span> <span class="s2">"05:00"</span> |
| <span class="p">},</span> |
| <span class="s2">"activationPolicy"</span><span class="p">:</span> <span class="s2">"ALWAYS"</span><span class="p">,</span> |
| <span class="s2">"dataDiskSizeGb"</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span> |
| <span class="s2">"dataDiskType"</span><span class="p">:</span> <span class="s2">"PD_SSD"</span><span class="p">,</span> |
| <span class="s2">"databaseFlags"</span><span class="p">:</span> <span class="p">[],</span> |
| <span class="s2">"ipConfiguration"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"ipv4Enabled"</span><span class="p">:</span> <span class="bp">True</span><span class="p">,</span> |
| <span class="s2">"requireSsl"</span><span class="p">:</span> <span class="bp">True</span><span class="p">,</span> |
| <span class="p">},</span> |
| <span class="s2">"locationPreference"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"zone"</span><span class="p">:</span> <span class="s2">"europe-west4-a"</span> |
| <span class="p">},</span> |
| <span class="s2">"maintenanceWindow"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"hour"</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> |
| <span class="s2">"day"</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span> |
| <span class="s2">"updateTrack"</span><span class="p">:</span> <span class="s2">"canary"</span> |
| <span class="p">},</span> |
| <span class="s2">"pricingPlan"</span><span class="p">:</span> <span class="s2">"PER_USE"</span><span class="p">,</span> |
| <span class="s2">"replicationType"</span><span class="p">:</span> <span class="s2">"ASYNCHRONOUS"</span><span class="p">,</span> |
| <span class="s2">"storageAutoResize"</span><span class="p">:</span> <span class="bp">False</span><span class="p">,</span> |
| <span class="s2">"storageAutoResizeLimit"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> |
| <span class="s2">"userLabels"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"my-key"</span><span class="p">:</span> <span class="s2">"my-value"</span> |
| <span class="p">}</span> |
| <span class="p">},</span> |
| <span class="s2">"databaseVersion"</span><span class="p">:</span> <span class="s2">"MYSQL_5_7"</span><span class="p">,</span> |
| <span class="s2">"region"</span><span class="p">:</span> <span class="s2">"europe-west4"</span><span class="p">,</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id88"> |
| <h4><a class="toc-backref" href="#id234">Using the operator</a><a class="headerlink" href="#id88" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_instance_create_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceCreateOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_instance_create_task'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id89"> |
| <h4><a class="toc-backref" href="#id235">Templating</a><a class="headerlink" href="#id89" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id90"> |
| <h4><a class="toc-backref" href="#id236">More information</a><a class="headerlink" href="#id90" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/insert">Google Cloud SQL API documentation for insert</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlinstancepatchoperator"> |
| <span id="id91"></span><h3><a class="toc-backref" href="#id237">CloudSqlInstancePatchOperator</a><a class="headerlink" href="#cloudsqlinstancepatchoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Updates settings of a Cloud SQL instance in Google Cloud Platform (partial update).</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstancePatchOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstancePatchOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstancePatchOperator</span></code></a>.</p> |
| <p>This is a partial update, so only values for the settings specified in the body |
| will be set / updated. The rest of the existing instance’s configuration will remain |
| unchanged.</p> |
| <div class="section" id="id92"> |
| <h4><a class="toc-backref" href="#id238">Arguments</a><a class="headerlink" href="#id92" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from OS environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'test-mysql'</span><span class="p">)</span> |
| <span class="n">INSTANCE_NAME2</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME2'</span><span class="p">,</span> <span class="s1">'test-mysql2'</span><span class="p">)</span> |
| <span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_DATABASE_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| <p>Example body defining the instance:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">patch_body</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s2">"name"</span><span class="p">:</span> <span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="s2">"settings"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"dataDiskSizeGb"</span><span class="p">:</span> <span class="mi">35</span><span class="p">,</span> |
| <span class="s2">"maintenanceWindow"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"hour"</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> |
| <span class="s2">"day"</span><span class="p">:</span> <span class="mi">6</span><span class="p">,</span> |
| <span class="s2">"updateTrack"</span><span class="p">:</span> <span class="s2">"canary"</span> |
| <span class="p">},</span> |
| <span class="s2">"userLabels"</span><span class="p">:</span> <span class="p">{</span> |
| <span class="s2">"my-key-patch"</span><span class="p">:</span> <span class="s2">"my-value-patch"</span> |
| <span class="p">}</span> |
| <span class="p">}</span> |
| <span class="p">}</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id93"> |
| <h4><a class="toc-backref" href="#id239">Using the operator</a><a class="headerlink" href="#id93" title="Permalink to this headline">¶</a></h4> |
| <p>You can create the operator with or without project id. If project id is missing |
| it will be retrieved from the GCP connection used. Both variants are shown:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_instance_patch_task</span> <span class="o">=</span> <span class="n">CloudSqlInstancePatchOperator</span><span class="p">(</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">patch_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_instance_patch_task'</span> |
| <span class="p">)</span> |
| |
| <span class="n">sql_instance_patch_task2</span> <span class="o">=</span> <span class="n">CloudSqlInstancePatchOperator</span><span class="p">(</span> |
| <span class="n">body</span><span class="o">=</span><span class="n">patch_body</span><span class="p">,</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_instance_patch_task2'</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id94"> |
| <h4><a class="toc-backref" href="#id240">Templating</a><a class="headerlink" href="#id94" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id95"> |
| <h4><a class="toc-backref" href="#id241">More information</a><a class="headerlink" href="#id95" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/patch">Google Cloud SQL API documentation for patch</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="cloudsqlqueryoperator"> |
| <h3><a class="toc-backref" href="#id242">CloudSqlQueryOperator</a><a class="headerlink" href="#cloudsqlqueryoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Performs DDL or DML SQL queries in Google Cloud SQL instance. The DQL |
| (retrieving data from Google Cloud SQL) is not supported. You might run the SELECT |
| queries, but the results of those queries are discarded.</p> |
| <p>You can specify various connectivity methods to connect to running instance, |
| starting from public IP plain connection through public IP with SSL or both TCP and |
| socket connection via Cloud SQL Proxy. The proxy is downloaded and started/stopped |
| dynamically as needed by the operator.</p> |
| <p>There is a <em>gcpcloudsql://</em> connection type that you should use to define what |
| kind of connectivity you want the operator to use. The connection is a “meta” |
| type of connection. It is not used to make an actual connectivity on its own, but it |
| determines whether Cloud SQL Proxy should be started by <cite>CloudSqlDatabaseHook</cite> |
| and what kind of database connection (Postgres or MySQL) should be created |
| dynamically to connect to Cloud SQL via public IP address or via the proxy. |
| The ‘CloudSqlDatabaseHook` uses |
| <a class="reference internal" href="../integration.html#airflow.contrib.hooks.gcp_sql_hook.CloudSqlProxyRunner" title="airflow.contrib.hooks.gcp_sql_hook.CloudSqlProxyRunner"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlProxyRunner</span></code></a> to manage Cloud SQL |
| Proxy lifecycle (each task has its own Cloud SQL Proxy)</p> |
| <p>When you build connection, you should use connection parameters as described in |
| <a class="reference internal" href="../integration.html#airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook" title="airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlDatabaseHook</span></code></a>. You can see |
| examples of connections below for all the possible types of connectivity. Such connection |
| can be reused between different tasks (instances of <cite>CloudSqlQueryOperator</cite>). Each |
| task will get their own proxy started if needed with their own TCP or UNIX socket.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlQueryOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlQueryOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlQueryOperator</span></code></a>.</p> |
| <p>Since query operator can run arbitrary query, it cannot be guaranteed to be |
| idempotent. SQL query designer should design the queries to be idempotent. For example, |
| both Postgres and MySQL support CREATE TABLE IF NOT EXISTS statements that can be |
| used to create tables in an idempotent way.</p> |
| <div class="section" id="id96"> |
| <h4><a class="toc-backref" href="#id243">Arguments</a><a class="headerlink" href="#id96" title="Permalink to this headline">¶</a></h4> |
| <p>If you define connection via <cite>AIRFLOW_CONN_*</cite> URL defined in an environment |
| variable, make sure the URL components in the URL are URL-encoded. |
| See examples below for details.</p> |
| <p>Note that in case of SSL connections you need to have a mechanism to make the |
| certificate/key files available in predefined locations for all the workers on |
| which the operator can run. This can be provided for example by mounting |
| NFS-like volumes in the same path for all the workers.</p> |
| <p>Some arguments in the example DAG are taken from the OS environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span> |
| <span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> |
| <span class="n">GCP_REGION</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCP_REGION'</span><span class="p">,</span> <span class="s1">'europe-west-1b'</span><span class="p">)</span> |
| |
| <span class="n">GCSQL_POSTGRES_INSTANCE_NAME_QUERY</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span> |
| <span class="s1">'GCSQL_POSTGRES_INSTANCE_NAME_QUERY'</span><span class="p">,</span> |
| <span class="s1">'testpostgres'</span><span class="p">)</span> |
| <span class="n">GCSQL_POSTGRES_DATABASE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_POSTGRES_DATABASE_NAME'</span><span class="p">,</span> |
| <span class="s1">'postgresdb'</span><span class="p">)</span> |
| <span class="n">GCSQL_POSTGRES_USER</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_POSTGRES_USER'</span><span class="p">,</span> <span class="s1">'postgres_user'</span><span class="p">)</span> |
| <span class="n">GCSQL_POSTGRES_PASSWORD</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_POSTGRES_PASSWORD'</span><span class="p">,</span> <span class="s1">'password'</span><span class="p">)</span> |
| <span class="n">GCSQL_POSTGRES_PUBLIC_IP</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_POSTGRES_PUBLIC_IP'</span><span class="p">,</span> <span class="s1">'0.0.0.0'</span><span class="p">)</span> |
| <span class="n">GCSQL_POSTGRES_PUBLIC_PORT</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_POSTGRES_PUBLIC_PORT'</span><span class="p">,</span> <span class="mi">5432</span><span class="p">)</span> |
| <span class="n">GCSQL_POSTGRES_CLIENT_CERT_FILE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_POSTGRES_CLIENT_CERT_FILE'</span><span class="p">,</span> |
| <span class="s2">".key/postgres-client-cert.pem"</span><span class="p">)</span> |
| <span class="n">GCSQL_POSTGRES_CLIENT_KEY_FILE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_POSTGRES_CLIENT_KEY_FILE'</span><span class="p">,</span> |
| <span class="s2">".key/postgres-client-key.pem"</span><span class="p">)</span> |
| <span class="n">GCSQL_POSTGRES_SERVER_CA_FILE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_POSTGRES_SERVER_CA_FILE'</span><span class="p">,</span> |
| <span class="s2">".key/postgres-server-ca.pem"</span><span class="p">)</span> |
| |
| <span class="n">GCSQL_MYSQL_INSTANCE_NAME_QUERY</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_INSTANCE_NAME_QUERY'</span><span class="p">,</span> |
| <span class="s1">'testmysql'</span><span class="p">)</span> |
| <span class="n">GCSQL_MYSQL_DATABASE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_DATABASE_NAME'</span><span class="p">,</span> <span class="s1">'mysqldb'</span><span class="p">)</span> |
| <span class="n">GCSQL_MYSQL_USER</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_USER'</span><span class="p">,</span> <span class="s1">'mysql_user'</span><span class="p">)</span> |
| <span class="n">GCSQL_MYSQL_PASSWORD</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_PASSWORD'</span><span class="p">,</span> <span class="s1">'password'</span><span class="p">)</span> |
| <span class="n">GCSQL_MYSQL_PUBLIC_IP</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_PUBLIC_IP'</span><span class="p">,</span> <span class="s1">'0.0.0.0'</span><span class="p">)</span> |
| <span class="n">GCSQL_MYSQL_PUBLIC_PORT</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_PUBLIC_PORT'</span><span class="p">,</span> <span class="mi">3306</span><span class="p">)</span> |
| <span class="n">GCSQL_MYSQL_CLIENT_CERT_FILE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_CLIENT_CERT_FILE'</span><span class="p">,</span> |
| <span class="s2">".key/mysql-client-cert.pem"</span><span class="p">)</span> |
| <span class="n">GCSQL_MYSQL_CLIENT_KEY_FILE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_CLIENT_KEY_FILE'</span><span class="p">,</span> |
| <span class="s2">".key/mysql-client-key.pem"</span><span class="p">)</span> |
| <span class="n">GCSQL_MYSQL_SERVER_CA_FILE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCSQL_MYSQL_SERVER_CA_FILE'</span><span class="p">,</span> |
| <span class="s2">".key/mysql-server-ca.pem"</span><span class="p">)</span> |
| |
| <span class="n">SQL</span> <span class="o">=</span> <span class="p">[</span> |
| <span class="s1">'CREATE TABLE IF NOT EXISTS TABLE_TEST (I INTEGER)'</span><span class="p">,</span> |
| <span class="s1">'CREATE TABLE IF NOT EXISTS TABLE_TEST (I INTEGER)'</span><span class="p">,</span> <span class="c1"># shows warnings logged</span> |
| <span class="s1">'INSERT INTO TABLE_TEST VALUES (0)'</span><span class="p">,</span> |
| <span class="s1">'CREATE TABLE IF NOT EXISTS TABLE_TEST2 (I INTEGER)'</span><span class="p">,</span> |
| <span class="s1">'DROP TABLE TABLE_TEST'</span><span class="p">,</span> |
| <span class="s1">'DROP TABLE TABLE_TEST2'</span><span class="p">,</span> |
| <span class="p">]</span> |
| |
| </pre></div> |
| </div> |
| <p>Example connection definitions for all connectivity cases. Note that all the components |
| of the connection URI should be URL-encoded:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span> |
| <span class="n">HOME_DIR</span> <span class="o">=</span> <span class="n">expanduser</span><span class="p">(</span><span class="s2">"~"</span><span class="p">)</span> |
| |
| |
| <span class="k">def</span> <span class="nf">get_absolute_path</span><span class="p">(</span><span class="n">path</span><span class="p">):</span> |
| <span class="k">if</span> <span class="n">path</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"/"</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">path</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">HOME_DIR</span><span class="p">,</span> <span class="n">path</span><span class="p">)</span> |
| |
| |
| <span class="n">postgres_kwargs</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span> |
| <span class="n">user</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCSQL_POSTGRES_USER</span><span class="p">),</span> |
| <span class="n">password</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCSQL_POSTGRES_PASSWORD</span><span class="p">),</span> |
| <span class="n">public_port</span><span class="o">=</span><span class="n">GCSQL_POSTGRES_PUBLIC_PORT</span><span class="p">,</span> |
| <span class="n">public_ip</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCSQL_POSTGRES_PUBLIC_IP</span><span class="p">),</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCP_PROJECT_ID</span><span class="p">),</span> |
| <span class="n">location</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCP_REGION</span><span class="p">),</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCSQL_POSTGRES_INSTANCE_NAME_QUERY</span><span class="p">),</span> |
| <span class="n">database</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCSQL_POSTGRES_DATABASE_NAME</span><span class="p">),</span> |
| <span class="n">client_cert_file</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">get_absolute_path</span><span class="p">(</span><span class="n">GCSQL_POSTGRES_CLIENT_CERT_FILE</span><span class="p">)),</span> |
| <span class="n">client_key_file</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">get_absolute_path</span><span class="p">(</span><span class="n">GCSQL_POSTGRES_CLIENT_KEY_FILE</span><span class="p">)),</span> |
| <span class="n">server_ca_file</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">get_absolute_path</span><span class="p">(</span><span class="n">GCSQL_POSTGRES_SERVER_CA_FILE</span><span class="p">))</span> |
| <span class="p">)</span> |
| |
| <span class="c1"># The connections below are created using one of the standard approaches - via environment</span> |
| <span class="c1"># variables named AIRFLOW_CONN_* . The connections can also be created in the database</span> |
| <span class="c1"># of AIRFLOW (using command line or UI).</span> |
| |
| <span class="c1"># Postgres: connect via proxy over TCP</span> |
| <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'AIRFLOW_CONN_PROXY_POSTGRES_TCP'</span><span class="p">]</span> <span class="o">=</span> \ |
| <span class="s2">"gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?"</span> \ |
| <span class="s2">"database_type=postgres&"</span> \ |
| <span class="s2">"project_id={project_id}&"</span> \ |
| <span class="s2">"location={location}&"</span> \ |
| <span class="s2">"instance={instance}&"</span> \ |
| <span class="s2">"use_proxy=True&"</span> \ |
| <span class="s2">"sql_proxy_use_tcp=True"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">postgres_kwargs</span><span class="p">)</span> |
| |
| <span class="c1"># Postgres: connect via proxy over UNIX socket (specific proxy version)</span> |
| <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'AIRFLOW_CONN_PROXY_POSTGRES_SOCKET'</span><span class="p">]</span> <span class="o">=</span> \ |
| <span class="s2">"gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?"</span> \ |
| <span class="s2">"database_type=postgres&"</span> \ |
| <span class="s2">"project_id={project_id}&"</span> \ |
| <span class="s2">"location={location}&"</span> \ |
| <span class="s2">"instance={instance}&"</span> \ |
| <span class="s2">"use_proxy=True&"</span> \ |
| <span class="s2">"sql_proxy_version=v1.13&"</span> \ |
| <span class="s2">"sql_proxy_use_tcp=False"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">postgres_kwargs</span><span class="p">)</span> |
| |
| <span class="c1"># Postgres: connect directly via TCP (non-SSL)</span> |
| <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'AIRFLOW_CONN_PUBLIC_POSTGRES_TCP'</span><span class="p">]</span> <span class="o">=</span> \ |
| <span class="s2">"gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?"</span> \ |
| <span class="s2">"database_type=postgres&"</span> \ |
| <span class="s2">"project_id={project_id}&"</span> \ |
| <span class="s2">"location={location}&"</span> \ |
| <span class="s2">"instance={instance}&"</span> \ |
| <span class="s2">"use_proxy=False&"</span> \ |
| <span class="s2">"use_ssl=False"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">postgres_kwargs</span><span class="p">)</span> |
| |
| <span class="c1"># Postgres: connect directly via TCP (SSL)</span> |
| <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'AIRFLOW_CONN_PUBLIC_POSTGRES_TCP_SSL'</span><span class="p">]</span> <span class="o">=</span> \ |
| <span class="s2">"gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?"</span> \ |
| <span class="s2">"database_type=postgres&"</span> \ |
| <span class="s2">"project_id={project_id}&"</span> \ |
| <span class="s2">"location={location}&"</span> \ |
| <span class="s2">"instance={instance}&"</span> \ |
| <span class="s2">"use_proxy=False&"</span> \ |
| <span class="s2">"use_ssl=True&"</span> \ |
| <span class="s2">"sslcert={client_cert_file}&"</span> \ |
| <span class="s2">"sslkey={client_key_file}&"</span> \ |
| <span class="s2">"sslrootcert={server_ca_file}"</span>\ |
| <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">postgres_kwargs</span><span class="p">)</span> |
| |
| <span class="n">mysql_kwargs</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span> |
| <span class="n">user</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCSQL_MYSQL_USER</span><span class="p">),</span> |
| <span class="n">password</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCSQL_MYSQL_PASSWORD</span><span class="p">),</span> |
| <span class="n">public_port</span><span class="o">=</span><span class="n">GCSQL_MYSQL_PUBLIC_PORT</span><span class="p">,</span> |
| <span class="n">public_ip</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCSQL_MYSQL_PUBLIC_IP</span><span class="p">),</span> |
| <span class="n">project_id</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCP_PROJECT_ID</span><span class="p">),</span> |
| <span class="n">location</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCP_REGION</span><span class="p">),</span> |
| <span class="n">instance</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCSQL_MYSQL_INSTANCE_NAME_QUERY</span><span class="p">),</span> |
| <span class="n">database</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">GCSQL_MYSQL_DATABASE_NAME</span><span class="p">),</span> |
| <span class="n">client_cert_file</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">get_absolute_path</span><span class="p">(</span><span class="n">GCSQL_MYSQL_CLIENT_CERT_FILE</span><span class="p">)),</span> |
| <span class="n">client_key_file</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">get_absolute_path</span><span class="p">(</span><span class="n">GCSQL_MYSQL_CLIENT_KEY_FILE</span><span class="p">)),</span> |
| <span class="n">server_ca_file</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">get_absolute_path</span><span class="p">(</span><span class="n">GCSQL_MYSQL_SERVER_CA_FILE</span><span class="p">))</span> |
| <span class="p">)</span> |
| |
| <span class="c1"># MySQL: connect via proxy over TCP (specific proxy version)</span> |
| <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'AIRFLOW_CONN_PROXY_MYSQL_TCP'</span><span class="p">]</span> <span class="o">=</span> \ |
| <span class="s2">"gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?"</span> \ |
| <span class="s2">"database_type=mysql&"</span> \ |
| <span class="s2">"project_id={project_id}&"</span> \ |
| <span class="s2">"location={location}&"</span> \ |
| <span class="s2">"instance={instance}&"</span> \ |
| <span class="s2">"use_proxy=True&"</span> \ |
| <span class="s2">"sql_proxy_version=v1.13&"</span> \ |
| <span class="s2">"sql_proxy_use_tcp=True"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">mysql_kwargs</span><span class="p">)</span> |
| |
| <span class="c1"># MySQL: connect via proxy over UNIX socket using pre-downloaded Cloud Sql Proxy binary</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="n">sql_proxy_binary_path</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">check_output</span><span class="p">(</span> |
| <span class="p">[</span><span class="s1">'which'</span><span class="p">,</span> <span class="s1">'cloud_sql_proxy'</span><span class="p">])</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">()</span> |
| <span class="k">except</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">CalledProcessError</span><span class="p">:</span> |
| <span class="n">sql_proxy_binary_path</span> <span class="o">=</span> <span class="s2">"/tmp/anyhow_download_cloud_sql_proxy"</span> |
| |
| <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'AIRFLOW_CONN_PROXY_MYSQL_SOCKET'</span><span class="p">]</span> <span class="o">=</span> \ |
| <span class="s2">"gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?"</span> \ |
| <span class="s2">"database_type=mysql&"</span> \ |
| <span class="s2">"project_id={project_id}&"</span> \ |
| <span class="s2">"location={location}&"</span> \ |
| <span class="s2">"instance={instance}&"</span> \ |
| <span class="s2">"use_proxy=True&"</span> \ |
| <span class="s2">"sql_proxy_binary_path={sql_proxy_binary_path}&"</span> \ |
| <span class="s2">"sql_proxy_use_tcp=False"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> |
| <span class="n">sql_proxy_binary_path</span><span class="o">=</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">sql_proxy_binary_path</span><span class="p">),</span> <span class="o">**</span><span class="n">mysql_kwargs</span><span class="p">)</span> |
| |
| <span class="c1"># MySQL: connect directly via TCP (non-SSL)</span> |
| <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'AIRFLOW_CONN_PUBLIC_MYSQL_TCP'</span><span class="p">]</span> <span class="o">=</span> \ |
| <span class="s2">"gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?"</span> \ |
| <span class="s2">"database_type=mysql&"</span> \ |
| <span class="s2">"project_id={project_id}&"</span> \ |
| <span class="s2">"location={location}&"</span> \ |
| <span class="s2">"instance={instance}&"</span> \ |
| <span class="s2">"use_proxy=False&"</span> \ |
| <span class="s2">"use_ssl=False"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">mysql_kwargs</span><span class="p">)</span> |
| |
| <span class="c1"># MySQL: connect directly via TCP (SSL) and with fixed Cloud Sql Proxy binary path</span> |
| <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'AIRFLOW_CONN_PUBLIC_MYSQL_TCP_SSL'</span><span class="p">]</span> <span class="o">=</span> \ |
| <span class="s2">"gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?"</span> \ |
| <span class="s2">"database_type=mysql&"</span> \ |
| <span class="s2">"project_id={project_id}&"</span> \ |
| <span class="s2">"location={location}&"</span> \ |
| <span class="s2">"instance={instance}&"</span> \ |
| <span class="s2">"use_proxy=False&"</span> \ |
| <span class="s2">"use_ssl=True&"</span> \ |
| <span class="s2">"sslcert={client_cert_file}&"</span> \ |
| <span class="s2">"sslkey={client_key_file}&"</span> \ |
| <span class="s2">"sslrootcert={server_ca_file}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">mysql_kwargs</span><span class="p">)</span> |
| |
| <span class="c1"># Special case: MySQL: connect directly via TCP (SSL) and with fixed Cloud Sql</span> |
| <span class="c1"># Proxy binary path AND with missing project_id</span> |
| |
| <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'AIRFLOW_CONN_PUBLIC_MYSQL_TCP_SSL_NO_PROJECT_ID'</span><span class="p">]</span> <span class="o">=</span> \ |
| <span class="s2">"gcpcloudsql://{user}:{password}@{public_ip}:{public_port}/{database}?"</span> \ |
| <span class="s2">"database_type=mysql&"</span> \ |
| <span class="s2">"location={location}&"</span> \ |
| <span class="s2">"instance={instance}&"</span> \ |
| <span class="s2">"use_proxy=False&"</span> \ |
| <span class="s2">"use_ssl=True&"</span> \ |
| <span class="s2">"sslcert={client_cert_file}&"</span> \ |
| <span class="s2">"sslkey={client_key_file}&"</span> \ |
| <span class="s2">"sslrootcert={server_ca_file}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">mysql_kwargs</span><span class="p">)</span> |
| |
| |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id97"> |
| <h4><a class="toc-backref" href="#id244">Using the operator</a><a class="headerlink" href="#id97" title="Permalink to this headline">¶</a></h4> |
| <p>Example operators below are using all connectivity options. Note connection id |
| from the operator matches the <cite>AIRFLOW_CONN_*</cite> postfix uppercase. This is |
| standard AIRFLOW notation for defining connection via environment variables):</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span> |
| <span class="n">connection_names</span> <span class="o">=</span> <span class="p">[</span> |
| <span class="s2">"proxy_postgres_tcp"</span><span class="p">,</span> |
| <span class="s2">"proxy_postgres_socket"</span><span class="p">,</span> |
| <span class="s2">"public_postgres_tcp"</span><span class="p">,</span> |
| <span class="s2">"public_postgres_tcp_ssl"</span><span class="p">,</span> |
| <span class="s2">"proxy_mysql_tcp"</span><span class="p">,</span> |
| <span class="s2">"proxy_mysql_socket"</span><span class="p">,</span> |
| <span class="s2">"public_mysql_tcp"</span><span class="p">,</span> |
| <span class="s2">"public_mysql_tcp_ssl"</span><span class="p">,</span> |
| <span class="s2">"public_mysql_tcp_ssl_no_project_id"</span> |
| <span class="p">]</span> |
| |
| <span class="n">tasks</span> <span class="o">=</span> <span class="p">[]</span> |
| |
| |
| <span class="k">with</span> <span class="n">models</span><span class="o">.</span><span class="n">DAG</span><span class="p">(</span> |
| <span class="n">dag_id</span><span class="o">=</span><span class="s1">'example_gcp_sql_query'</span><span class="p">,</span> |
| <span class="n">default_args</span><span class="o">=</span><span class="n">default_args</span><span class="p">,</span> |
| <span class="n">schedule_interval</span><span class="o">=</span><span class="bp">None</span> |
| <span class="p">)</span> <span class="k">as</span> <span class="n">dag</span><span class="p">:</span> |
| <span class="n">prev_task</span> <span class="o">=</span> <span class="bp">None</span> |
| |
| <span class="k">for</span> <span class="n">connection_name</span> <span class="ow">in</span> <span class="n">connection_names</span><span class="p">:</span> |
| <span class="n">task</span> <span class="o">=</span> <span class="n">CloudSqlQueryOperator</span><span class="p">(</span> |
| <span class="n">gcp_cloudsql_conn_id</span><span class="o">=</span><span class="n">connection_name</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s2">"example_gcp_sql_task_"</span> <span class="o">+</span> <span class="n">connection_name</span><span class="p">,</span> |
| <span class="n">sql</span><span class="o">=</span><span class="n">SQL</span> |
| <span class="p">)</span> |
| <span class="n">tasks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">task</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">prev_task</span><span class="p">:</span> |
| <span class="n">prev_task</span> <span class="o">>></span> <span class="n">task</span> |
| <span class="n">prev_task</span> <span class="o">=</span> <span class="n">task</span> |
| |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id98"> |
| <h4><a class="toc-backref" href="#id245">Templating</a><a class="headerlink" href="#id98" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'gcp_cloudsql_conn_id'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">)</span> |
| <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id99"> |
| <h4><a class="toc-backref" href="#id246">More information</a><a class="headerlink" href="#id99" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/sql/docs/postgres/sql-proxy">Google Cloud SQL Proxy documentation</a>.</p> |
| </div> |
| </div> |
| </div> |
| <div class="section" id="id100"> |
| <h2><a class="toc-backref" href="#id247">Google Cloud Storage Operators</a><a class="headerlink" href="#id100" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="googlecloudstoragebucketcreateaclentryoperator"> |
| <h3><a class="toc-backref" href="#id248">GoogleCloudStorageBucketCreateAclEntryOperator</a><a class="headerlink" href="#googlecloudstoragebucketcreateaclentryoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Creates a new ACL entry on the specified bucket.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageBucketCreateAclEntryOperator" title="airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageBucketCreateAclEntryOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GoogleCloudStorageBucketCreateAclEntryOperator</span></code></a></p> |
| <div class="section" id="id101"> |
| <h4><a class="toc-backref" href="#id249">Arguments</a><a class="headerlink" href="#id101" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from the OS environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCS_ACL_BUCKET</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCS_ACL_BUCKET'</span><span class="p">,</span> <span class="s1">'example-bucket'</span><span class="p">)</span> |
| <span class="n">GCS_ACL_OBJECT</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCS_ACL_OBJECT'</span><span class="p">,</span> <span class="s1">'example-object'</span><span class="p">)</span> |
| <span class="n">GCS_ACL_ENTITY</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCS_ACL_ENTITY'</span><span class="p">,</span> <span class="s1">'example-entity'</span><span class="p">)</span> |
| <span class="n">GCS_ACL_BUCKET_ROLE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCS_ACL_BUCKET_ROLE'</span><span class="p">,</span> <span class="s1">'example-bucket-role'</span><span class="p">)</span> |
| <span class="n">GCS_ACL_OBJECT_ROLE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCS_ACL_OBJECT_ROLE'</span><span class="p">,</span> <span class="s1">'example-object-role'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id102"> |
| <h4><a class="toc-backref" href="#id250">Using the operator</a><a class="headerlink" href="#id102" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gcs_bucket_create_acl_entry_task</span> <span class="o">=</span> <span class="n">GoogleCloudStorageBucketCreateAclEntryOperator</span><span class="p">(</span> |
| <span class="n">bucket</span><span class="o">=</span><span class="n">GCS_ACL_BUCKET</span><span class="p">,</span> |
| <span class="n">entity</span><span class="o">=</span><span class="n">GCS_ACL_ENTITY</span><span class="p">,</span> |
| <span class="n">role</span><span class="o">=</span><span class="n">GCS_ACL_BUCKET_ROLE</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s2">"gcs_bucket_create_acl_entry_task"</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id103"> |
| <h4><a class="toc-backref" href="#id251">Templating</a><a class="headerlink" href="#id103" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'entity'</span><span class="p">,</span> <span class="s1">'role'</span><span class="p">,</span> <span class="s1">'user_project'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id104"> |
| <h4><a class="toc-backref" href="#id252">More information</a><a class="headerlink" href="#id104" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/insert">Google Cloud Storage BucketAccessControls insert documentation</a>.</p> |
| </div> |
| </div> |
| <div class="section" id="googlecloudstorageobjectcreateaclentryoperator"> |
| <h3><a class="toc-backref" href="#id253">GoogleCloudStorageObjectCreateAclEntryOperator</a><a class="headerlink" href="#googlecloudstorageobjectcreateaclentryoperator" title="Permalink to this headline">¶</a></h3> |
| <p>Creates a new ACL entry on the specified object.</p> |
| <p>For parameter definition, take a look at |
| <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageObjectCreateAclEntryOperator" title="airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageObjectCreateAclEntryOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GoogleCloudStorageObjectCreateAclEntryOperator</span></code></a></p> |
| <div class="section" id="id105"> |
| <h4><a class="toc-backref" href="#id254">Arguments</a><a class="headerlink" href="#id105" title="Permalink to this headline">¶</a></h4> |
| <p>Some arguments in the example DAG are taken from the OS environment variables:</p> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCS_ACL_BUCKET</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCS_ACL_BUCKET'</span><span class="p">,</span> <span class="s1">'example-bucket'</span><span class="p">)</span> |
| <span class="n">GCS_ACL_OBJECT</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCS_ACL_OBJECT'</span><span class="p">,</span> <span class="s1">'example-object'</span><span class="p">)</span> |
| <span class="n">GCS_ACL_ENTITY</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCS_ACL_ENTITY'</span><span class="p">,</span> <span class="s1">'example-entity'</span><span class="p">)</span> |
| <span class="n">GCS_ACL_BUCKET_ROLE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCS_ACL_BUCKET_ROLE'</span><span class="p">,</span> <span class="s1">'example-bucket-role'</span><span class="p">)</span> |
| <span class="n">GCS_ACL_OBJECT_ROLE</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'GCS_ACL_OBJECT_ROLE'</span><span class="p">,</span> <span class="s1">'example-object-role'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id106"> |
| <h4><a class="toc-backref" href="#id255">Using the operator</a><a class="headerlink" href="#id106" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gcs_object_create_acl_entry_task</span> <span class="o">=</span> <span class="n">GoogleCloudStorageObjectCreateAclEntryOperator</span><span class="p">(</span> |
| <span class="n">bucket</span><span class="o">=</span><span class="n">GCS_ACL_BUCKET</span><span class="p">,</span> |
| <span class="n">object_name</span><span class="o">=</span><span class="n">GCS_ACL_OBJECT</span><span class="p">,</span> |
| <span class="n">entity</span><span class="o">=</span><span class="n">GCS_ACL_ENTITY</span><span class="p">,</span> |
| <span class="n">role</span><span class="o">=</span><span class="n">GCS_ACL_OBJECT_ROLE</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="o">=</span><span class="s2">"gcs_object_create_acl_entry_task"</span> |
| <span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id107"> |
| <h4><a class="toc-backref" href="#id256">Templating</a><a class="headerlink" href="#id107" title="Permalink to this headline">¶</a></h4> |
| <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span> <span class="s1">'object_name'</span><span class="p">,</span> <span class="s1">'entity'</span><span class="p">,</span> <span class="s1">'role'</span><span class="p">,</span> <span class="s1">'generation'</span><span class="p">,</span> |
| <span class="s1">'user_project'</span><span class="p">)</span> |
| </pre></div> |
| </div> |
| </div> |
| <div class="section" id="id108"> |
| <h4><a class="toc-backref" href="#id257">More information</a><a class="headerlink" href="#id108" title="Permalink to this headline">¶</a></h4> |
| <p>See <a class="reference external" href="https://cloud.google.com/storage/docs/json_api/v1/objectAccessControls/insert">Google Cloud Storage ObjectAccessControls insert documentation</a></p> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| </div> |
| <footer> |
| |
| <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> |
| |
| <a href="manage-connections.html" class="btn btn-neutral float-right" title="Managing Connections" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> |
| |
| |
| <a href="initialize-database.html" class="btn btn-neutral" title="Initializing a Database Backend" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> |
| |
| </div> |
| |
| |
| <hr/> |
| |
| <div role="contentinfo"> |
| <p> |
| |
| </p> |
| </div> |
| Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. |
| |
| </footer> |
| |
| </div> |
| </div> |
| |
| </section> |
| |
| </div> |
| |
| |
| |
| |
| |
| |
| |
| <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> |
| <script type="text/javascript" src="../_static/jquery.js"></script> |
| <script type="text/javascript" src="../_static/underscore.js"></script> |
| <script type="text/javascript" src="../_static/doctools.js"></script> |
| <script type="text/javascript" src="../_static/language_data.js"></script> |
| |
| |
| |
| |
| <script type="text/javascript" src="../_static/js/theme.js"></script> |
| |
| <script type="text/javascript"> |
| jQuery(function () { |
| SphinxRtdTheme.Navigation.enable(true); |
| }); |
| </script> |
| |
| </body> |
| </html> |
