Google Git
Sign in
apache / airflow-site / f81e48aadee354653679693fa54b04ac42d24bfd / . / docs-archive / 1.10.5 / howto / operator / gcp / bigtable.html
blob: 3e2da4de74ab9bf7418861edb7c94bf1e7a9c9b3 [file] [log] [blame]
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Google Cloud Bigtable Operators &mdash; Airflow Documentation</title>
<script type="text/javascript" src="../../../_static/js/modernizr.min.js"></script>
<script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
<script type="text/javascript" src="../../../_static/jquery.js"></script>
<script type="text/javascript" src="../../../_static/underscore.js"></script>
<script type="text/javascript" src="../../../_static/doctools.js"></script>
<script type="text/javascript" src="../../../_static/language_data.js"></script>
<script type="text/javascript" src="../../../_static/js/theme.js"></script>
<link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
<link rel="next" title="Google Cloud Build Operators" href="cloud_build.html" />
<link rel="prev" title="Google Cloud Operators" href="index.html" />
<script>
document.addEventListener('DOMContentLoaded', function() {
var el = document.getElementById('changelog');
if (el !== null ) {
// [AIRFLOW-...]
el.innerHTML = el.innerHTML.replace(
/\[(AIRFLOW-[\d]+)\]/g,
`<a href="https://issues.apache.org/jira/browse/$1">[$1]</a>`
);
// (#...)
el.innerHTML = el.innerHTML.replace(
/\(#([\d]+)\)/g,
`<a href="https://github.com/apache/airflow/pull/$1">(#$1)</a>`
);
};
})
</script>
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-140539454-1']);
_gaq.push(['_trackPageview']);
</script>
<style>
.example-header {
position: relative;
background: #9AAA7A;
padding: 8px 16px;
margin-bottom: 0;
}
.example-header--with-button {
padding-right: 166px;
}
.example-header:after{
content: '';
display: table;
clear: both;
}
.example-title {
display:block;
padding: 4px;
margin-right: 16px;
color: white;
overflow-x: auto;
}
.example-header-button {
top: 8px;
right: 16px;
position: absolute;
}
.example-header + .highlight-python {
margin-top: 0 !important;
}
.viewcode-button {
display: inline-block;
padding: 8px 16px;
border: 0;
margin: 0;
outline: 0;
border-radius: 2px;
-webkit-box-shadow: 0 3px 5px 0 rgba(0,0,0,.3);
box-shadow: 0 3px 6px 0 rgba(0,0,0,.3);
color: #404040;
background-color: #e7e7e7;
cursor: pointer;
font-size: 16px;
font-weight: 500;
line-height: 1;
text-decoration: none;
text-overflow: ellipsis;
overflow: hidden;
text-transform: uppercase;
-webkit-transition: background-color .2s;
transition: background-color .2s;
vertical-align: middle;
white-space: nowrap;
}
.viewcode-button:visited {
color: #404040;
}
.viewcode-button:hover, .viewcode-button:focus {
color: #404040;
background-color: #d6d6d6;
}
</style>
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="../../../index.html" class="icon icon-home"> Airflow
</a>
<div class="version">
1.10.5
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../../index.html">How-to Guides</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../../set-config.html">Setting Configuration Options</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../initialize-database.html">Initializing a Database Backend</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="../index.html">Using Operators</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="../bash.html">BashOperator</a></li>
<li class="toctree-l3"><a class="reference internal" href="../dingding.html">Dingding Operators</a></li>
<li class="toctree-l3 current"><a class="reference internal" href="index.html">Google Cloud Operators</a><ul class="current">
<li class="toctree-l4 current"><a class="current reference internal" href="#">Google Cloud Bigtable Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="cloud_build.html">Google Cloud Build Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="compute.html">Google Compute Engine Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="function.html">Google Cloud Functions Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="gcs.html">Google Cloud Storage Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="natural_language.html">Google Cloud Natural Language Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="spanner.html">Google Cloud Spanner Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="speech.html">Google Cloud Text to Speech Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="speech.html#google-cloud-speech-to-text-operators">Google Cloud Speech to Text Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="sql.html">Google Cloud Sql Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="transfer.html">Google Cloud Transfer Service Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="translate.html">Google Cloud Translate Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="translate-speech.html">Google Cloud Speech Translate Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="video.html">Google Cloud Video Intelligence Operators</a></li>
<li class="toctree-l4"><a class="reference internal" href="vision.html">Google Cloud Vision Operators</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../python.html">PythonOperator</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../connection/index.html">Managing Connections</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../secure-connections.html">Securing Connections</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../secure-connections.html#rotating-encryption-keys">Rotating encryption keys</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../write-logs.html">Writing Logs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../executor/use-celery.html">Celery Executor</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../executor/use-dask.html">Dask Executor</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../executor/use-mesos.html">Scaling Out with Mesos (community contributed)</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../run-behind-proxy.html">Running Airflow behind a reverse proxy</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../run-with-systemd.html">Running Airflow with systemd</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../run-with-upstart.html">Running Airflow with upstart</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../use-test-config.html">Using the Test Mode Configuration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../check-health.html">Checking Airflow Health Status</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../define_extra_link.html">Define an operator extra link</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../tracking-user-activity.html">Tracking User Activity</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface Reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling &amp; Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../api.html">REST API Reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../metrics.html">Metrics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../kubernetes.html">Kubernetes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../changelog.html">Changelog</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../macros.html">Macros reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../_api/index.html">Python API Reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../index.html">Airflow</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../../index.html">Docs</a> &raquo;</li>
<li><a href="../../index.html">How-to Guides</a> &raquo;</li>
<li><a href="../index.html">Using Operators</a> &raquo;</li>
<li><a href="index.html">Google Cloud Operators</a> &raquo;</li>
<li>Google Cloud Bigtable Operators</li>
<li class="wy-breadcrumbs-aside">
<a href="../../../_sources/howto/operator/gcp/bigtable.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<blockquote>
<div></div></blockquote>
<div class="section" id="google-cloud-bigtable-operators">
<h1>Google Cloud Bigtable Operators<a class="headerlink" href="#google-cloud-bigtable-operators" title="Permalink to this headline"></a></h1>
<div class="contents local topic" id="contents">
<ul class="simple">
<li><p><a class="reference internal" href="#bigtableinstancecreateoperator" id="id6">BigtableInstanceCreateOperator</a></p></li>
<li><p><a class="reference internal" href="#bigtableinstancedeleteoperator" id="id7">BigtableInstanceDeleteOperator</a></p></li>
<li><p><a class="reference internal" href="#bigtableclusterupdateoperator" id="id8">BigtableClusterUpdateOperator</a></p></li>
<li><p><a class="reference internal" href="#bigtabletabledeleteoperator" id="id9">BigtableTableDeleteOperator</a></p></li>
<li><p><a class="reference internal" href="#bigtabletablewaitforreplicationsensor" id="id10">BigtableTableWaitForReplicationSensor</a></p></li>
</ul>
</div>
<p>All examples below rely on the following variables, which can be passed via environment variables.</p>
<div class="example-block-wrapper docutils container">
<p class="example-header example-header--with-button"><span class="example-title">airflow/contrib/example_dags/example_gcp_bigtable_operators.py</span><a class="example-header-button viewcode-button reference internal" href="../../../_modules/airflow/contrib/example_dags/example_gcp_bigtable_operators.html"><span>View Source</span></a></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">GCP_PROJECT_ID</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">&#39;GCP_PROJECT_ID&#39;</span><span class="p">,</span> <span class="s1">&#39;example-project&#39;</span><span class="p">)</span>
<span class="n">CBT_INSTANCE_ID</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">&#39;CBT_INSTANCE_ID&#39;</span><span class="p">,</span> <span class="s1">&#39;some-instance-id&#39;</span><span class="p">)</span>
<span class="n">CBT_INSTANCE_DISPLAY_NAME</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">&#39;CBT_INSTANCE_DISPLAY_NAME&#39;</span><span class="p">,</span> <span class="s1">&#39;Human-readable name&#39;</span><span class="p">)</span>
<span class="n">CBT_INSTANCE_TYPE</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">&#39;CBT_INSTANCE_TYPE&#39;</span><span class="p">,</span> <span class="s1">&#39;2&#39;</span><span class="p">)</span>
<span class="n">CBT_INSTANCE_LABELS</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">&#39;CBT_INSTANCE_LABELS&#39;</span><span class="p">,</span> <span class="s1">&#39;{}&#39;</span><span class="p">)</span>
<span class="n">CBT_CLUSTER_ID</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">&#39;CBT_CLUSTER_ID&#39;</span><span class="p">,</span> <span class="s1">&#39;some-cluster-id&#39;</span><span class="p">)</span>
<span class="n">CBT_CLUSTER_ZONE</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">&#39;CBT_CLUSTER_ZONE&#39;</span><span class="p">,</span> <span class="s1">&#39;europe-west1-b&#39;</span><span class="p">)</span>
<span class="n">CBT_CLUSTER_NODES</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">&#39;CBT_CLUSTER_NODES&#39;</span><span class="p">,</span> <span class="s1">&#39;3&#39;</span><span class="p">)</span>
<span class="n">CBT_CLUSTER_NODES_UPDATED</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">&#39;CBT_CLUSTER_NODES_UPDATED&#39;</span><span class="p">,</span> <span class="s1">&#39;5&#39;</span><span class="p">)</span>
<span class="n">CBT_CLUSTER_STORAGE_TYPE</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">&#39;CBT_CLUSTER_STORAGE_TYPE&#39;</span><span class="p">,</span> <span class="s1">&#39;2&#39;</span><span class="p">)</span>
<span class="n">CBT_TABLE_ID</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">&#39;CBT_TABLE_ID&#39;</span><span class="p">,</span> <span class="s1">&#39;some-table-id&#39;</span><span class="p">)</span>
<span class="n">CBT_POKE_INTERVAL</span> <span class="o">=</span> <span class="n">getenv</span><span class="p">(</span><span class="s1">&#39;CBT_POKE_INTERVAL&#39;</span><span class="p">,</span> <span class="s1">&#39;60&#39;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="bigtableinstancecreateoperator">
<span id="howto-operator-bigtableinstancecreateoperator"></span><h2><a class="toc-backref" href="#id6">BigtableInstanceCreateOperator</a><a class="headerlink" href="#bigtableinstancecreateoperator" title="Permalink to this headline"></a></h2>
<p>Use the <a class="reference internal" href="../../../_api/airflow/contrib/operators/gcp_bigtable_operator/index.html#airflow.contrib.operators.gcp_bigtable_operator.BigtableInstanceCreateOperator" title="airflow.contrib.operators.gcp_bigtable_operator.BigtableInstanceCreateOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">BigtableInstanceCreateOperator</span></code></a>
to create a Google Cloud Bigtable instance.</p>
<p>If the Cloud Bigtable instance with the given ID exists, the operator does not compare its configuration
and immediately succeeds. No changes are made to the existing instance.</p>
<div class="section" id="using-the-operator">
<h3>Using the operator<a class="headerlink" href="#using-the-operator" title="Permalink to this headline"></a></h3>
<p>You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:</p>
<div class="example-block-wrapper docutils container">
<p class="example-header example-header--with-button"><span class="example-title">airflow/contrib/example_dags/example_gcp_bigtable_operators.py</span><a class="example-header-button viewcode-button reference internal" href="../../../_modules/airflow/contrib/example_dags/example_gcp_bigtable_operators.html"><span>View Source</span></a></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">create_instance_task</span> <span class="o">=</span> <span class="n">BigtableInstanceCreateOperator</span><span class="p">(</span>
<span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span>
<span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span>
<span class="n">main_cluster_id</span><span class="o">=</span><span class="n">CBT_CLUSTER_ID</span><span class="p">,</span>
<span class="n">main_cluster_zone</span><span class="o">=</span><span class="n">CBT_CLUSTER_ZONE</span><span class="p">,</span>
<span class="n">instance_display_name</span><span class="o">=</span><span class="n">CBT_INSTANCE_DISPLAY_NAME</span><span class="p">,</span>
<span class="n">instance_type</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_INSTANCE_TYPE</span><span class="p">),</span>
<span class="n">instance_labels</span><span class="o">=</span><span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">CBT_INSTANCE_LABELS</span><span class="p">),</span>
<span class="n">cluster_nodes</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_CLUSTER_NODES</span><span class="p">),</span>
<span class="n">cluster_storage_type</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_CLUSTER_STORAGE_TYPE</span><span class="p">),</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;create_instance_task&#39;</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">create_instance_task2</span> <span class="o">=</span> <span class="n">BigtableInstanceCreateOperator</span><span class="p">(</span>
<span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span>
<span class="n">main_cluster_id</span><span class="o">=</span><span class="n">CBT_CLUSTER_ID</span><span class="p">,</span>
<span class="n">main_cluster_zone</span><span class="o">=</span><span class="n">CBT_CLUSTER_ZONE</span><span class="p">,</span>
<span class="n">instance_display_name</span><span class="o">=</span><span class="n">CBT_INSTANCE_DISPLAY_NAME</span><span class="p">,</span>
<span class="n">instance_type</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_INSTANCE_TYPE</span><span class="p">),</span>
<span class="n">instance_labels</span><span class="o">=</span><span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">CBT_INSTANCE_LABELS</span><span class="p">),</span>
<span class="n">cluster_nodes</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_CLUSTER_NODES</span><span class="p">),</span>
<span class="n">cluster_storage_type</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_CLUSTER_STORAGE_TYPE</span><span class="p">),</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;create_instance_task2&#39;</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">create_instance_task</span> <span class="o">&gt;&gt;</span> <span class="n">create_instance_task2</span>
</pre></div>
</div>
</div>
</div>
</div>
<div class="section" id="bigtableinstancedeleteoperator">
<span id="howto-operator-bigtableinstancedeleteoperator"></span><h2><a class="toc-backref" href="#id7">BigtableInstanceDeleteOperator</a><a class="headerlink" href="#bigtableinstancedeleteoperator" title="Permalink to this headline"></a></h2>
<p>Use the <a class="reference internal" href="../../../_api/airflow/contrib/operators/gcp_bigtable_operator/index.html#airflow.contrib.operators.gcp_bigtable_operator.BigtableInstanceDeleteOperator" title="airflow.contrib.operators.gcp_bigtable_operator.BigtableInstanceDeleteOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">BigtableInstanceDeleteOperator</span></code></a>
to delete a Google Cloud Bigtable instance.</p>
<div class="section" id="id1">
<h3>Using the operator<a class="headerlink" href="#id1" title="Permalink to this headline"></a></h3>
<p>You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:</p>
<div class="example-block-wrapper docutils container">
<p class="example-header example-header--with-button"><span class="example-title">airflow/contrib/example_dags/example_gcp_bigtable_operators.py</span><a class="example-header-button viewcode-button reference internal" href="../../../_modules/airflow/contrib/example_dags/example_gcp_bigtable_operators.html"><span>View Source</span></a></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">delete_instance_task</span> <span class="o">=</span> <span class="n">BigtableInstanceDeleteOperator</span><span class="p">(</span>
<span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span>
<span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;delete_instance_task&#39;</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">delete_instance_task2</span> <span class="o">=</span> <span class="n">BigtableInstanceDeleteOperator</span><span class="p">(</span>
<span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;delete_instance_task2&#39;</span><span class="p">,</span>
<span class="p">)</span>
</pre></div>
</div>
</div>
</div>
</div>
<div class="section" id="bigtableclusterupdateoperator">
<span id="howto-operator-bigtableclusterupdateoperator"></span><h2><a class="toc-backref" href="#id8">BigtableClusterUpdateOperator</a><a class="headerlink" href="#bigtableclusterupdateoperator" title="Permalink to this headline"></a></h2>
<p>Use the <a class="reference internal" href="../../../_api/airflow/contrib/operators/gcp_bigtable_operator/index.html#airflow.contrib.operators.gcp_bigtable_operator.BigtableClusterUpdateOperator" title="airflow.contrib.operators.gcp_bigtable_operator.BigtableClusterUpdateOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">BigtableClusterUpdateOperator</span></code></a>
to modify number of nodes in a Cloud Bigtable cluster.</p>
<div class="section" id="id2">
<h3>Using the operator<a class="headerlink" href="#id2" title="Permalink to this headline"></a></h3>
<p>You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:</p>
<div class="example-block-wrapper docutils container">
<p class="example-header example-header--with-button"><span class="example-title">airflow/contrib/example_dags/example_gcp_bigtable_operators.py</span><a class="example-header-button viewcode-button reference internal" href="../../../_modules/airflow/contrib/example_dags/example_gcp_bigtable_operators.html"><span>View Source</span></a></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">cluster_update_task</span> <span class="o">=</span> <span class="n">BigtableClusterUpdateOperator</span><span class="p">(</span>
<span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span>
<span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span>
<span class="n">cluster_id</span><span class="o">=</span><span class="n">CBT_CLUSTER_ID</span><span class="p">,</span>
<span class="n">nodes</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_CLUSTER_NODES_UPDATED</span><span class="p">),</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;update_cluster_task&#39;</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">cluster_update_task2</span> <span class="o">=</span> <span class="n">BigtableClusterUpdateOperator</span><span class="p">(</span>
<span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span>
<span class="n">cluster_id</span><span class="o">=</span><span class="n">CBT_CLUSTER_ID</span><span class="p">,</span>
<span class="n">nodes</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_CLUSTER_NODES_UPDATED</span><span class="p">),</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;update_cluster_task2&#39;</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">cluster_update_task</span> <span class="o">&gt;&gt;</span> <span class="n">cluster_update_task2</span>
</pre></div>
</div>
</div>
<div class="section" id="bigtabletablecreateoperator">
<span id="howto-operator-bigtabletablecreateoperator"></span><h4>BigtableTableCreateOperator<a class="headerlink" href="#bigtabletablecreateoperator" title="Permalink to this headline"></a></h4>
<p>Creates a table in a Cloud Bigtable instance.</p>
<p>If the table with given ID exists in the Cloud Bigtable instance, the operator compares the Column Families.
If the Column Families are identical operator succeeds. Otherwise, the operator fails with the appropriate
error message.</p>
</div>
</div>
<div class="section" id="id3">
<h3>Using the operator<a class="headerlink" href="#id3" title="Permalink to this headline"></a></h3>
<p>You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:</p>
<div class="example-block-wrapper docutils container">
<p class="example-header example-header--with-button"><span class="example-title">airflow/contrib/example_dags/example_gcp_bigtable_operators.py</span><a class="example-header-button viewcode-button reference internal" href="../../../_modules/airflow/contrib/example_dags/example_gcp_bigtable_operators.html"><span>View Source</span></a></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">create_table_task</span> <span class="o">=</span> <span class="n">BigtableTableCreateOperator</span><span class="p">(</span>
<span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span>
<span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span>
<span class="n">table_id</span><span class="o">=</span><span class="n">CBT_TABLE_ID</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;create_table&#39;</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">create_table_task2</span> <span class="o">=</span> <span class="n">BigtableTableCreateOperator</span><span class="p">(</span>
<span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span>
<span class="n">table_id</span><span class="o">=</span><span class="n">CBT_TABLE_ID</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;create_table_task2&#39;</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">create_table_task</span> <span class="o">&gt;&gt;</span> <span class="n">create_table_task2</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="advanced">
<h3>Advanced<a class="headerlink" href="#advanced" title="Permalink to this headline"></a></h3>
<p>When creating a table, you can specify the optional <code class="docutils literal notranslate"><span class="pre">initial_split_keys</span></code> and <code class="docutils literal notranslate"><span class="pre">column_families</span></code>.
Please refer to the Python Client for Google Cloud Bigtable documentation
<a class="reference external" href="https://googleapis.github.io/google-cloud-python/latest/bigtable/table.html">for Table</a> and <a class="reference external" href="https://googleapis.github.io/google-cloud-python/latest/bigtable/column-family.html">for Column
Families</a>.</p>
</div>
</div>
<div class="section" id="bigtabletabledeleteoperator">
<span id="howto-operator-bigtabletabledeleteoperator"></span><h2><a class="toc-backref" href="#id9">BigtableTableDeleteOperator</a><a class="headerlink" href="#bigtabletabledeleteoperator" title="Permalink to this headline"></a></h2>
<p>Use the <a class="reference internal" href="../../../_api/airflow/contrib/operators/gcp_bigtable_operator/index.html#airflow.contrib.operators.gcp_bigtable_operator.BigtableTableDeleteOperator" title="airflow.contrib.operators.gcp_bigtable_operator.BigtableTableDeleteOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">BigtableTableDeleteOperator</span></code></a>
to delete a table in Google Cloud Bigtable.</p>
<div class="section" id="id4">
<h3>Using the operator<a class="headerlink" href="#id4" title="Permalink to this headline"></a></h3>
<p>You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:</p>
<div class="example-block-wrapper docutils container">
<p class="example-header example-header--with-button"><span class="example-title">airflow/contrib/example_dags/example_gcp_bigtable_operators.py</span><a class="example-header-button viewcode-button reference internal" href="../../../_modules/airflow/contrib/example_dags/example_gcp_bigtable_operators.html"><span>View Source</span></a></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">delete_table_task</span> <span class="o">=</span> <span class="n">BigtableTableDeleteOperator</span><span class="p">(</span>
<span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span>
<span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span>
<span class="n">table_id</span><span class="o">=</span><span class="n">CBT_TABLE_ID</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;delete_table_task&#39;</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">delete_table_task2</span> <span class="o">=</span> <span class="n">BigtableTableDeleteOperator</span><span class="p">(</span>
<span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span>
<span class="n">table_id</span><span class="o">=</span><span class="n">CBT_TABLE_ID</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;delete_table_task2&#39;</span><span class="p">,</span>
<span class="p">)</span>
</pre></div>
</div>
</div>
</div>
</div>
<div class="section" id="bigtabletablewaitforreplicationsensor">
<span id="howto-operator-bigtabletablewaitforreplicationsensor"></span><h2><a class="toc-backref" href="#id10">BigtableTableWaitForReplicationSensor</a><a class="headerlink" href="#bigtabletablewaitforreplicationsensor" title="Permalink to this headline"></a></h2>
<p>You can create the operator with or without project id. If project id is missing
it will be retrieved from the GCP connection used. Both variants are shown:</p>
<p>Use the <a class="reference internal" href="../../../_api/airflow/contrib/operators/gcp_bigtable_operator/index.html#airflow.contrib.operators.gcp_bigtable_operator.BigtableTableWaitForReplicationSensor" title="airflow.contrib.operators.gcp_bigtable_operator.BigtableTableWaitForReplicationSensor"><code class="xref py py-class docutils literal notranslate"><span class="pre">BigtableTableWaitForReplicationSensor</span></code></a>
to wait for the table to replicate fully.</p>
<p>The same arguments apply to this sensor as the <a class="reference internal" href="#bigtabletablecreateoperator">BigtableTableCreateOperator</a>.</p>
<p><strong>Note:</strong> If the table or the Cloud Bigtable instance does not exist, this sensor waits for the table until
timeout hits and does not raise any exception.</p>
<div class="section" id="id5">
<h3>Using the operator<a class="headerlink" href="#id5" title="Permalink to this headline"></a></h3>
<div class="example-block-wrapper docutils container">
<p class="example-header example-header--with-button"><span class="example-title">airflow/contrib/example_dags/example_gcp_bigtable_operators.py</span><a class="example-header-button viewcode-button reference internal" href="../../../_modules/airflow/contrib/example_dags/example_gcp_bigtable_operators.html"><span>View Source</span></a></p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">wait_for_table_replication_task</span> <span class="o">=</span> <span class="n">BigtableTableWaitForReplicationSensor</span><span class="p">(</span>
<span class="n">project_id</span><span class="o">=</span><span class="n">GCP_PROJECT_ID</span><span class="p">,</span>
<span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span>
<span class="n">table_id</span><span class="o">=</span><span class="n">CBT_TABLE_ID</span><span class="p">,</span>
<span class="n">poke_interval</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_POKE_INTERVAL</span><span class="p">),</span>
<span class="n">timeout</span><span class="o">=</span><span class="mi">180</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;wait_for_table_replication_task&#39;</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">wait_for_table_replication_task2</span> <span class="o">=</span> <span class="n">BigtableTableWaitForReplicationSensor</span><span class="p">(</span>
<span class="n">instance_id</span><span class="o">=</span><span class="n">CBT_INSTANCE_ID</span><span class="p">,</span>
<span class="n">table_id</span><span class="o">=</span><span class="n">CBT_TABLE_ID</span><span class="p">,</span>
<span class="n">poke_interval</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">CBT_POKE_INTERVAL</span><span class="p">),</span>
<span class="n">timeout</span><span class="o">=</span><span class="mi">180</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;wait_for_table_replication_task2&#39;</span><span class="p">,</span>
<span class="p">)</span>
</pre></div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="cloud_build.html" class="btn btn-neutral float-right" title="Google Cloud Build Operators" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="index.html" class="btn btn-neutral float-left" title="Google Cloud Operators" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
<div class="footer">This page uses <a href="https://analytics.google.com/">
Google Analytics</a> to collect statistics. You can disable it by blocking
the JavaScript coming from www.google-analytics.com. Check our
<a href="../../../privacy_notice.html">Privacy Policy</a>
for more details.
<script type="text/javascript">
(function() {
var ga = document.createElement('script');
ga.src = ('https:' == document.location.protocol ?
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
ga.setAttribute('async', 'true');
var nodes = document.documentElement.childNodes;
var i = -1;
var node;
do {
i++;
node = nodes[i]
} while(node.nodeType !== Node.ELEMENT_NODE);
node.appendChild(ga);
})();
</script>
</div>
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>