Google Git
Sign in
apache / airflow-site / f81e48aadee354653679693fa54b04ac42d24bfd / . / docs-archive / 1.10.2 / _modules / airflow / contrib / operators / bigquery_operator.html
blob: 6dd00cd64793c9bcc0c13fd5842280b9165e25a3 [file] [log] [blame]
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>airflow.contrib.operators.bigquery_operator &mdash; Airflow Documentation</title>
<link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="../../../../genindex.html" />
<link rel="search" title="Search" href="../../../../search.html" />
<script src="../../../../_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search">
<a href="../../../../index.html" class="icon icon-home"> Airflow
</a>
<div class="version">
1.10.2
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../../../index.html">Docs</a> &raquo;</li>
<li><a href="../../../index.html">Module code</a> &raquo;</li>
<li>airflow.contrib.operators.bigquery_operator</li>
<li class="wy-breadcrumbs-aside">
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<h1>Source code for airflow.contrib.operators.bigquery_operator</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
<span class="c1"># distributed with this work for additional information</span>
<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
<span class="c1"># KIND, either express or implied. See the License for the</span>
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
<span class="kn">import</span> <span class="nn">json</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="k">import</span> <span class="n">BigQueryHook</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="k">import</span> <span class="n">GoogleCloudStorageHook</span><span class="p">,</span> <span class="n">_parse_gcs_url</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
<div class="viewcode-block" id="BigQueryOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Executes BigQuery SQL queries in a specific BigQuery database</span>
<span class="sd"> :param bql: (Deprecated. Use `sql` parameter instead) the sql code to be</span>
<span class="sd"> executed (templated)</span>
<span class="sd"> :type bql: Can receive a str representing a sql statement,</span>
<span class="sd"> a list of str (sql statements), or reference to a template file.</span>
<span class="sd"> Template reference are recognized by str ending in &#39;.sql&#39;.</span>
<span class="sd"> :param sql: the sql code to be executed (templated)</span>
<span class="sd"> :type sql: Can receive a str representing a sql statement,</span>
<span class="sd"> a list of str (sql statements), or reference to a template file.</span>
<span class="sd"> Template reference are recognized by str ending in &#39;.sql&#39;.</span>
<span class="sd"> :param destination_dataset_table: A dotted</span>
<span class="sd"> (&lt;project&gt;.|&lt;project&gt;:)&lt;dataset&gt;.&lt;table&gt; that, if set, will store the results</span>
<span class="sd"> of the query. (templated)</span>
<span class="sd"> :type destination_dataset_table: string</span>
<span class="sd"> :param write_disposition: Specifies the action that occurs if the destination table</span>
<span class="sd"> already exists. (default: &#39;WRITE_EMPTY&#39;)</span>
<span class="sd"> :type write_disposition: string</span>
<span class="sd"> :param create_disposition: Specifies whether the job is allowed to create new tables.</span>
<span class="sd"> (default: &#39;CREATE_IF_NEEDED&#39;)</span>
<span class="sd"> :type create_disposition: string</span>
<span class="sd"> :param allow_large_results: Whether to allow large results.</span>
<span class="sd"> :type allow_large_results: boolean</span>
<span class="sd"> :param flatten_results: If true and query uses legacy SQL dialect, flattens</span>
<span class="sd"> all nested and repeated fields in the query results. ``allow_large_results``</span>
<span class="sd"> must be ``true`` if this is set to ``false``. For standard SQL queries, this</span>
<span class="sd"> flag is ignored and results are never flattened.</span>
<span class="sd"> :type flatten_results: boolean</span>
<span class="sd"> :param bigquery_conn_id: reference to a specific BigQuery hook.</span>
<span class="sd"> :type bigquery_conn_id: string</span>
<span class="sd"> :param delegate_to: The account to impersonate, if any.</span>
<span class="sd"> For this to work, the service account making the request must have domain-wide</span>
<span class="sd"> delegation enabled.</span>
<span class="sd"> :type delegate_to: string</span>
<span class="sd"> :param udf_config: The User Defined Function configuration for the query.</span>
<span class="sd"> See https://cloud.google.com/bigquery/user-defined-functions for details.</span>
<span class="sd"> :type udf_config: list</span>
<span class="sd"> :param use_legacy_sql: Whether to use legacy SQL (true) or standard SQL (false).</span>
<span class="sd"> :type use_legacy_sql: boolean</span>
<span class="sd"> :param maximum_billing_tier: Positive integer that serves as a multiplier</span>
<span class="sd"> of the basic price.</span>
<span class="sd"> Defaults to None, in which case it uses the value set in the project.</span>
<span class="sd"> :type maximum_billing_tier: integer</span>
<span class="sd"> :param maximum_bytes_billed: Limits the bytes billed for this job.</span>
<span class="sd"> Queries that will have bytes billed beyond this limit will fail</span>
<span class="sd"> (without incurring a charge). If unspecified, this will be</span>
<span class="sd"> set to your project default.</span>
<span class="sd"> :type maximum_bytes_billed: float</span>
<span class="sd"> :param api_resource_configs: a dictionary that contain params</span>
<span class="sd"> &#39;configuration&#39; applied for Google BigQuery Jobs API:</span>
<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs</span>
<span class="sd"> for example, {&#39;query&#39;: {&#39;useQueryCache&#39;: False}}. You could use it</span>
<span class="sd"> if you need to provide some params that are not supported by BigQueryOperator</span>
<span class="sd"> like args.</span>
<span class="sd"> :type api_resource_configs: dict</span>
<span class="sd"> :param schema_update_options: Allows the schema of the destination</span>
<span class="sd"> table to be updated as a side effect of the load job.</span>
<span class="sd"> :type schema_update_options: tuple</span>
<span class="sd"> :param query_params: a dictionary containing query parameter types and</span>
<span class="sd"> values, passed to BigQuery.</span>
<span class="sd"> :type query_params: dict</span>
<span class="sd"> :param labels: a dictionary containing labels for the job/query,</span>
<span class="sd"> passed to BigQuery</span>
<span class="sd"> :type labels: dict</span>
<span class="sd"> :param priority: Specifies a priority for the query.</span>
<span class="sd"> Possible values include INTERACTIVE and BATCH.</span>
<span class="sd"> The default value is INTERACTIVE.</span>
<span class="sd"> :type priority: string</span>
<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
<span class="sd"> partition by field, type and expiration as per API specifications.</span>
<span class="sd"> :type time_partitioning: dict</span>
<span class="sd"> :param cluster_fields: Request that the result of this query be stored sorted</span>
<span class="sd"> by one or more columns. This is only available in conjunction with</span>
<span class="sd"> time_partitioning. The order of columns given determines the sort order.</span>
<span class="sd"> :type cluster_fields: list of str</span>
<span class="sd"> :param location: The geographic location of the job. Required except for</span>
<span class="sd"> US and EU. See details at</span>
<span class="sd"> https://cloud.google.com/bigquery/docs/locations#specifying_your_location</span>
<span class="sd"> :type location: str</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bql&#39;</span><span class="p">,</span> <span class="s1">&#39;sql&#39;</span><span class="p">,</span> <span class="s1">&#39;destination_dataset_table&#39;</span><span class="p">,</span> <span class="s1">&#39;labels&#39;</span><span class="p">)</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;.sql&#39;</span><span class="p">,</span> <span class="p">)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#e4f0e8&#39;</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">bql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">sql</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">destination_dataset_table</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">write_disposition</span><span class="o">=</span><span class="s1">&#39;WRITE_EMPTY&#39;</span><span class="p">,</span>
<span class="n">allow_large_results</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">flatten_results</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">udf_config</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">use_legacy_sql</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">maximum_billing_tier</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">maximum_bytes_billed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">create_disposition</span><span class="o">=</span><span class="s1">&#39;CREATE_IF_NEEDED&#39;</span><span class="p">,</span>
<span class="n">schema_update_options</span><span class="o">=</span><span class="p">(),</span>
<span class="n">query_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">priority</span><span class="o">=</span><span class="s1">&#39;INTERACTIVE&#39;</span><span class="p">,</span>
<span class="n">time_partitioning</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">api_resource_configs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">cluster_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">location</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bql</span> <span class="o">=</span> <span class="n">bql</span>
<span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span> <span class="k">if</span> <span class="n">sql</span> <span class="k">else</span> <span class="n">bql</span>
<span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span> <span class="o">=</span> <span class="n">destination_dataset_table</span>
<span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span> <span class="o">=</span> <span class="n">write_disposition</span>
<span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span> <span class="o">=</span> <span class="n">create_disposition</span>
<span class="bp">self</span><span class="o">.</span><span class="n">allow_large_results</span> <span class="o">=</span> <span class="n">allow_large_results</span>
<span class="bp">self</span><span class="o">.</span><span class="n">flatten_results</span> <span class="o">=</span> <span class="n">flatten_results</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
<span class="bp">self</span><span class="o">.</span><span class="n">udf_config</span> <span class="o">=</span> <span class="n">udf_config</span>
<span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span> <span class="o">=</span> <span class="n">use_legacy_sql</span>
<span class="bp">self</span><span class="o">.</span><span class="n">maximum_billing_tier</span> <span class="o">=</span> <span class="n">maximum_billing_tier</span>
<span class="bp">self</span><span class="o">.</span><span class="n">maximum_bytes_billed</span> <span class="o">=</span> <span class="n">maximum_bytes_billed</span>
<span class="bp">self</span><span class="o">.</span><span class="n">schema_update_options</span> <span class="o">=</span> <span class="n">schema_update_options</span>
<span class="bp">self</span><span class="o">.</span><span class="n">query_params</span> <span class="o">=</span> <span class="n">query_params</span>
<span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="o">=</span> <span class="kc">None</span>
<span class="bp">self</span><span class="o">.</span><span class="n">priority</span> <span class="o">=</span> <span class="n">priority</span>
<span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span> <span class="o">=</span> <span class="n">time_partitioning</span>
<span class="bp">self</span><span class="o">.</span><span class="n">api_resource_configs</span> <span class="o">=</span> <span class="n">api_resource_configs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">cluster_fields</span> <span class="o">=</span> <span class="n">cluster_fields</span>
<span class="bp">self</span><span class="o">.</span><span class="n">location</span> <span class="o">=</span> <span class="n">location</span>
<span class="c1"># TODO remove `bql` in Airflow 2.0</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bql</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">warnings</span>
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">&#39;Deprecated parameter `bql` used in Task id: </span><span class="si">{}</span><span class="s1">. &#39;</span>
<span class="s1">&#39;Use `sql` parameter instead to pass the sql to be &#39;</span>
<span class="s1">&#39;executed. `bql` parameter is deprecated and &#39;</span>
<span class="s1">&#39;will be removed in a future version of &#39;</span>
<span class="s1">&#39;Airflow.&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">),</span>
<span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> missing 1 required positional &#39;</span>
<span class="s1">&#39;argument: `sql`&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span>
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
<span class="n">use_legacy_sql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_legacy_sql</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">,</span>
<span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">conn</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span><span class="o">.</span><span class="n">run_query</span><span class="p">(</span>
<span class="n">sql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span>
<span class="n">destination_dataset_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">destination_dataset_table</span><span class="p">,</span>
<span class="n">write_disposition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">write_disposition</span><span class="p">,</span>
<span class="n">allow_large_results</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_large_results</span><span class="p">,</span>
<span class="n">flatten_results</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">flatten_results</span><span class="p">,</span>
<span class="n">udf_config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">udf_config</span><span class="p">,</span>
<span class="n">maximum_billing_tier</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">maximum_billing_tier</span><span class="p">,</span>
<span class="n">maximum_bytes_billed</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">maximum_bytes_billed</span><span class="p">,</span>
<span class="n">create_disposition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">create_disposition</span><span class="p">,</span>
<span class="n">query_params</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">query_params</span><span class="p">,</span>
<span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">,</span>
<span class="n">schema_update_options</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema_update_options</span><span class="p">,</span>
<span class="n">priority</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">priority</span><span class="p">,</span>
<span class="n">time_partitioning</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span><span class="p">,</span>
<span class="n">api_resource_configs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">api_resource_configs</span><span class="p">,</span>
<span class="n">cluster_fields</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster_fields</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">on_kill</span><span class="p">()</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Cancelling running query&#39;</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bq_cursor</span><span class="o">.</span><span class="n">cancel_query</span><span class="p">()</span></div>
<div class="viewcode-block" id="BigQueryCreateEmptyTableOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryCreateEmptyTableOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryCreateEmptyTableOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Creates a new, empty table in the specified BigQuery dataset,</span>
<span class="sd"> optionally with schema.</span>
<span class="sd"> The schema to be used for the BigQuery table may be specified in one of</span>
<span class="sd"> two ways. You may either directly pass the schema fields in, or you may</span>
<span class="sd"> point the operator to a Google cloud storage object name. The object in</span>
<span class="sd"> Google cloud storage must be a JSON file with the schema fields in it.</span>
<span class="sd"> You can also create a table without schema.</span>
<span class="sd"> :param project_id: The project to create the table into. (templated)</span>
<span class="sd"> :type project_id: string</span>
<span class="sd"> :param dataset_id: The dataset to create the table into. (templated)</span>
<span class="sd"> :type dataset_id: string</span>
<span class="sd"> :param table_id: The Name of the table to be created. (templated)</span>
<span class="sd"> :type table_id: string</span>
<span class="sd"> :param schema_fields: If set, the schema field list as defined here:</span>
<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema</span>
<span class="sd"> **Example**: ::</span>
<span class="sd"> schema_fields=[{&quot;name&quot;: &quot;emp_name&quot;, &quot;type&quot;: &quot;STRING&quot;, &quot;mode&quot;: &quot;REQUIRED&quot;},</span>
<span class="sd"> {&quot;name&quot;: &quot;salary&quot;, &quot;type&quot;: &quot;INTEGER&quot;, &quot;mode&quot;: &quot;NULLABLE&quot;}]</span>
<span class="sd"> :type schema_fields: list</span>
<span class="sd"> :param gcs_schema_object: Full path to the JSON file containing</span>
<span class="sd"> schema (templated). For</span>
<span class="sd"> example: ``gs://test-bucket/dir1/dir2/employee_schema.json``</span>
<span class="sd"> :type gcs_schema_object: string</span>
<span class="sd"> :param time_partitioning: configure optional time partitioning fields i.e.</span>
<span class="sd"> partition by field, type and expiration as per API specifications.</span>
<span class="sd"> .. seealso::</span>
<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#timePartitioning</span>
<span class="sd"> :type time_partitioning: dict</span>
<span class="sd"> :param bigquery_conn_id: Reference to a specific BigQuery hook.</span>
<span class="sd"> :type bigquery_conn_id: string</span>
<span class="sd"> :param google_cloud_storage_conn_id: Reference to a specific Google</span>
<span class="sd"> cloud storage hook.</span>
<span class="sd"> :type google_cloud_storage_conn_id: string</span>
<span class="sd"> :param delegate_to: The account to impersonate, if any. For this to</span>
<span class="sd"> work, the service account making the request must have domain-wide</span>
<span class="sd"> delegation enabled.</span>
<span class="sd"> :type delegate_to: string</span>
<span class="sd"> :param labels: a dictionary containing labels for the table, passed to BigQuery</span>
<span class="sd"> **Example (with schema JSON in GCS)**: ::</span>
<span class="sd"> CreateTable = BigQueryCreateEmptyTableOperator(</span>
<span class="sd"> task_id=&#39;BigQueryCreateEmptyTableOperator_task&#39;,</span>
<span class="sd"> dataset_id=&#39;ODS&#39;,</span>
<span class="sd"> table_id=&#39;Employees&#39;,</span>
<span class="sd"> project_id=&#39;internal-gcp-project&#39;,</span>
<span class="sd"> gcs_schema_object=&#39;gs://schema-bucket/employee_schema.json&#39;,</span>
<span class="sd"> bigquery_conn_id=&#39;airflow-service-account&#39;,</span>
<span class="sd"> google_cloud_storage_conn_id=&#39;airflow-service-account&#39;</span>
<span class="sd"> )</span>
<span class="sd"> **Corresponding Schema file** (``employee_schema.json``): ::</span>
<span class="sd"> [</span>
<span class="sd"> {</span>
<span class="sd"> &quot;mode&quot;: &quot;NULLABLE&quot;,</span>
<span class="sd"> &quot;name&quot;: &quot;emp_name&quot;,</span>
<span class="sd"> &quot;type&quot;: &quot;STRING&quot;</span>
<span class="sd"> },</span>
<span class="sd"> {</span>
<span class="sd"> &quot;mode&quot;: &quot;REQUIRED&quot;,</span>
<span class="sd"> &quot;name&quot;: &quot;salary&quot;,</span>
<span class="sd"> &quot;type&quot;: &quot;INTEGER&quot;</span>
<span class="sd"> }</span>
<span class="sd"> ]</span>
<span class="sd"> **Example (with schema in the DAG)**: ::</span>
<span class="sd"> CreateTable = BigQueryCreateEmptyTableOperator(</span>
<span class="sd"> task_id=&#39;BigQueryCreateEmptyTableOperator_task&#39;,</span>
<span class="sd"> dataset_id=&#39;ODS&#39;,</span>
<span class="sd"> table_id=&#39;Employees&#39;,</span>
<span class="sd"> project_id=&#39;internal-gcp-project&#39;,</span>
<span class="sd"> schema_fields=[{&quot;name&quot;: &quot;emp_name&quot;, &quot;type&quot;: &quot;STRING&quot;, &quot;mode&quot;: &quot;REQUIRED&quot;},</span>
<span class="sd"> {&quot;name&quot;: &quot;salary&quot;, &quot;type&quot;: &quot;INTEGER&quot;, &quot;mode&quot;: &quot;NULLABLE&quot;}],</span>
<span class="sd"> bigquery_conn_id=&#39;airflow-service-account&#39;,</span>
<span class="sd"> google_cloud_storage_conn_id=&#39;airflow-service-account&#39;</span>
<span class="sd"> )</span>
<span class="sd"> :type labels: dict</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;dataset_id&#39;</span><span class="p">,</span> <span class="s1">&#39;table_id&#39;</span><span class="p">,</span> <span class="s1">&#39;project_id&#39;</span><span class="p">,</span>
<span class="s1">&#39;gcs_schema_object&#39;</span><span class="p">,</span> <span class="s1">&#39;labels&#39;</span><span class="p">)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f0eee4&#39;</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">dataset_id</span><span class="p">,</span>
<span class="n">table_id</span><span class="p">,</span>
<span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">schema_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">gcs_schema_object</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">time_partitioning</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
<span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCreateEmptyTableOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span> <span class="o">=</span> <span class="n">dataset_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">table_id</span> <span class="o">=</span> <span class="n">table_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span> <span class="o">=</span> <span class="n">schema_fields</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gcs_schema_object</span> <span class="o">=</span> <span class="n">gcs_schema_object</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
<span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span> <span class="o">=</span> <span class="p">{}</span> <span class="k">if</span> <span class="n">time_partitioning</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">time_partitioning</span>
<span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">bq_hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">gcs_schema_object</span><span class="p">:</span>
<span class="n">gcs_bucket</span><span class="p">,</span> <span class="n">gcs_object</span> <span class="o">=</span> <span class="n">_parse_gcs_url</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">gcs_schema_object</span><span class="p">)</span>
<span class="n">gcs_hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
<span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="n">schema_fields</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">gcs_hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span>
<span class="n">gcs_bucket</span><span class="p">,</span>
<span class="n">gcs_object</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">&quot;utf-8&quot;</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">schema_fields</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span>
<span class="n">conn</span> <span class="o">=</span> <span class="n">bq_hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
<span class="n">cursor</span><span class="o">.</span><span class="n">create_empty_table</span><span class="p">(</span>
<span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
<span class="n">dataset_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span>
<span class="n">table_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_id</span><span class="p">,</span>
<span class="n">schema_fields</span><span class="o">=</span><span class="n">schema_fields</span><span class="p">,</span>
<span class="n">time_partitioning</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span><span class="p">,</span>
<span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span>
<span class="p">)</span></div>
<div class="viewcode-block" id="BigQueryCreateExternalTableOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryCreateExternalTableOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryCreateExternalTableOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Creates a new external table in the dataset with the data in Google Cloud</span>
<span class="sd"> Storage.</span>
<span class="sd"> The schema to be used for the BigQuery table may be specified in one of</span>
<span class="sd"> two ways. You may either directly pass the schema fields in, or you may</span>
<span class="sd"> point the operator to a Google cloud storage object name. The object in</span>
<span class="sd"> Google cloud storage must be a JSON file with the schema fields in it.</span>
<span class="sd"> :param bucket: The bucket to point the external table to. (templated)</span>
<span class="sd"> :type bucket: string</span>
<span class="sd"> :param source_objects: List of Google cloud storage URIs to point</span>
<span class="sd"> table to. (templated)</span>
<span class="sd"> If source_format is &#39;DATASTORE_BACKUP&#39;, the list must only contain a single URI.</span>
<span class="sd"> :type source_objects: list</span>
<span class="sd"> :param destination_project_dataset_table: The dotted (&lt;project&gt;.)&lt;dataset&gt;.&lt;table&gt;</span>
<span class="sd"> BigQuery table to load data into (templated). If &lt;project&gt; is not included,</span>
<span class="sd"> project will be the project defined in the connection json.</span>
<span class="sd"> :type destination_project_dataset_table: string</span>
<span class="sd"> :param schema_fields: If set, the schema field list as defined here:</span>
<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema</span>
<span class="sd"> **Example**: ::</span>
<span class="sd"> schema_fields=[{&quot;name&quot;: &quot;emp_name&quot;, &quot;type&quot;: &quot;STRING&quot;, &quot;mode&quot;: &quot;REQUIRED&quot;},</span>
<span class="sd"> {&quot;name&quot;: &quot;salary&quot;, &quot;type&quot;: &quot;INTEGER&quot;, &quot;mode&quot;: &quot;NULLABLE&quot;}]</span>
<span class="sd"> Should not be set when source_format is &#39;DATASTORE_BACKUP&#39;.</span>
<span class="sd"> :type schema_fields: list</span>
<span class="sd"> :param schema_object: If set, a GCS object path pointing to a .json file that</span>
<span class="sd"> contains the schema for the table. (templated)</span>
<span class="sd"> :type schema_object: string</span>
<span class="sd"> :param source_format: File format of the data.</span>
<span class="sd"> :type source_format: string</span>
<span class="sd"> :param compression: [Optional] The compression type of the data source.</span>
<span class="sd"> Possible values include GZIP and NONE.</span>
<span class="sd"> The default value is NONE.</span>
<span class="sd"> This setting is ignored for Google Cloud Bigtable,</span>
<span class="sd"> Google Cloud Datastore backups and Avro formats.</span>
<span class="sd"> :type compression: string</span>
<span class="sd"> :param skip_leading_rows: Number of rows to skip when loading from a CSV.</span>
<span class="sd"> :type skip_leading_rows: int</span>
<span class="sd"> :param field_delimiter: The delimiter to use for the CSV.</span>
<span class="sd"> :type field_delimiter: string</span>
<span class="sd"> :param max_bad_records: The maximum number of bad records that BigQuery can</span>
<span class="sd"> ignore when running the job.</span>
<span class="sd"> :type max_bad_records: int</span>
<span class="sd"> :param quote_character: The value that is used to quote data sections in a CSV file.</span>
<span class="sd"> :type quote_character: string</span>
<span class="sd"> :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not (false).</span>
<span class="sd"> :type allow_quoted_newlines: boolean</span>
<span class="sd"> :param allow_jagged_rows: Accept rows that are missing trailing optional columns.</span>
<span class="sd"> The missing values are treated as nulls. If false, records with missing trailing</span>
<span class="sd"> columns are treated as bad records, and if there are too many bad records, an</span>
<span class="sd"> invalid error is returned in the job result. Only applicable to CSV, ignored</span>
<span class="sd"> for other formats.</span>
<span class="sd"> :type allow_jagged_rows: bool</span>
<span class="sd"> :param bigquery_conn_id: Reference to a specific BigQuery hook.</span>
<span class="sd"> :type bigquery_conn_id: string</span>
<span class="sd"> :param google_cloud_storage_conn_id: Reference to a specific Google</span>
<span class="sd"> cloud storage hook.</span>
<span class="sd"> :type google_cloud_storage_conn_id: string</span>
<span class="sd"> :param delegate_to: The account to impersonate, if any. For this to</span>
<span class="sd"> work, the service account making the request must have domain-wide</span>
<span class="sd"> delegation enabled.</span>
<span class="sd"> :type delegate_to: string</span>
<span class="sd"> :param src_fmt_configs: configure optional fields specific to the source format</span>
<span class="sd"> :type src_fmt_configs: dict</span>
<span class="sd"> :param labels a dictionary containing labels for the table, passed to BigQuery</span>
<span class="sd"> :type labels: dict</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;bucket&#39;</span><span class="p">,</span> <span class="s1">&#39;source_objects&#39;</span><span class="p">,</span>
<span class="s1">&#39;schema_object&#39;</span><span class="p">,</span> <span class="s1">&#39;destination_project_dataset_table&#39;</span><span class="p">,</span> <span class="s1">&#39;labels&#39;</span><span class="p">)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f0eee4&#39;</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">bucket</span><span class="p">,</span>
<span class="n">source_objects</span><span class="p">,</span>
<span class="n">destination_project_dataset_table</span><span class="p">,</span>
<span class="n">schema_fields</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">schema_object</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">source_format</span><span class="o">=</span><span class="s1">&#39;CSV&#39;</span><span class="p">,</span>
<span class="n">compression</span><span class="o">=</span><span class="s1">&#39;NONE&#39;</span><span class="p">,</span>
<span class="n">skip_leading_rows</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
<span class="n">field_delimiter</span><span class="o">=</span><span class="s1">&#39;,&#39;</span><span class="p">,</span>
<span class="n">max_bad_records</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
<span class="n">quote_character</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">allow_jagged_rows</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
<span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">&#39;google_cloud_default&#39;</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">src_fmt_configs</span><span class="o">=</span><span class="p">{},</span>
<span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCreateExternalTableOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="c1"># GCS config</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
<span class="bp">self</span><span class="o">.</span><span class="n">source_objects</span> <span class="o">=</span> <span class="n">source_objects</span>
<span class="bp">self</span><span class="o">.</span><span class="n">schema_object</span> <span class="o">=</span> <span class="n">schema_object</span>
<span class="c1"># BQ config</span>
<span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span> <span class="o">=</span> <span class="n">destination_project_dataset_table</span>
<span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span> <span class="o">=</span> <span class="n">schema_fields</span>
<span class="bp">self</span><span class="o">.</span><span class="n">source_format</span> <span class="o">=</span> <span class="n">source_format</span>
<span class="bp">self</span><span class="o">.</span><span class="n">compression</span> <span class="o">=</span> <span class="n">compression</span>
<span class="bp">self</span><span class="o">.</span><span class="n">skip_leading_rows</span> <span class="o">=</span> <span class="n">skip_leading_rows</span>
<span class="bp">self</span><span class="o">.</span><span class="n">field_delimiter</span> <span class="o">=</span> <span class="n">field_delimiter</span>
<span class="bp">self</span><span class="o">.</span><span class="n">max_bad_records</span> <span class="o">=</span> <span class="n">max_bad_records</span>
<span class="bp">self</span><span class="o">.</span><span class="n">quote_character</span> <span class="o">=</span> <span class="n">quote_character</span>
<span class="bp">self</span><span class="o">.</span><span class="n">allow_quoted_newlines</span> <span class="o">=</span> <span class="n">allow_quoted_newlines</span>
<span class="bp">self</span><span class="o">.</span><span class="n">allow_jagged_rows</span> <span class="o">=</span> <span class="n">allow_jagged_rows</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
<span class="bp">self</span><span class="o">.</span><span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="n">src_fmt_configs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">bq_hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_object</span> \
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_format</span> <span class="o">!=</span> <span class="s1">&#39;DATASTORE_BACKUP&#39;</span><span class="p">:</span>
<span class="n">gcs_hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span>
<span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="n">schema_fields</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">gcs_hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">schema_object</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">&quot;utf-8&quot;</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">schema_fields</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema_fields</span>
<span class="n">source_uris</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;gs://</span><span class="si">{}</span><span class="s1">/</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">source_object</span><span class="p">)</span>
<span class="k">for</span> <span class="n">source_object</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_objects</span><span class="p">]</span>
<span class="n">conn</span> <span class="o">=</span> <span class="n">bq_hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
<span class="n">cursor</span><span class="o">.</span><span class="n">create_external_table</span><span class="p">(</span>
<span class="n">external_project_dataset_table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span><span class="p">,</span>
<span class="n">schema_fields</span><span class="o">=</span><span class="n">schema_fields</span><span class="p">,</span>
<span class="n">source_uris</span><span class="o">=</span><span class="n">source_uris</span><span class="p">,</span>
<span class="n">source_format</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source_format</span><span class="p">,</span>
<span class="n">compression</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compression</span><span class="p">,</span>
<span class="n">skip_leading_rows</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">skip_leading_rows</span><span class="p">,</span>
<span class="n">field_delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">field_delimiter</span><span class="p">,</span>
<span class="n">max_bad_records</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_bad_records</span><span class="p">,</span>
<span class="n">quote_character</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">quote_character</span><span class="p">,</span>
<span class="n">allow_quoted_newlines</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_quoted_newlines</span><span class="p">,</span>
<span class="n">allow_jagged_rows</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">allow_jagged_rows</span><span class="p">,</span>
<span class="n">src_fmt_configs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">src_fmt_configs</span><span class="p">,</span>
<span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span>
<span class="p">)</span></div>
<div class="viewcode-block" id="BigQueryDeleteDatasetOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryDeleteDatasetOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryDeleteDatasetOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;&quot;</span>
<span class="sd"> This operator deletes an existing dataset from your Project in Big query.</span>
<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/delete</span>
<span class="sd"> :param project_id: The project id of the dataset.</span>
<span class="sd"> :type project_id: string</span>
<span class="sd"> :param dataset_id: The dataset to be deleted.</span>
<span class="sd"> :type dataset_id: string</span>
<span class="sd"> **Example**: ::</span>
<span class="sd"> delete_temp_data = BigQueryDeleteDatasetOperator(dataset_id = &#39;temp-dataset&#39;,</span>
<span class="sd"> project_id = &#39;temp-project&#39;,</span>
<span class="sd"> bigquery_conn_id=&#39;_my_gcp_conn_&#39;,</span>
<span class="sd"> task_id=&#39;Deletetemp&#39;,</span>
<span class="sd"> dag=dag)</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;dataset_id&#39;</span><span class="p">,</span> <span class="s1">&#39;project_id&#39;</span><span class="p">)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f00004&#39;</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">dataset_id</span><span class="p">,</span>
<span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span> <span class="o">=</span> <span class="n">dataset_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Dataset id: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Project id: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryDeleteDatasetOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">bq_hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="n">conn</span> <span class="o">=</span> <span class="n">bq_hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
<span class="n">cursor</span><span class="o">.</span><span class="n">delete_dataset</span><span class="p">(</span>
<span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
<span class="n">dataset_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span>
<span class="p">)</span></div>
<div class="viewcode-block" id="BigQueryCreateEmptyDatasetOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_operator.BigQueryCreateEmptyDatasetOperator">[docs]</a><span class="k">class</span> <span class="nc">BigQueryCreateEmptyDatasetOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;&quot;</span>
<span class="sd"> This operator is used to create new dataset for your Project in Big query.</span>
<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource</span>
<span class="sd"> :param project_id: The name of the project where we want to create the dataset.</span>
<span class="sd"> Don&#39;t need to provide, if projectId in dataset_reference.</span>
<span class="sd"> :type project_id: str</span>
<span class="sd"> :param dataset_id: The id of dataset. Don&#39;t need to provide,</span>
<span class="sd"> if datasetId in dataset_reference.</span>
<span class="sd"> :type dataset_id: str</span>
<span class="sd"> :param dataset_reference: Dataset reference that could be provided with request body.</span>
<span class="sd"> More info:</span>
<span class="sd"> https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource</span>
<span class="sd"> :type dataset_reference: dict</span>
<span class="sd"> **Example**: ::</span>
<span class="sd"> create_new_dataset = BigQueryCreateEmptyDatasetOperator(</span>
<span class="sd"> dataset_id = &#39;new-dataset&#39;,</span>
<span class="sd"> project_id = &#39;my-project&#39;,</span>
<span class="sd"> dataset_reference = {&quot;friendlyName&quot;: &quot;New Dataset&quot;}</span>
<span class="sd"> bigquery_conn_id=&#39;_my_gcp_conn_&#39;,</span>
<span class="sd"> task_id=&#39;newDatasetCreator&#39;,</span>
<span class="sd"> dag=dag)</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;dataset_id&#39;</span><span class="p">,</span> <span class="s1">&#39;project_id&#39;</span><span class="p">)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#f0eee4&#39;</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">dataset_id</span><span class="p">,</span>
<span class="n">project_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">dataset_reference</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">&#39;bigquery_default&#39;</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span> <span class="o">=</span> <span class="n">dataset_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">project_id</span> <span class="o">=</span> <span class="n">project_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataset_reference</span> <span class="o">=</span> <span class="n">dataset_reference</span> <span class="k">if</span> <span class="n">dataset_reference</span> <span class="k">else</span> <span class="p">{}</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Dataset id: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Project id: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">)</span>
<span class="nb">super</span><span class="p">(</span><span class="n">BigQueryCreateEmptyDatasetOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">bq_hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
<span class="n">conn</span> <span class="o">=</span> <span class="n">bq_hook</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
<span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
<span class="n">cursor</span><span class="o">.</span><span class="n">create_empty_dataset</span><span class="p">(</span>
<span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span>
<span class="n">dataset_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span>
<span class="n">dataset_reference</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_reference</span><span class="p">)</span></div>
</pre></div>
</div>
</div>
<footer>
<hr/>
<div role="contentinfo">
<p>
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
<script type="text/javascript" src="../../../../_static/jquery.js"></script>
<script type="text/javascript" src="../../../../_static/underscore.js"></script>
<script type="text/javascript" src="../../../../_static/doctools.js"></script>
<script type="text/javascript" src="../../../../_static/language_data.js"></script>
<script type="text/javascript" src="../../../../_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>