blob: aacbba1afc8e1430e6f70fa36e055cad5293ba3a [file] [log] [blame]
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>apache_beam.io.gcp.bigquery_tools module &mdash; Apache Beam documentation</title>
<script type="text/javascript" src="_static/js/modernizr.min.js"></script>
<script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
<script type="text/javascript" src="_static/jquery.js"></script>
<script type="text/javascript" src="_static/underscore.js"></script>
<script type="text/javascript" src="_static/doctools.js"></script>
<script type="text/javascript" src="_static/language_data.js"></script>
<script async="async" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
<script type="text/javascript" src="_static/js/theme.js"></script>
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="apache_beam.io.gcp.bigtableio module" href="apache_beam.io.gcp.bigtableio.html" />
<link rel="prev" title="apache_beam.io.gcp.bigquery_read_internal module" href="apache_beam.io.gcp.bigquery_read_internal.html" />
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="index.html" class="icon icon-home"> Apache Beam
</a>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="apache_beam.coders.html">apache_beam.coders package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.dataframe.html">apache_beam.dataframe package</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="apache_beam.io.html">apache_beam.io package</a><ul class="current">
<li class="toctree-l2 current"><a class="reference internal" href="apache_beam.io.html#subpackages">Subpackages</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="apache_beam.io.aws.html">apache_beam.io.aws package</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.io.azure.html">apache_beam.io.azure package</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.io.external.html">apache_beam.io.external package</a></li>
<li class="toctree-l3"><a class="reference internal" href="apache_beam.io.flink.html">apache_beam.io.flink package</a></li>
<li class="toctree-l3 current"><a class="reference internal" href="apache_beam.io.gcp.html">apache_beam.io.gcp package</a><ul class="current">
<li class="toctree-l4"><a class="reference internal" href="apache_beam.io.gcp.html#subpackages">Subpackages</a></li>
<li class="toctree-l4 current"><a class="reference internal" href="apache_beam.io.gcp.html#submodules">Submodules</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="apache_beam.io.html#submodules">Submodules</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.metrics.html">apache_beam.metrics package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.ml.html">apache_beam.ml package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.options.html">apache_beam.options package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.portability.html">apache_beam.portability package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.runners.html">apache_beam.runners package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.transforms.html">apache_beam.transforms package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.typehints.html">apache_beam.typehints package</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.utils.html">apache_beam.utils package</a></li>
</ul>
<ul>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.error.html">apache_beam.error module</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
<li class="toctree-l1"><a class="reference internal" href="apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Apache Beam</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="index.html">Docs</a> &raquo;</li>
<li><a href="apache_beam.io.html">apache_beam.io package</a> &raquo;</li>
<li><a href="apache_beam.io.gcp.html">apache_beam.io.gcp package</a> &raquo;</li>
<li>apache_beam.io.gcp.bigquery_tools module</li>
<li class="wy-breadcrumbs-aside">
<a href="_sources/apache_beam.io.gcp.bigquery_tools.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="module-apache_beam.io.gcp.bigquery_tools">
<span id="apache-beam-io-gcp-bigquery-tools-module"></span><h1>apache_beam.io.gcp.bigquery_tools module<a class="headerlink" href="#module-apache_beam.io.gcp.bigquery_tools" title="Permalink to this headline"></a></h1>
<p>Tools used by BigQuery sources and sinks.</p>
<p>Classes, constants and functions in this file are experimental and have no
backwards compatibility guarantees.</p>
<p>These tools include wrappers and clients to interact with BigQuery APIs.</p>
<p>NOTHING IN THIS FILE HAS BACKWARDS COMPATIBILITY GUARANTEES.</p>
<dl class="class">
<dt id="apache_beam.io.gcp.bigquery_tools.FileFormat">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">FileFormat</code><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#FileFormat"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.FileFormat" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.10)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.FileFormat.CSV">
<code class="descname">CSV</code><em class="property"> = 'CSV'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.FileFormat.CSV" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.FileFormat.JSON">
<code class="descname">JSON</code><em class="property"> = 'NEWLINE_DELIMITED_JSON'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.FileFormat.JSON" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.FileFormat.AVRO">
<code class="descname">AVRO</code><em class="property"> = 'AVRO'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.FileFormat.AVRO" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
<dl class="class">
<dt id="apache_beam.io.gcp.bigquery_tools.ExportCompression">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">ExportCompression</code><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#ExportCompression"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.ExportCompression" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.10)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.ExportCompression.GZIP">
<code class="descname">GZIP</code><em class="property"> = 'GZIP'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.ExportCompression.GZIP" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.ExportCompression.DEFLATE">
<code class="descname">DEFLATE</code><em class="property"> = 'DEFLATE'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.ExportCompression.DEFLATE" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.ExportCompression.SNAPPY">
<code class="descname">SNAPPY</code><em class="property"> = 'SNAPPY'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.ExportCompression.SNAPPY" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.ExportCompression.NONE">
<code class="descname">NONE</code><em class="property"> = 'NONE'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.ExportCompression.NONE" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.bigquery_tools.default_encoder">
<code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">default_encoder</code><span class="sig-paren">(</span><em>obj</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#default_encoder"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.default_encoder" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.bigquery_tools.get_hashable_destination">
<code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">get_hashable_destination</code><span class="sig-paren">(</span><em>destination</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#get_hashable_destination"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.get_hashable_destination" title="Permalink to this definition"></a></dt>
<dd><p>Parses a table reference into a (project, dataset, table) tuple.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>destination</strong> – Either a TableReference object from the bigquery API.
The object has the following attributes: projectId, datasetId, and
tableId. Or a string representing the destination containing
‘PROJECT:DATASET.TABLE’.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">A string representing the destination containing
‘PROJECT:DATASET.TABLE’.</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.bigquery_tools.to_hashable_table_ref">
<code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">to_hashable_table_ref</code><span class="sig-paren">(</span><em>table_ref_elem_kv: Tuple[Union[str, apache_beam.io.gcp.internal.clients.bigquery.bigquery_v2_messages.TableReference], V]</em><span class="sig-paren">)</span> &#x2192; Tuple[str, V]<a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#to_hashable_table_ref"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.to_hashable_table_ref" title="Permalink to this definition"></a></dt>
<dd><p>Turns the key of the input tuple to its string representation. The key
should be either a string or a TableReference.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>table_ref_elem_kv</strong> – A tuple of table reference and element.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">A tuple of string representation of input table and input element.</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.bigquery_tools.parse_table_schema_from_json">
<code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">parse_table_schema_from_json</code><span class="sig-paren">(</span><em>schema_string</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#parse_table_schema_from_json"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.parse_table_schema_from_json" title="Permalink to this definition"></a></dt>
<dd><p>Parse the Table Schema provided as string.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>schema_string</strong> – String serialized table schema, should be a valid JSON.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">A TableSchema of the BigQuery export from either the Query or the Table.</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.bigquery_tools.parse_table_reference">
<code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">parse_table_reference</code><span class="sig-paren">(</span><em>table</em>, <em>dataset=None</em>, <em>project=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#parse_table_reference"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.parse_table_reference" title="Permalink to this definition"></a></dt>
<dd><p>Parses a table reference into a (project, dataset, table) tuple.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>table</strong> – The ID of the table. The ID must contain only letters
(a-z, A-Z), numbers (0-9), connectors (-_). If dataset argument is None
then the table argument must contain the entire table reference:
‘DATASET.TABLE’ or ‘PROJECT:DATASET.TABLE’. This argument can be a
TableReference instance in which case dataset and project are
ignored and the reference is returned as a result. Additionally, for date
partitioned tables, appending ‘$YYYYmmdd’ to the table name is supported,
e.g. ‘DATASET.TABLE$YYYYmmdd’.</li>
<li><strong>dataset</strong> – The ID of the dataset containing this table or null if the table
reference is specified entirely by the table argument.</li>
<li><strong>project</strong> – The ID of the project containing this table or null if the table
reference is specified entirely by the table (and possibly dataset)
argument.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">A TableReference object from the bigquery API. The object has the following
attributes: projectId, datasetId, and tableId.
If the input is a TableReference object, a new object will be returned.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Raises:</th><td class="field-body"><p class="first last"><a class="reference external" href="https://docs.python.org/3/library/exceptions.html#ValueError" title="(in Python v3.10)"><code class="xref py py-exc docutils literal notranslate"><span class="pre">ValueError</span></code></a> – if the table reference as a string does not match the expected
format.</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="class">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">BigQueryWrapper</code><span class="sig-paren">(</span><em>client=None</em>, <em>temp_dataset_id=None</em>, <em>temp_table_ref=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.10)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p>
<p>BigQuery client wrapper with utilities for querying.</p>
<p>The wrapper is used to organize all the BigQuery integration points and
offer a common place where retry logic for failures can be controlled.
In addition it offers various functions used both in sources and sinks
(e.g., find and create tables, query a table, etc.).</p>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.TEMP_TABLE">
<code class="descname">TEMP_TABLE</code><em class="property"> = 'beam_temp_table_'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.TEMP_TABLE" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.TEMP_DATASET">
<code class="descname">TEMP_DATASET</code><em class="property"> = 'beam_temp_dataset_'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.TEMP_DATASET" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.HISTOGRAM_METRIC_LOGGER">
<code class="descname">HISTOGRAM_METRIC_LOGGER</code><em class="property"> = &lt;apache_beam.internal.metrics.metric.MetricLogger object&gt;</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.HISTOGRAM_METRIC_LOGGER" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.unique_row_id">
<code class="descname">unique_row_id</code><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.unique_row_id" title="Permalink to this definition"></a></dt>
<dd><p>Returns a unique row ID (str) used to avoid multiple insertions.</p>
<p>If the row ID is provided, BigQuery will make a best effort to not insert
the same row multiple times for fail and retry scenarios in which the insert
request may be issued several times. This comes into play for sinks executed
in a local runner.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body">a unique row ID string</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_query_location">
<code class="descname">get_query_location</code><span class="sig-paren">(</span><em>project_id</em>, <em>query</em>, <em>use_legacy_sql</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.get_query_location"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_query_location" title="Permalink to this definition"></a></dt>
<dd><p>Get the location of tables referenced in a query.</p>
<p>This method returns the location of the first available referenced
table for user in the query and depends on the BigQuery service to
provide error handling for queries that reference tables in multiple
locations.</p>
</dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.wait_for_bq_job">
<code class="descname">wait_for_bq_job</code><span class="sig-paren">(</span><em>job_reference</em>, <em>sleep_duration_sec=5</em>, <em>max_retries=0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.wait_for_bq_job"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.wait_for_bq_job" title="Permalink to this definition"></a></dt>
<dd><p>Poll job until it is DONE.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>job_reference</strong> – bigquery.JobReference instance.</li>
<li><strong>sleep_duration_sec</strong> – Specifies the delay in seconds between retries.</li>
<li><strong>max_retries</strong> – The total number of times to retry. If equals to 0,
the function waits forever.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Raises:</th><td class="field-body"><p class="first last"><cite>RuntimeError</cite> – If the job is FAILED or the number of retries has been
reached.</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_table">
<code class="descname">get_table</code><span class="sig-paren">(</span><em>project_id</em>, <em>dataset_id</em>, <em>table_id</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.get_table"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_table" title="Permalink to this definition"></a></dt>
<dd><p>Lookup a table’s metadata object.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>client</strong> – bigquery.BigqueryV2 instance</li>
<li><strong>project_id</strong> – table lookup parameter</li>
<li><strong>dataset_id</strong> – table lookup parameter</li>
<li><strong>table_id</strong> – table lookup parameter</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">bigquery.Table instance</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Raises:</th><td class="field-body"><p class="first last"><code class="xref py py-exc docutils literal notranslate"><span class="pre">HttpError</span></code> – if lookup failed.</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_or_create_dataset">
<code class="descname">get_or_create_dataset</code><span class="sig-paren">(</span><em>project_id</em>, <em>dataset_id</em>, <em>location=None</em>, <em>labels=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.get_or_create_dataset"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_or_create_dataset" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_table_location">
<code class="descname">get_table_location</code><span class="sig-paren">(</span><em>project_id</em>, <em>dataset_id</em>, <em>table_id</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.get_table_location"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_table_location" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.is_user_configured_dataset">
<code class="descname">is_user_configured_dataset</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.is_user_configured_dataset"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.is_user_configured_dataset" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.create_temporary_dataset">
<code class="descname">create_temporary_dataset</code><span class="sig-paren">(</span><em>project_id</em>, <em>location</em>, <em>labels=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.create_temporary_dataset"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.create_temporary_dataset" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.clean_up_temporary_dataset">
<code class="descname">clean_up_temporary_dataset</code><span class="sig-paren">(</span><em>project_id</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.clean_up_temporary_dataset"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.clean_up_temporary_dataset" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_job">
<code class="descname">get_job</code><span class="sig-paren">(</span><em>project</em>, <em>job_id</em>, <em>location=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.get_job"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_job" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.perform_load_job">
<code class="descname">perform_load_job</code><span class="sig-paren">(</span><em>destination</em>, <em>job_id</em>, <em>source_uris=None</em>, <em>source_stream=None</em>, <em>schema=None</em>, <em>write_disposition=None</em>, <em>create_disposition=None</em>, <em>additional_load_parameters=None</em>, <em>source_format=None</em>, <em>job_labels=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.perform_load_job"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.perform_load_job" title="Permalink to this definition"></a></dt>
<dd><p>Starts a job to load data into BigQuery.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body">bigquery.JobReference with the information about the job that was started.</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.perform_extract_job">
<code class="descname">perform_extract_job</code><span class="sig-paren">(</span><em>destination</em>, <em>job_id</em>, <em>table_reference</em>, <em>destination_format</em>, <em>project=None</em>, <em>include_header=True</em>, <em>compression='NONE'</em>, <em>use_avro_logical_types=False</em>, <em>job_labels=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.perform_extract_job"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.perform_extract_job" title="Permalink to this definition"></a></dt>
<dd><p>Starts a job to export data from BigQuery.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body">bigquery.JobReference with the information about the job that was started.</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_or_create_table">
<code class="descname">get_or_create_table</code><span class="sig-paren">(</span><em>project_id</em>, <em>dataset_id</em>, <em>table_id</em>, <em>schema</em>, <em>create_disposition</em>, <em>write_disposition</em>, <em>additional_create_parameters=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.get_or_create_table"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_or_create_table" title="Permalink to this definition"></a></dt>
<dd><p>Gets or creates a table based on create and write dispositions.</p>
<p>The function mimics the behavior of BigQuery import jobs when using the
same create and write dispositions.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>project_id</strong> – The project id owning the table.</li>
<li><strong>dataset_id</strong> – The dataset id owning the table.</li>
<li><strong>table_id</strong> – The table id.</li>
<li><strong>schema</strong> – A bigquery.TableSchema instance or None.</li>
<li><strong>create_disposition</strong> – CREATE_NEVER or CREATE_IF_NEEDED.</li>
<li><strong>write_disposition</strong> – WRITE_APPEND, WRITE_EMPTY or WRITE_TRUNCATE.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">A bigquery.Table instance if table was found or created.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Raises:</th><td class="field-body"><p class="first last"><cite>RuntimeError</cite> – For various mismatches between the state of the table and
the create/write dispositions passed in. For example if the table is not
empty and WRITE_EMPTY was specified then an error will be raised since
the table was expected to be empty.</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.run_query">
<code class="descname">run_query</code><span class="sig-paren">(</span><em>project_id</em>, <em>query</em>, <em>use_legacy_sql</em>, <em>flatten_results</em>, <em>priority</em>, <em>dry_run=False</em>, <em>job_labels=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.run_query"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.run_query" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.insert_rows">
<code class="descname">insert_rows</code><span class="sig-paren">(</span><em>project_id</em>, <em>dataset_id</em>, <em>table_id</em>, <em>rows</em>, <em>insert_ids=None</em>, <em>skip_invalid_rows=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.insert_rows"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.insert_rows" title="Permalink to this definition"></a></dt>
<dd><p>Inserts rows into the specified table.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>project_id</strong> – The project id owning the table.</li>
<li><strong>dataset_id</strong> – The dataset id owning the table.</li>
<li><strong>table_id</strong> – The table id.</li>
<li><strong>rows</strong> – A list of plain Python dictionaries. Each dictionary is a row and
each key in it is the name of a field.</li>
<li><strong>skip_invalid_rows</strong> – If there are rows with insertion errors, whether they
should be skipped, and all others should be inserted successfully.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first last">A tuple (bool, errors). If first element is False then the second element
will be a bigquery.InserttErrorsValueListEntry instance containing
specific errors.</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.convert_row_to_dict">
<code class="descname">convert_row_to_dict</code><span class="sig-paren">(</span><em>row</em>, <em>schema</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.convert_row_to_dict"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.convert_row_to_dict" title="Permalink to this definition"></a></dt>
<dd><p>Converts a TableRow instance using the schema to a Python dict.</p>
</dd></dl>
</dd></dl>
<dl class="class">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryReader">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">BigQueryReader</code><span class="sig-paren">(</span><em>source</em>, <em>test_bigquery_client=None</em>, <em>use_legacy_sql=True</em>, <em>flatten_results=True</em>, <em>kms_key=None</em>, <em>query_priority=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryReader"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryReader" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference internal" href="apache_beam.runners.dataflow.native_io.iobase.html#apache_beam.runners.dataflow.native_io.iobase.NativeSourceReader" title="apache_beam.runners.dataflow.native_io.iobase.NativeSourceReader"><code class="xref py py-class docutils literal notranslate"><span class="pre">apache_beam.runners.dataflow.native_io.iobase.NativeSourceReader</span></code></a></p>
<p>A reader for a BigQuery source.</p>
</dd></dl>
<dl class="class">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWriter">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">BigQueryWriter</code><span class="sig-paren">(</span><em>sink</em>, <em>test_bigquery_client=None</em>, <em>buffer_size=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWriter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWriter" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference internal" href="apache_beam.runners.dataflow.native_io.iobase.html#apache_beam.runners.dataflow.native_io.iobase.NativeSinkWriter" title="apache_beam.runners.dataflow.native_io.iobase.NativeSinkWriter"><code class="xref py py-class docutils literal notranslate"><span class="pre">apache_beam.runners.dataflow.native_io.iobase.NativeSinkWriter</span></code></a></p>
<p>The sink writer for a BigQuerySink.</p>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryWriter.Write">
<code class="descname">Write</code><span class="sig-paren">(</span><em>row</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWriter.Write"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWriter.Write" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
<dl class="class">
<dt id="apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">RowAsDictJsonCoder</code><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#RowAsDictJsonCoder"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference internal" href="apache_beam.coders.coders.html#apache_beam.coders.coders.Coder" title="apache_beam.coders.coders.Coder"><code class="xref py py-class docutils literal notranslate"><span class="pre">apache_beam.coders.coders.Coder</span></code></a></p>
<p>A coder for a table row (represented as a dict) to/from a JSON string.</p>
<p>This is the default coder for sources and sinks if the coder argument is not
specified.</p>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder.encode">
<code class="descname">encode</code><span class="sig-paren">(</span><em>table_row</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#RowAsDictJsonCoder.encode"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder.encode" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder.decode">
<code class="descname">decode</code><span class="sig-paren">(</span><em>encoded_table_row</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#RowAsDictJsonCoder.decode"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder.decode" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder.to_type_hint">
<code class="descname">to_type_hint</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#RowAsDictJsonCoder.to_type_hint"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder.to_type_hint" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
<dl class="class">
<dt id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">JsonRowWriter</code><span class="sig-paren">(</span><em>file_handle</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/io.html#io.IOBase" title="(in Python v3.10)"><code class="xref py py-class docutils literal notranslate"><span class="pre">io.IOBase</span></code></a></p>
<p>A writer which provides an IOBase-like interface for writing table rows
(represented as dicts) as newline-delimited JSON strings.</p>
<p>Initialize an JsonRowWriter.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>file_handle</strong> (<a class="reference external" href="https://docs.python.org/3/library/io.html#io.IOBase" title="(in Python v3.10)"><em>io.IOBase</em></a>) – Output stream to write to.</td>
</tr>
</tbody>
</table>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.close">
<code class="descname">close</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter.close"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.close" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.closed">
<code class="descname">closed</code><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.closed" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.flush">
<code class="descname">flush</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter.flush"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.flush" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.read">
<code class="descname">read</code><span class="sig-paren">(</span><em>size=-1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter.read"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.read" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.tell">
<code class="descname">tell</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter.tell"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.tell" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.writable">
<code class="descname">writable</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter.writable"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.writable" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.write">
<code class="descname">write</code><span class="sig-paren">(</span><em>row</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter.write"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.write" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
<dl class="class">
<dt id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">AvroRowWriter</code><span class="sig-paren">(</span><em>file_handle</em>, <em>schema</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/io.html#io.IOBase" title="(in Python v3.10)"><code class="xref py py-class docutils literal notranslate"><span class="pre">io.IOBase</span></code></a></p>
<p>A writer which provides an IOBase-like interface for writing table rows
(represented as dicts) as Avro records.</p>
<p>Initialize an AvroRowWriter.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>file_handle</strong> (<a class="reference external" href="https://docs.python.org/3/library/io.html#io.IOBase" title="(in Python v3.10)"><em>io.IOBase</em></a>) – Output stream to write Avro records to.</li>
<li><strong>schema</strong> (<em>Dict</em><em>[</em><em>Text</em><em>, </em><em>Any</em><em>]</em>) – BigQuery table schema.</li>
</ul>
</td>
</tr>
</tbody>
</table>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.close">
<code class="descname">close</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter.close"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.close" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.closed">
<code class="descname">closed</code><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.closed" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.flush">
<code class="descname">flush</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter.flush"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.flush" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.read">
<code class="descname">read</code><span class="sig-paren">(</span><em>size=-1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter.read"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.read" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.tell">
<code class="descname">tell</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter.tell"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.tell" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.writable">
<code class="descname">writable</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter.writable"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.writable" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.write">
<code class="descname">write</code><span class="sig-paren">(</span><em>row</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter.write"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.write" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
<dl class="class">
<dt id="apache_beam.io.gcp.bigquery_tools.RetryStrategy">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">RetryStrategy</code><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#RetryStrategy"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RetryStrategy" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.10)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.RetryStrategy.RETRY_ALWAYS">
<code class="descname">RETRY_ALWAYS</code><em class="property"> = 'RETRY_ALWAYS'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RetryStrategy.RETRY_ALWAYS" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.RetryStrategy.RETRY_NEVER">
<code class="descname">RETRY_NEVER</code><em class="property"> = 'RETRY_NEVER'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RetryStrategy.RETRY_NEVER" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.RetryStrategy.RETRY_ON_TRANSIENT_ERROR">
<code class="descname">RETRY_ON_TRANSIENT_ERROR</code><em class="property"> = 'RETRY_ON_TRANSIENT_ERROR'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RetryStrategy.RETRY_ON_TRANSIENT_ERROR" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="staticmethod">
<dt id="apache_beam.io.gcp.bigquery_tools.RetryStrategy.should_retry">
<em class="property">static </em><code class="descname">should_retry</code><span class="sig-paren">(</span><em>strategy</em>, <em>error_message</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#RetryStrategy.should_retry"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RetryStrategy.should_retry" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
<dl class="class">
<dt id="apache_beam.io.gcp.bigquery_tools.AppendDestinationsFn">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">AppendDestinationsFn</code><span class="sig-paren">(</span><em>destination</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AppendDestinationsFn"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AppendDestinationsFn" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference internal" href="apache_beam.transforms.core.html#apache_beam.transforms.core.DoFn" title="apache_beam.transforms.core.DoFn"><code class="xref py py-class docutils literal notranslate"><span class="pre">apache_beam.transforms.core.DoFn</span></code></a></p>
<p>Adds the destination to an element, making it a KV pair.</p>
<p>Outputs a PCollection of KV-pairs where the key is a TableReference for the
destination, and the value is the record itself.</p>
<p>Experimental; no backwards compatibility guarantees.</p>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.AppendDestinationsFn.display_data">
<code class="descname">display_data</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AppendDestinationsFn.display_data"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AppendDestinationsFn.display_data" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="apache_beam.io.gcp.bigquery_tools.AppendDestinationsFn.process">
<code class="descname">process</code><span class="sig-paren">(</span><em>element</em>, <em>*side_inputs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AppendDestinationsFn.process"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AppendDestinationsFn.process" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.bigquery_tools.get_table_schema_from_string">
<code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">get_table_schema_from_string</code><span class="sig-paren">(</span><em>schema</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#get_table_schema_from_string"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.get_table_schema_from_string" title="Permalink to this definition"></a></dt>
<dd><p>Transform the string table schema into a
<code class="xref py py-class docutils literal notranslate"><span class="pre">TableSchema</span></code> instance.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>schema</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><em>str</em></a>) – The sting schema to be used if the BigQuery table to write
has to be created.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">The schema to be used if the BigQuery table to write has to be created
but in the <code class="xref py py-class docutils literal notranslate"><span class="pre">TableSchema</span></code> format.</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">TableSchema</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.bigquery_tools.table_schema_to_dict">
<code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">table_schema_to_dict</code><span class="sig-paren">(</span><em>table_schema</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#table_schema_to_dict"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.table_schema_to_dict" title="Permalink to this definition"></a></dt>
<dd><p>Create a dictionary representation of table schema for serialization</p>
</dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.bigquery_tools.get_dict_table_schema">
<code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">get_dict_table_schema</code><span class="sig-paren">(</span><em>schema</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#get_dict_table_schema"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.get_dict_table_schema" title="Permalink to this definition"></a></dt>
<dd><p>Transform the table schema into a dictionary instance.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>schema</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.10)"><em>dict</em></a><em>, </em><em>TableSchema</em>) – The schema to be used if the BigQuery table to write has to be created.
This can either be a dict or string or in the TableSchema format.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">The schema to be used if the BigQuery table to write has
to be created but in the dictionary format.</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">Dict[<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)">str</a>, Any]</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.bigquery_tools.get_avro_schema_from_table_schema">
<code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">get_avro_schema_from_table_schema</code><span class="sig-paren">(</span><em>schema</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#get_avro_schema_from_table_schema"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.get_avro_schema_from_table_schema" title="Permalink to this definition"></a></dt>
<dd><p>Transform the table schema into an Avro schema.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>schema</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.10)"><em>dict</em></a><em>, </em><em>TableSchema</em>) – The TableSchema to convert to Avro schema. This can either be a dict or
string or in the TableSchema format.</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body">An Avro schema, which can be used by fastavro.</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body">Dict[<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.10)">str</a>, Any]</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="class">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes">
<em class="property">class </em><code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">BigQueryJobTypes</code><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryJobTypes"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.10)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.EXPORT">
<code class="descname">EXPORT</code><em class="property"> = 'EXPORT'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.EXPORT" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.COPY">
<code class="descname">COPY</code><em class="property"> = 'COPY'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.COPY" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.LOAD">
<code class="descname">LOAD</code><em class="property"> = 'LOAD'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.LOAD" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.QUERY">
<code class="descname">QUERY</code><em class="property"> = 'QUERY'</em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.QUERY" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.bigquery_tools.generate_bq_job_name">
<code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">generate_bq_job_name</code><span class="sig-paren">(</span><em>job_name</em>, <em>step_id</em>, <em>job_type</em>, <em>random=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#generate_bq_job_name"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.generate_bq_job_name" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="function">
<dt id="apache_beam.io.gcp.bigquery_tools.check_schema_equal">
<code class="descclassname">apache_beam.io.gcp.bigquery_tools.</code><code class="descname">check_schema_equal</code><span class="sig-paren">(</span><em>left</em>, <em>right</em>, <em>*</em>, <em>ignore_descriptions=False</em>, <em>ignore_field_order=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#check_schema_equal"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.check_schema_equal" title="Permalink to this definition"></a></dt>
<dd><p>Check whether schemas are equivalent.</p>
<p>This comparison function differs from using == to compare TableSchema
because it ignores categories, policy tags, descriptions (optionally), and
field ordering (optionally).</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>left</strong> (<em>TableSchema</em><em>, </em><em>TableFieldSchema</em>) – One schema to compare.</li>
<li><strong>right</strong> (<em>TableSchema</em><em>, </em><em>TableFieldSchema</em>) – The other schema to compare.</li>
<li><strong>ignore_descriptions</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.10)"><em>bool</em></a>) – (optional) Whether or not to ignore field
descriptions when comparing. Defaults to False.</li>
<li><strong>ignore_field_order</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.10)"><em>bool</em></a>) – (optional) Whether or not to ignore struct field
order when comparing. Defaults to False.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">True if the schemas are equivalent, False otherwise.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last"><a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.10)">bool</a></p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="apache_beam.io.gcp.bigtableio.html" class="btn btn-neutral float-right" title="apache_beam.io.gcp.bigtableio module" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="apache_beam.io.gcp.bigquery_read_internal.html" class="btn btn-neutral float-left" title="apache_beam.io.gcp.bigquery_read_internal module" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>