| |
| |
| <!DOCTYPE html> |
| <html class="writer-html5" lang="en" data-content_root="./"> |
| <head> |
| <meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" /> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> |
| <title>apache_beam.io.gcp.bigquery_tools module — Apache Beam 2.67.0 documentation</title> |
| <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" /> |
| <link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" /> |
| |
| |
| <script src="_static/jquery.js?v=5d32c60e"></script> |
| <script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script> |
| <script src="_static/documentation_options.js?v=959b4fbe"></script> |
| <script src="_static/doctools.js?v=9a2dae69"></script> |
| <script src="_static/sphinx_highlight.js?v=dc90522c"></script> |
| <script src="_static/js/theme.js"></script> |
| <link rel="index" title="Index" href="genindex.html" /> |
| <link rel="search" title="Search" href="search.html" /> |
| <link rel="next" title="apache_beam.io.gcp.bigtableio module" href="apache_beam.io.gcp.bigtableio.html" /> |
| <link rel="prev" title="apache_beam.io.gcp.bigquery_schema_tools module" href="apache_beam.io.gcp.bigquery_schema_tools.html" /> |
| </head> |
| |
| <body class="wy-body-for-nav"> |
| <div class="wy-grid-for-nav"> |
| <nav data-toggle="wy-nav-shift" class="wy-nav-side"> |
| <div class="wy-side-scroll"> |
| <div class="wy-side-nav-search" > |
| |
| |
| |
| <a href="index.html" class="icon icon-home"> |
| Apache Beam |
| </a> |
| <div role="search"> |
| <form id="rtd-search-form" class="wy-form" action="search.html" method="get"> |
| <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" /> |
| <input type="hidden" name="check_keywords" value="yes" /> |
| <input type="hidden" name="area" value="default" /> |
| </form> |
| </div> |
| </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu"> |
| <ul class="current"> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.coders.html">apache_beam.coders package</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.dataframe.html">apache_beam.dataframe package</a></li> |
| <li class="toctree-l1 current"><a class="reference internal" href="apache_beam.io.html">apache_beam.io package</a><ul class="current"> |
| <li class="toctree-l2 current"><a class="reference internal" href="apache_beam.io.html#subpackages">Subpackages</a><ul class="current"> |
| <li class="toctree-l3"><a class="reference internal" href="apache_beam.io.aws.html">apache_beam.io.aws package</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="apache_beam.io.azure.html">apache_beam.io.azure package</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="apache_beam.io.components.html">apache_beam.io.components package</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="apache_beam.io.external.html">apache_beam.io.external package</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="apache_beam.io.flink.html">apache_beam.io.flink package</a></li> |
| <li class="toctree-l3 current"><a class="reference internal" href="apache_beam.io.gcp.html">apache_beam.io.gcp package</a><ul class="current"> |
| <li class="toctree-l4"><a class="reference internal" href="apache_beam.io.gcp.html#subpackages">Subpackages</a></li> |
| <li class="toctree-l4 current"><a class="reference internal" href="apache_beam.io.gcp.html#submodules">Submodules</a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="apache_beam.io.html#submodules">Submodules</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.metrics.html">apache_beam.metrics package</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.ml.html">apache_beam.ml package</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.options.html">apache_beam.options package</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.portability.html">apache_beam.portability package</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.runners.html">apache_beam.runners package</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.testing.html">apache_beam.testing package</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.transforms.html">apache_beam.transforms package</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.typehints.html">apache_beam.typehints package</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.utils.html">apache_beam.utils package</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.yaml.html">apache_beam.yaml package</a></li> |
| </ul> |
| <ul> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.error.html">apache_beam.error module</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.pipeline.html">apache_beam.pipeline module</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="apache_beam.pvalue.html">apache_beam.pvalue module</a></li> |
| </ul> |
| |
| </div> |
| </div> |
| </nav> |
| |
| <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" > |
| <i data-toggle="wy-nav-top" class="fa fa-bars"></i> |
| <a href="index.html">Apache Beam</a> |
| </nav> |
| |
| <div class="wy-nav-content"> |
| <div class="rst-content"> |
| <div role="navigation" aria-label="Page navigation"> |
| <ul class="wy-breadcrumbs"> |
| <li><a href="index.html" class="icon icon-home" aria-label="Home"></a></li> |
| <li class="breadcrumb-item"><a href="apache_beam.io.html">apache_beam.io package</a></li> |
| <li class="breadcrumb-item"><a href="apache_beam.io.gcp.html">apache_beam.io.gcp package</a></li> |
| <li class="breadcrumb-item active">apache_beam.io.gcp.bigquery_tools module</li> |
| <li class="wy-breadcrumbs-aside"> |
| <a href="_sources/apache_beam.io.gcp.bigquery_tools.rst.txt" rel="nofollow"> View page source</a> |
| </li> |
| </ul> |
| <hr/> |
| </div> |
| <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> |
| <div itemprop="articleBody"> |
| |
| <section id="module-apache_beam.io.gcp.bigquery_tools"> |
| <span id="apache-beam-io-gcp-bigquery-tools-module"></span><h1>apache_beam.io.gcp.bigquery_tools module<a class="headerlink" href="#module-apache_beam.io.gcp.bigquery_tools" title="Link to this heading"></a></h1> |
| <p>Tools used by BigQuery sources and sinks.</p> |
| <p>Classes, constants and functions in this file are experimental and have no |
| backwards compatibility guarantees.</p> |
| <p>These tools include wrappers and clients to interact with BigQuery APIs.</p> |
| <p>NOTHING IN THIS FILE HAS BACKWARDS COMPATIBILITY GUARANTEES.</p> |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.FileFormat"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">FileFormat</span></span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#FileFormat"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.FileFormat" title="Link to this definition"></a></dt> |
| <dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.13)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p> |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.FileFormat.CSV"> |
| <span class="sig-name descname"><span class="pre">CSV</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'CSV'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.FileFormat.CSV" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.FileFormat.JSON"> |
| <span class="sig-name descname"><span class="pre">JSON</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'NEWLINE_DELIMITED_JSON'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.FileFormat.JSON" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.FileFormat.AVRO"> |
| <span class="sig-name descname"><span class="pre">AVRO</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'AVRO'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.FileFormat.AVRO" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.ExportCompression"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">ExportCompression</span></span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#ExportCompression"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.ExportCompression" title="Link to this definition"></a></dt> |
| <dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.13)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p> |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.ExportCompression.GZIP"> |
| <span class="sig-name descname"><span class="pre">GZIP</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'GZIP'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.ExportCompression.GZIP" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.ExportCompression.DEFLATE"> |
| <span class="sig-name descname"><span class="pre">DEFLATE</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'DEFLATE'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.ExportCompression.DEFLATE" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.ExportCompression.SNAPPY"> |
| <span class="sig-name descname"><span class="pre">SNAPPY</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'SNAPPY'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.ExportCompression.SNAPPY" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.ExportCompression.NONE"> |
| <span class="sig-name descname"><span class="pre">NONE</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'NONE'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.ExportCompression.NONE" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.default_encoder"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">default_encoder</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">obj</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#default_encoder"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.default_encoder" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.get_hashable_destination"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">get_hashable_destination</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">destination</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#get_hashable_destination"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.get_hashable_destination" title="Link to this definition"></a></dt> |
| <dd><p>Parses a table reference into a (project, dataset, table) tuple.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>destination</strong> – Either a TableReference object from the bigquery API. |
| The object has the following attributes: projectId, datasetId, and |
| tableId. Or a string representing the destination containing |
| ‘PROJECT:DATASET.TABLE’.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>A string representing the destination containing |
| ‘PROJECT:DATASET.TABLE’.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.to_hashable_table_ref"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">to_hashable_table_ref</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">table_ref_elem_kv</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.Tuple" title="(in Python v3.13)"><span class="pre">Tuple</span></a><span class="p"><span class="pre">[</span></span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.13)"><span class="pre">str</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">TableReference</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">V</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.Tuple" title="(in Python v3.13)"><span class="pre">Tuple</span></a><span class="p"><span class="pre">[</span></span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.13)"><span class="pre">str</span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">V</span><span class="p"><span class="pre">]</span></span></span></span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#to_hashable_table_ref"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.to_hashable_table_ref" title="Link to this definition"></a></dt> |
| <dd><p>Turns the key of the input tuple to its string representation. The key |
| should be either a string or a TableReference.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>table_ref_elem_kv</strong> – A tuple of table reference and element.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>A tuple of string representation of input table and input element.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.parse_table_schema_from_json"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">parse_table_schema_from_json</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">schema_string</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#parse_table_schema_from_json"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.parse_table_schema_from_json" title="Link to this definition"></a></dt> |
| <dd><p>Parse the Table Schema provided as string.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>schema_string</strong> – String serialized table schema, should be a valid JSON.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>A TableSchema of the BigQuery export from either the Query or the Table.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.parse_table_reference"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">parse_table_reference</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">table</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dataset</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">project</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#parse_table_reference"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.parse_table_reference" title="Link to this definition"></a></dt> |
| <dd><p>Parses a table reference into a (project, dataset, table) tuple.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>table</strong> – The ID of the table. The ID must contain only letters |
| (a-z, A-Z), numbers (0-9), connectors (-_). If dataset argument is None |
| then the table argument must contain the entire table reference: |
| ‘DATASET.TABLE’ or ‘PROJECT:DATASET.TABLE’. This argument can be a |
| TableReference instance in which case dataset and project are |
| ignored and the reference is returned as a result. Additionally, for date |
| partitioned tables, appending ‘$YYYYmmdd’ to the table name is supported, |
| e.g. ‘DATASET.TABLE$YYYYmmdd’.</p></li> |
| <li><p><strong>dataset</strong> – The ID of the dataset containing this table or null if the table |
| reference is specified entirely by the table argument.</p></li> |
| <li><p><strong>project</strong> – The ID of the project containing this table or null if the table |
| reference is specified entirely by the table (and possibly dataset) |
| argument.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>A TableReference object from the bigquery API. The object has the following |
| attributes: projectId, datasetId, and tableId. |
| If the input is a TableReference object, a new object will be returned.</p> |
| </dd> |
| <dt class="field-odd">Raises<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><a class="reference external" href="https://docs.python.org/3/library/exceptions.html#ValueError" title="(in Python v3.13)"><strong>ValueError</strong></a> – if the table reference as a string does not match the expected |
| format.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">BigQueryWrapper</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">client</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">temp_dataset_id</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">temp_table_ref</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper" title="Link to this definition"></a></dt> |
| <dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.13)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p> |
| <p>BigQuery client wrapper with utilities for querying.</p> |
| <p>The wrapper is used to organize all the BigQuery integration points and |
| offer a common place where retry logic for failures can be controlled. |
| In addition, it offers various functions used both in sources and sinks |
| (e.g., find and create tables, query a table, etc.).</p> |
| <p>Note that client parameter in constructor is only for testing purposes and |
| should not be used in production code.</p> |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.TEMP_TABLE"> |
| <span class="sig-name descname"><span class="pre">TEMP_TABLE</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'beam_temp_table_'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.TEMP_TABLE" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.TEMP_DATASET"> |
| <span class="sig-name descname"><span class="pre">TEMP_DATASET</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'beam_temp_dataset_'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.TEMP_DATASET" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.HISTOGRAM_METRIC_LOGGER"> |
| <span class="sig-name descname"><span class="pre">HISTOGRAM_METRIC_LOGGER</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre"><apache_beam.internal.metrics.metric.MetricLogger</span> <span class="pre">object></span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.HISTOGRAM_METRIC_LOGGER" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py property"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.unique_row_id"> |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">unique_row_id</span></span><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.unique_row_id" title="Link to this definition"></a></dt> |
| <dd><p>Returns a unique row ID (str) used to avoid multiple insertions.</p> |
| <p>If the row ID is provided, BigQuery will make a best effort to not insert |
| the same row multiple times for fail and retry scenarios in which the insert |
| request may be issued several times. This comes into play for sinks executed |
| in a local runner.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>a unique row ID string</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_query_location"> |
| <span class="sig-name descname"><span class="pre">get_query_location</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">project_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">query</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">use_legacy_sql</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.get_query_location"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_query_location" title="Link to this definition"></a></dt> |
| <dd><p>Get the location of tables referenced in a query.</p> |
| <p>This method returns the location of the first available referenced |
| table for user in the query and depends on the BigQuery service to |
| provide error handling for queries that reference tables in multiple |
| locations.</p> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.wait_for_bq_job"> |
| <span class="sig-name descname"><span class="pre">wait_for_bq_job</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">job_reference</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">sleep_duration_sec</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">5</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_retries</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">0</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.wait_for_bq_job"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.wait_for_bq_job" title="Link to this definition"></a></dt> |
| <dd><p>Poll job until it is DONE.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>job_reference</strong> – bigquery.JobReference instance.</p></li> |
| <li><p><strong>sleep_duration_sec</strong> – Specifies the delay in seconds between retries.</p></li> |
| <li><p><strong>max_retries</strong> – The total number of times to retry. If equals to 0, |
| the function waits forever.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Raises<span class="colon">:</span></dt> |
| <dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/exceptions.html#RuntimeError" title="(in Python v3.13)"><strong>RuntimeError</strong></a> – If the job is FAILED or the number of retries has been |
| reached.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_table"> |
| <span class="sig-name descname"><span class="pre">get_table</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">project_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dataset_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_id</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.get_table"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_table" title="Link to this definition"></a></dt> |
| <dd><p>Lookup a table’s metadata object.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>client</strong> – bigquery.BigqueryV2 instance</p></li> |
| <li><p><strong>project_id</strong> – table lookup parameter</p></li> |
| <li><p><strong>dataset_id</strong> – table lookup parameter</p></li> |
| <li><p><strong>table_id</strong> – table lookup parameter</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>bigquery.Table instance</p> |
| </dd> |
| <dt class="field-odd">Raises<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>HttpError</strong> – if lookup failed.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_or_create_dataset"> |
| <span class="sig-name descname"><span class="pre">get_or_create_dataset</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">project_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dataset_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">location</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">labels</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.get_or_create_dataset"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_or_create_dataset" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_table_location"> |
| <span class="sig-name descname"><span class="pre">get_table_location</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">project_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dataset_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_id</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.get_table_location"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_table_location" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.is_user_configured_dataset"> |
| <span class="sig-name descname"><span class="pre">is_user_configured_dataset</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.is_user_configured_dataset"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.is_user_configured_dataset" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.create_temporary_dataset"> |
| <span class="sig-name descname"><span class="pre">create_temporary_dataset</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">project_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">location</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">labels</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.create_temporary_dataset"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.create_temporary_dataset" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.clean_up_temporary_dataset"> |
| <span class="sig-name descname"><span class="pre">clean_up_temporary_dataset</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">project_id</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.clean_up_temporary_dataset"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.clean_up_temporary_dataset" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_job"> |
| <span class="sig-name descname"><span class="pre">get_job</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">project</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">job_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">location</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.get_job"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_job" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.perform_load_job"> |
| <span class="sig-name descname"><span class="pre">perform_load_job</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">destination</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">job_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">source_uris</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">source_stream</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">write_disposition</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">create_disposition</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">additional_load_parameters</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">source_format</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">job_labels</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">load_job_project_id</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.perform_load_job"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.perform_load_job" title="Link to this definition"></a></dt> |
| <dd><p>Starts a job to load data into BigQuery.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>bigquery.JobReference with the information about the job that was started.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.perform_extract_job"> |
| <span class="sig-name descname"><span class="pre">perform_extract_job</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">destination</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">job_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_reference</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">destination_format</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">project</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">include_header</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compression</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'NONE'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">use_avro_logical_types</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">job_labels</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.perform_extract_job"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.perform_extract_job" title="Link to this definition"></a></dt> |
| <dd><p>Starts a job to export data from BigQuery.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Returns<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>bigquery.JobReference with the information about the job that was started.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_or_create_table"> |
| <span class="sig-name descname"><span class="pre">get_or_create_table</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">project_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dataset_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">create_disposition</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">write_disposition</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">additional_create_parameters</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.get_or_create_table"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.get_or_create_table" title="Link to this definition"></a></dt> |
| <dd><p>Gets or creates a table based on create and write dispositions.</p> |
| <p>The function mimics the behavior of BigQuery import jobs when using the |
| same create and write dispositions.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>project_id</strong> – The project id owning the table.</p></li> |
| <li><p><strong>dataset_id</strong> – The dataset id owning the table.</p></li> |
| <li><p><strong>table_id</strong> – The table id.</p></li> |
| <li><p><strong>schema</strong> – A bigquery.TableSchema instance or None.</p></li> |
| <li><p><strong>create_disposition</strong> – CREATE_NEVER or CREATE_IF_NEEDED.</p></li> |
| <li><p><strong>write_disposition</strong> – WRITE_APPEND, WRITE_EMPTY or WRITE_TRUNCATE.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>A bigquery.Table instance if table was found or created.</p> |
| </dd> |
| <dt class="field-odd">Raises<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><a class="reference external" href="https://docs.python.org/3/library/exceptions.html#RuntimeError" title="(in Python v3.13)"><strong>RuntimeError</strong></a> – For various mismatches between the state of the table and |
| the create/write dispositions passed in. For example if the table is not |
| empty and WRITE_EMPTY was specified then an error will be raised since |
| the table was expected to be empty.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.run_query"> |
| <span class="sig-name descname"><span class="pre">run_query</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">project_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">query</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">use_legacy_sql</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">flatten_results</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">priority</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dry_run</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">job_labels</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.run_query"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.run_query" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.insert_rows"> |
| <span class="sig-name descname"><span class="pre">insert_rows</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">project_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dataset_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">table_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">rows</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">insert_ids</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">skip_invalid_rows</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">ignore_unknown_values</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.insert_rows"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.insert_rows" title="Link to this definition"></a></dt> |
| <dd><p>Inserts rows into the specified table.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>project_id</strong> – The project id owning the table.</p></li> |
| <li><p><strong>dataset_id</strong> – The dataset id owning the table.</p></li> |
| <li><p><strong>table_id</strong> – The table id.</p></li> |
| <li><p><strong>rows</strong> – A list of plain Python dictionaries. Each dictionary is a row and |
| each key in it is the name of a field.</p></li> |
| <li><p><strong>skip_invalid_rows</strong> – If there are rows with insertion errors, whether they |
| should be skipped, and all others should be inserted successfully.</p></li> |
| <li><p><strong>ignore_unknown_values</strong> – Set this option to true to ignore unknown column |
| names. If the input rows contain columns that are not |
| part of the existing table’s schema, those columns are ignored, and |
| the rows are successfully inserted.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>A tuple (bool, errors). If first element is False then the second element |
| will be a bigquery.InsertErrorsValueListEntry instance containing |
| specific errors.</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.convert_row_to_dict"> |
| <span class="sig-name descname"><span class="pre">convert_row_to_dict</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">row</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.convert_row_to_dict"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.convert_row_to_dict" title="Link to this definition"></a></dt> |
| <dd><p>Converts a TableRow instance using the schema to a Python dict.</p> |
| </dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.from_pipeline_options"> |
| <em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">from_pipeline_options</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">pipeline_options</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="apache_beam.options.pipeline_options.html#apache_beam.options.pipeline_options.PipelineOptions" title="apache_beam.options.pipeline_options.PipelineOptions"><span class="pre">PipelineOptions</span></a></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryWrapper.from_pipeline_options"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryWrapper.from_pipeline_options" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">RowAsDictJsonCoder</span></span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#RowAsDictJsonCoder"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder" title="Link to this definition"></a></dt> |
| <dd><p>Bases: <a class="reference internal" href="apache_beam.coders.coders.html#apache_beam.coders.coders.Coder" title="apache_beam.coders.coders.Coder"><code class="xref py py-class docutils literal notranslate"><span class="pre">Coder</span></code></a></p> |
| <p>A coder for a table row (represented as a dict) to/from a JSON string.</p> |
| <p>This is the default coder for sources and sinks if the coder argument is not |
| specified.</p> |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder.encode"> |
| <span class="sig-name descname"><span class="pre">encode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">table_row</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#RowAsDictJsonCoder.encode"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder.encode" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder.decode"> |
| <span class="sig-name descname"><span class="pre">decode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">encoded_table_row</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#RowAsDictJsonCoder.decode"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder.decode" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder.to_type_hint"> |
| <span class="sig-name descname"><span class="pre">to_type_hint</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#RowAsDictJsonCoder.to_type_hint"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RowAsDictJsonCoder.to_type_hint" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">JsonRowWriter</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">file_handle</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter" title="Link to this definition"></a></dt> |
| <dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/io.html#io.IOBase" title="(in Python v3.13)"><code class="xref py py-class docutils literal notranslate"><span class="pre">IOBase</span></code></a></p> |
| <p>A writer which provides an IOBase-like interface for writing table rows |
| (represented as dicts) as newline-delimited JSON strings.</p> |
| <p>Initialize an JsonRowWriter.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>file_handle</strong> (<a class="reference external" href="https://docs.python.org/3/library/io.html#io.IOBase" title="(in Python v3.13)"><em>io.IOBase</em></a>) – Output stream to write to.</p> |
| </dd> |
| </dl> |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.close"> |
| <span class="sig-name descname"><span class="pre">close</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter.close"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.close" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py property"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.closed"> |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">closed</span></span><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.closed" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.flush"> |
| <span class="sig-name descname"><span class="pre">flush</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter.flush"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.flush" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.read"> |
| <span class="sig-name descname"><span class="pre">read</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">size</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">-1</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter.read"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.read" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.tell"> |
| <span class="sig-name descname"><span class="pre">tell</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter.tell"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.tell" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.writable"> |
| <span class="sig-name descname"><span class="pre">writable</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter.writable"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.writable" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.JsonRowWriter.write"> |
| <span class="sig-name descname"><span class="pre">write</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">row</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#JsonRowWriter.write"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.JsonRowWriter.write" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">AvroRowWriter</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">file_handle</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter" title="Link to this definition"></a></dt> |
| <dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/io.html#io.IOBase" title="(in Python v3.13)"><code class="xref py py-class docutils literal notranslate"><span class="pre">IOBase</span></code></a></p> |
| <p>A writer which provides an IOBase-like interface for writing table rows |
| (represented as dicts) as Avro records.</p> |
| <p>Initialize an AvroRowWriter.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>file_handle</strong> (<a class="reference external" href="https://docs.python.org/3/library/io.html#io.IOBase" title="(in Python v3.13)"><em>io.IOBase</em></a>) – Output stream to write Avro records to.</p></li> |
| <li><p><strong>schema</strong> (<em>Dict</em><em>[</em><em>Text</em><em>, </em><em>Any</em><em>]</em>) – BigQuery table schema.</p></li> |
| </ul> |
| </dd> |
| </dl> |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.close"> |
| <span class="sig-name descname"><span class="pre">close</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter.close"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.close" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py property"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.closed"> |
| <em class="property"><span class="pre">property</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">closed</span></span><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.closed" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.flush"> |
| <span class="sig-name descname"><span class="pre">flush</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter.flush"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.flush" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.read"> |
| <span class="sig-name descname"><span class="pre">read</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">size</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">-1</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter.read"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.read" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.tell"> |
| <span class="sig-name descname"><span class="pre">tell</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter.tell"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.tell" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.writable"> |
| <span class="sig-name descname"><span class="pre">writable</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter.writable"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.writable" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.AvroRowWriter.write"> |
| <span class="sig-name descname"><span class="pre">write</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">row</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AvroRowWriter.write"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AvroRowWriter.write" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.RetryStrategy"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">RetryStrategy</span></span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#RetryStrategy"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RetryStrategy" title="Link to this definition"></a></dt> |
| <dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.13)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p> |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.RetryStrategy.RETRY_ALWAYS"> |
| <span class="sig-name descname"><span class="pre">RETRY_ALWAYS</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'RETRY_ALWAYS'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RetryStrategy.RETRY_ALWAYS" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.RetryStrategy.RETRY_NEVER"> |
| <span class="sig-name descname"><span class="pre">RETRY_NEVER</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'RETRY_NEVER'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RetryStrategy.RETRY_NEVER" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.RetryStrategy.RETRY_ON_TRANSIENT_ERROR"> |
| <span class="sig-name descname"><span class="pre">RETRY_ON_TRANSIENT_ERROR</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'RETRY_ON_TRANSIENT_ERROR'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RetryStrategy.RETRY_ON_TRANSIENT_ERROR" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.RetryStrategy.should_retry"> |
| <em class="property"><span class="pre">static</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">should_retry</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">strategy</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">error_message</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#RetryStrategy.should_retry"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.RetryStrategy.should_retry" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.AppendDestinationsFn"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">AppendDestinationsFn</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">destination</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AppendDestinationsFn"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AppendDestinationsFn" title="Link to this definition"></a></dt> |
| <dd><p>Bases: <a class="reference internal" href="apache_beam.transforms.core.html#apache_beam.transforms.core.DoFn" title="apache_beam.transforms.core.DoFn"><code class="xref py py-class docutils literal notranslate"><span class="pre">DoFn</span></code></a></p> |
| <p>Adds the destination to an element, making it a KV pair.</p> |
| <p>Outputs a PCollection of KV-pairs where the key is a TableReference for the |
| destination, and the value is the record itself.</p> |
| <p>Experimental; no backwards compatibility guarantees.</p> |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.AppendDestinationsFn.display_data"> |
| <span class="sig-name descname"><span class="pre">display_data</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AppendDestinationsFn.display_data"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AppendDestinationsFn.display_data" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py method"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.AppendDestinationsFn.process"> |
| <span class="sig-name descname"><span class="pre">process</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">element</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">side_inputs</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#AppendDestinationsFn.process"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.AppendDestinationsFn.process" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.beam_row_from_dict"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">beam_row_from_dict</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">row</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.13)"><span class="pre">dict</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">schema</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#beam_row_from_dict"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.beam_row_from_dict" title="Link to this definition"></a></dt> |
| <dd><p>Converts a dictionary row to a Beam Row. |
| Nested records and lists are supported.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>row</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.13)"><em>dict</em></a>) – The row to convert.</p></li> |
| <li><p><strong>schema</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.13)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.13)"><em>dict</em></a><em>, </em><em>TableSchema</em>) – The table schema. Will be used to help convert the row.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>The converted row.</p> |
| </dd> |
| <dt class="field-odd">Return type<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><a class="reference internal" href="apache_beam.pvalue.html#apache_beam.pvalue.Row" title="apache_beam.pvalue.Row"><em>Row</em></a></p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.get_table_schema_from_string"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">get_table_schema_from_string</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">schema</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#get_table_schema_from_string"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.get_table_schema_from_string" title="Link to this definition"></a></dt> |
| <dd><p>Transform the string table schema into a |
| <code class="xref py py-class docutils literal notranslate"><span class="pre">TableSchema</span></code> instance.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>schema</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.13)"><em>str</em></a>) – The string schema to be used if the BigQuery table to write |
| has to be created.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>The schema to be used if the BigQuery table to write has to be created |
| but in the <code class="xref py py-class docutils literal notranslate"><span class="pre">TableSchema</span></code> format.</p> |
| </dd> |
| <dt class="field-odd">Return type<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><em>TableSchema</em></p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.table_schema_to_dict"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">table_schema_to_dict</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">table_schema</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#table_schema_to_dict"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.table_schema_to_dict" title="Link to this definition"></a></dt> |
| <dd><p>Create a dictionary representation of table schema for serialization</p> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.get_dict_table_schema"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">get_dict_table_schema</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">schema</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#get_dict_table_schema"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.get_dict_table_schema" title="Link to this definition"></a></dt> |
| <dd><p>Transform the table schema into a dictionary instance.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>schema</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.13)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.13)"><em>dict</em></a><em>, </em><em>TableSchema</em>) – The schema to be used if the BigQuery table to write has to be created. |
| This can either be a dict or string or in the TableSchema format.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>The schema to be used if the BigQuery table to write has |
| to be created but in the dictionary format.</p> |
| </dd> |
| <dt class="field-odd">Return type<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Dict[<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.13)">str</a>, Any]</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.get_bq_tableschema"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">get_bq_tableschema</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">schema</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#get_bq_tableschema"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.get_bq_tableschema" title="Link to this definition"></a></dt> |
| <dd><p>Convert the table schema to a TableSchema object.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>schema</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.13)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.13)"><em>dict</em></a><em>, </em><em>TableSchema</em>) – The schema to be used if the BigQuery table to write has to be created. |
| This can either be a dict or string or in the TableSchema format.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>The schema as a TableSchema object.</p> |
| </dd> |
| <dt class="field-odd">Return type<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><em>TableSchema</em></p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.get_avro_schema_from_table_schema"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">get_avro_schema_from_table_schema</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">schema</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#get_avro_schema_from_table_schema"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.get_avro_schema_from_table_schema" title="Link to this definition"></a></dt> |
| <dd><p>Transform the table schema into an Avro schema.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>schema</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.13)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.13)"><em>dict</em></a><em>, </em><em>TableSchema</em>) – The TableSchema to convert to Avro schema. This can either be a dict or |
| string or in the TableSchema format.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>An Avro schema, which can be used by fastavro.</p> |
| </dd> |
| <dt class="field-odd">Return type<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>Dict[<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.13)">str</a>, Any]</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.get_beam_typehints_from_tableschema"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">get_beam_typehints_from_tableschema</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">schema</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#get_beam_typehints_from_tableschema"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.get_beam_typehints_from_tableschema" title="Link to this definition"></a></dt> |
| <dd><p>Extracts Beam Python type hints from the schema.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><strong>schema</strong> (<em>TableSchema</em>) – The TableSchema to extract type hints from.</p> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>A list of type hints that describe the input schema. |
| Nested and repeated fields are supported.</p> |
| </dd> |
| <dt class="field-odd">Return type<span class="colon">:</span></dt> |
| <dd class="field-odd"><p>List[Tuple[<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.13)">str</a>, Any]]</p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| <dl class="py class"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes"> |
| <em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">BigQueryJobTypes</span></span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#BigQueryJobTypes"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes" title="Link to this definition"></a></dt> |
| <dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.13)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p> |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.EXPORT"> |
| <span class="sig-name descname"><span class="pre">EXPORT</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'EXPORT'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.EXPORT" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.COPY"> |
| <span class="sig-name descname"><span class="pre">COPY</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'COPY'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.COPY" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.LOAD"> |
| <span class="sig-name descname"><span class="pre">LOAD</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'LOAD'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.LOAD" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py attribute"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.QUERY"> |
| <span class="sig-name descname"><span class="pre">QUERY</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'QUERY'</span></em><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.BigQueryJobTypes.QUERY" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| </dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.generate_bq_job_name"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">generate_bq_job_name</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">job_name</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">step_id</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">job_type</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">random</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#generate_bq_job_name"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.generate_bq_job_name" title="Link to this definition"></a></dt> |
| <dd></dd></dl> |
| |
| <dl class="py function"> |
| <dt class="sig sig-object py" id="apache_beam.io.gcp.bigquery_tools.check_schema_equal"> |
| <span class="sig-prename descclassname"><span class="pre">apache_beam.io.gcp.bigquery_tools.</span></span><span class="sig-name descname"><span class="pre">check_schema_equal</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">left</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">TableSchema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">TableFieldSchema</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">right</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">TableSchema</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">TableFieldSchema</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">ignore_descriptions</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.13)"><span class="pre">bool</span></a></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">ignore_field_order</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.13)"><span class="pre">bool</span></a></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.13)"><span class="pre">bool</span></a></span></span><a class="reference internal" href="_modules/apache_beam/io/gcp/bigquery_tools.html#check_schema_equal"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#apache_beam.io.gcp.bigquery_tools.check_schema_equal" title="Link to this definition"></a></dt> |
| <dd><p>Check whether schemas are equivalent.</p> |
| <p>This comparison function differs from using == to compare TableSchema |
| because it ignores categories, policy tags, descriptions (optionally), and |
| field ordering (optionally).</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Parameters<span class="colon">:</span></dt> |
| <dd class="field-odd"><ul class="simple"> |
| <li><p><strong>left</strong> (<em>TableSchema</em><em>, </em><em>TableFieldSchema</em>) – One schema to compare.</p></li> |
| <li><p><strong>right</strong> (<em>TableSchema</em><em>, </em><em>TableFieldSchema</em>) – The other schema to compare.</p></li> |
| <li><p><strong>ignore_descriptions</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.13)"><em>bool</em></a>) – (optional) Whether or not to ignore field |
| descriptions when comparing. Defaults to False.</p></li> |
| <li><p><strong>ignore_field_order</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.13)"><em>bool</em></a>) – (optional) Whether or not to ignore struct field |
| order when comparing. Defaults to False.</p></li> |
| </ul> |
| </dd> |
| <dt class="field-even">Returns<span class="colon">:</span></dt> |
| <dd class="field-even"><p>True if the schemas are equivalent, False otherwise.</p> |
| </dd> |
| <dt class="field-odd">Return type<span class="colon">:</span></dt> |
| <dd class="field-odd"><p><a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.13)">bool</a></p> |
| </dd> |
| </dl> |
| </dd></dl> |
| |
| </section> |
| |
| |
| </div> |
| </div> |
| <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer"> |
| <a href="apache_beam.io.gcp.bigquery_schema_tools.html" class="btn btn-neutral float-left" title="apache_beam.io.gcp.bigquery_schema_tools module" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a> |
| <a href="apache_beam.io.gcp.bigtableio.html" class="btn btn-neutral float-right" title="apache_beam.io.gcp.bigtableio module" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a> |
| </div> |
| |
| <hr/> |
| |
| <div role="contentinfo"> |
| <p>© Copyright %Y, Apache Beam.</p> |
| </div> |
| |
| Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a |
| <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a> |
| provided by <a href="https://readthedocs.org">Read the Docs</a>. |
| |
| |
| </footer> |
| </div> |
| </div> |
| </section> |
| </div> |
| <script> |
| jQuery(function () { |
| SphinxRtdTheme.Navigation.enable(true); |
| }); |
| </script> |
| |
| </body> |
| </html> |
