Google Git
Sign in
apache / airflow-site / f81e48aadee354653679693fa54b04ac42d24bfd / . / docs-archive / 1.10.6 / _api / airflow / contrib / operators / sql_to_gcs / index.html
blob: fecd3d12e99d9085c678053daf3e4d93e9719ff3 [file] [log] [blame]
<!--
Javascript to render AIRFLOW-XXX and PR references in text
as HTML links.
Overrides extrahead block from sphinx_rtd_theme
https://www.sphinx-doc.org/en/master/templating.html
-->
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>airflow.contrib.operators.sql_to_gcs &mdash; Airflow Documentation</title>
<link rel="shortcut icon" href="../../../../../_static/pin_32.png"/>
<script type="text/javascript" src="../../../../../_static/js/modernizr.min.js"></script>
<script type="text/javascript" id="documentation_options" data-url_root="../../../../../" src="../../../../../_static/documentation_options.js"></script>
<script type="text/javascript" src="../../../../../_static/jquery.js"></script>
<script type="text/javascript" src="../../../../../_static/underscore.js"></script>
<script type="text/javascript" src="../../../../../_static/doctools.js"></script>
<script type="text/javascript" src="../../../../../_static/language_data.js"></script>
<script type="text/javascript" src="../../../../../_static/js/theme.js"></script>
<link rel="stylesheet" href="../../../../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../../../../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../../../../../_static/graphviz.css" type="text/css" />
<link rel="index" title="Index" href="../../../../../genindex.html" />
<link rel="search" title="Search" href="../../../../../search.html" />
<link rel="next" title="airflow.contrib.operators.sqoop_operator" href="../sqoop_operator/index.html" />
<link rel="prev" title="airflow.contrib.operators.spark_submit_operator" href="../spark_submit_operator/index.html" />
<script>
document.addEventListener('DOMContentLoaded', function() {
var el = document.getElementById('changelog');
if (el !== null ) {
// [AIRFLOW-...]
el.innerHTML = el.innerHTML.replace(
/\[(AIRFLOW-[\d]+)\]/g,
`<a href="https://issues.apache.org/jira/browse/$1">[$1]</a>`
);
// (#...)
el.innerHTML = el.innerHTML.replace(
/\(#([\d]+)\)/g,
`<a href="https://github.com/apache/airflow/pull/$1">(#$1)</a>`
);
};
})
</script>
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-140539454-1']);
_gaq.push(['_trackPageview']);
</script>
<style>
.example-header {
position: relative;
background: #9AAA7A;
padding: 8px 16px;
margin-bottom: 0;
}
.example-header--with-button {
padding-right: 166px;
}
.example-header:after{
content: '';
display: table;
clear: both;
}
.example-title {
display:block;
padding: 4px;
margin-right: 16px;
color: white;
overflow-x: auto;
}
.example-header-button {
top: 8px;
right: 16px;
position: absolute;
}
.example-header + .highlight-python {
margin-top: 0 !important;
}
.viewcode-button {
display: inline-block;
padding: 8px 16px;
border: 0;
margin: 0;
outline: 0;
border-radius: 2px;
-webkit-box-shadow: 0 3px 5px 0 rgba(0,0,0,.3);
box-shadow: 0 3px 6px 0 rgba(0,0,0,.3);
color: #404040;
background-color: #e7e7e7;
cursor: pointer;
font-size: 16px;
font-weight: 500;
line-height: 1;
text-decoration: none;
text-overflow: ellipsis;
overflow: hidden;
text-transform: uppercase;
-webkit-transition: background-color .2s;
transition: background-color .2s;
vertical-align: middle;
white-space: nowrap;
}
.viewcode-button:visited {
color: #404040;
}
.viewcode-button:hover, .viewcode-button:focus {
color: #404040;
background-color: #d6d6d6;
}
</style>
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="../../../../../index.html" class="icon icon-home"> Airflow
</a>
<div class="version">
1.10.6
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../../../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../../../../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../profiling.html">Data Profiling</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../cli.html">Command Line Interface Reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../security.html">Security</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../api.html">REST API Reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../metrics.html">Metrics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../errors.html">Error Tracking</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../kubernetes.html">Kubernetes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../changelog.html">Changelog</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../macros.html">Macros reference</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../../../../index.html">Python API Reference</a><ul class="current">
<li class="toctree-l2 current"><a class="reference internal" href="../../../../index.html#operators">Operators</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="../../../../index.html#baseoperator">BaseOperator</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../../index.html#basesensoroperator">BaseSensorOperator</a></li>
<li class="toctree-l3 current"><a class="reference internal" href="../../../../index.html#operators-packages">Operators packages</a><ul class="current">
<li class="toctree-l4"><a class="reference internal" href="../../../operators/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.operators</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../sensors/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.sensors</span></code></a></li>
<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.operators</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../sensors/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.sensors</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#hooks">Hooks</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#executors">Executors</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#models">Models</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#core-and-community-package">Core and community package</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../../../../index.html">Python API</a><ul class="current">
<li class="toctree-l2 current"><a class="reference internal" href="../../../../index.html#operators">Operators</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="../../../../index.html#baseoperator">BaseOperator</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../../index.html#basesensoroperator">BaseSensorOperator</a></li>
<li class="toctree-l3 current"><a class="reference internal" href="../../../../index.html#operators-packages">Operators packages</a><ul class="current">
<li class="toctree-l4"><a class="reference internal" href="../../../operators/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.operators</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../sensors/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.sensors</span></code></a></li>
<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.operators</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../sensors/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.sensors</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#hooks">Hooks</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#executors">Executors</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#models">Models</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#core-and-community-package">Core and community package</a></li>
</ul>
</li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../../index.html">Airflow</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../../../../index.html">Docs</a> &raquo;</li>
<li><a href="../../../../index.html">Python API Reference</a> &raquo;</li>
<li><a href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.operators</span></code></a> &raquo;</li>
<li><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.operators.sql_to_gcs</span></code></li>
<li class="wy-breadcrumbs-aside">
<a href="../../../../../_sources/_api/airflow/contrib/operators/sql_to_gcs/index.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="module-airflow.contrib.operators.sql_to_gcs">
<span id="airflow-contrib-operators-sql-to-gcs"></span><h1><a class="reference internal" href="#module-airflow.contrib.operators.sql_to_gcs" title="airflow.contrib.operators.sql_to_gcs"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.operators.sql_to_gcs</span></code></a><a class="headerlink" href="#module-airflow.contrib.operators.sql_to_gcs" title="Permalink to this headline"></a></h1>
<p><p>Base operator for SQL to GCS operators.</p>
</p>
<div class="section" id="module-contents">
<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline"></a></h2>
<dl class="data">
<dt id="airflow.contrib.operators.sql_to_gcs.PY3">
<code class="sig-prename descclassname">airflow.contrib.operators.sql_to_gcs.</code><code class="sig-name descname">PY3</code><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#PY3"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.PY3" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="class">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator">
<em class="property">class </em><code class="sig-prename descclassname">airflow.contrib.operators.sql_to_gcs.</code><code class="sig-name descname">BaseSQLToGoogleCloudStorageOperator</code><span class="sig-paren">(</span><em class="sig-param">sql</em>, <em class="sig-param">bucket</em>, <em class="sig-param">filename</em>, <em class="sig-param">schema_filename=None</em>, <em class="sig-param">approx_max_file_size_bytes=1900000000</em>, <em class="sig-param">export_format='json'</em>, <em class="sig-param">field_delimiter='</em>, <em class="sig-param">'</em>, <em class="sig-param">gzip=False</em>, <em class="sig-param">schema=None</em>, <em class="sig-param">parameters=None</em>, <em class="sig-param">google_cloud_storage_conn_id='google_cloud_default'</em>, <em class="sig-param">delegate_to=None</em>, <em class="sig-param">*args</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference internal" href="../../../models/index.html#airflow.models.BaseOperator" title="airflow.models.BaseOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.models.BaseOperator</span></code></a></p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>sql</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – The SQL to execute.</p></li>
<li><p><strong>bucket</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – The bucket to upload to.</p></li>
<li><p><strong>filename</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – The filename to use as the object name when uploading
to Google Cloud Storage. A {} should be specified in the filename
to allow the operator to inject file numbers in cases where the
file is split due to size.</p></li>
<li><p><strong>schema_filename</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – If set, the filename to use as the object name
when uploading a .json file containing the BigQuery schema fields
for the table that was dumped from the database.</p></li>
<li><p><strong>approx_max_file_size_bytes</strong> (<em>long</em>) – This operator supports the ability
to split large table dumps into multiple files (see notes in the
filename param docs above). This param allows developers to specify the
file size of the splits. Check <a class="reference external" href="https://cloud.google.com/storage/quotas">https://cloud.google.com/storage/quotas</a>
to see the maximum allowed file size for a single object.</p></li>
<li><p><strong>export_format</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – Desired format of files to be exported.</p></li>
<li><p><strong>field_delimiter</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – The delimiter to be used for CSV files.</p></li>
<li><p><strong>gzip</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a>) – Option to compress file for upload (does not apply to schemas).</p></li>
<li><p><strong>schema</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a>) – The schema to use, if any. Should be a list of dict or
a str. Pass a string if using Jinja template, otherwise, pass a list of
dict. Examples could be seen: <a class="reference external" href="https://cloud.google.com/bigquery/docs">https://cloud.google.com/bigquery/docs</a>
/schemas#specifying_a_json_schema_file</p></li>
<li><p><strong>google_cloud_storage_conn_id</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – Reference to a specific Google
cloud storage hook.</p></li>
<li><p><strong>delegate_to</strong> – The account to impersonate, if any. For this to
work, the service account making the request must have domain-wide
delegation enabled.</p></li>
<li><p><strong>parameters</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.8)"><em>dict</em></a>) – a parameters dict that is substituted at query runtime.</p></li>
</ul>
</dd>
</dl>
<dl class="attribute">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.template_fields">
<code class="sig-name descname">template_fields</code><em class="property"> = ['sql', 'bucket', 'filename', 'schema_filename', 'schema', 'parameters']</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator.template_fields"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.template_fields" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.template_ext">
<code class="sig-name descname">template_ext</code><em class="property"> = ['.sql']</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator.template_ext"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.template_ext" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.ui_color">
<code class="sig-name descname">ui_color</code><em class="property"> = #a0e08c</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator.ui_color"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.ui_color" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.execute">
<code class="sig-name descname">execute</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">context</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator.execute"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.execute" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.convert_types">
<code class="sig-name descname">convert_types</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">schema</em>, <em class="sig-param">col_type_dict</em>, <em class="sig-param">row</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator.convert_types"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.convert_types" title="Permalink to this definition"></a></dt>
<dd><p>Convert values from DBAPI to output-friendly formats.</p>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator._write_local_data_files">
<code class="sig-name descname">_write_local_data_files</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">cursor</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator._write_local_data_files"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator._write_local_data_files" title="Permalink to this definition"></a></dt>
<dd><p>Takes a cursor, and writes results to a local file.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>A dictionary where keys are filenames to be used as object
names in GCS, and values are file handles to local files that
contain the data for the GCS objects.</p>
</dd>
</dl>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator._configure_csv_file">
<code class="sig-name descname">_configure_csv_file</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">file_handle</em>, <em class="sig-param">schema</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator._configure_csv_file"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator._configure_csv_file" title="Permalink to this definition"></a></dt>
<dd><p>Configure a csv writer with the file_handle and write schema
as headers for the new file.</p>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.query">
<code class="sig-name descname">query</code><span class="sig-paren">(</span><em class="sig-param">self</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator.query"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.query" title="Permalink to this definition"></a></dt>
<dd><p>Execute DBAPI query.</p>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.field_to_bigquery">
<code class="sig-name descname">field_to_bigquery</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">field</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator.field_to_bigquery"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.field_to_bigquery" title="Permalink to this definition"></a></dt>
<dd><p>Convert a DBAPI field to BigQuery schema format.</p>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.convert_type">
<code class="sig-name descname">convert_type</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">value</em>, <em class="sig-param">schema_type</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator.convert_type"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator.convert_type" title="Permalink to this definition"></a></dt>
<dd><p>Convert a value from DBAPI to output-friendly formats.</p>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator._get_col_type_dict">
<code class="sig-name descname">_get_col_type_dict</code><span class="sig-paren">(</span><em class="sig-param">self</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator._get_col_type_dict"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator._get_col_type_dict" title="Permalink to this definition"></a></dt>
<dd><p>Return a dict of column name and column type based on self.schema if not None.</p>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator._write_local_schema_file">
<code class="sig-name descname">_write_local_schema_file</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">cursor</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator._write_local_schema_file"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator._write_local_schema_file" title="Permalink to this definition"></a></dt>
<dd><p>Takes a cursor, and writes the BigQuery schema for the results to a
local file system.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>A dictionary where key is a filename to be used as an object
name in GCS, and values are file handles to local files that
contains the BigQuery schema fields in .json format.</p>
</dd>
</dl>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator._upload_to_gcs">
<code class="sig-name descname">_upload_to_gcs</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">files_to_upload</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/sql_to_gcs.html#BaseSQLToGoogleCloudStorageOperator._upload_to_gcs"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator._upload_to_gcs" title="Permalink to this definition"></a></dt>
<dd><p>Upload all of the file splits (and optionally the schema .json file) to
Google cloud storage.</p>
</dd></dl>
</dd></dl>
</div>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="../sqoop_operator/index.html" class="btn btn-neutral float-right" title="airflow.contrib.operators.sqoop_operator" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="../spark_submit_operator/index.html" class="btn btn-neutral float-left" title="airflow.contrib.operators.spark_submit_operator" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
<div class="footer">This page uses <a href="https://analytics.google.com/">
Google Analytics</a> to collect statistics. You can disable it by blocking
the JavaScript coming from www.google-analytics.com. Check our
<a href="../../../../../privacy_notice.html">Privacy Policy</a>
for more details.
<script type="text/javascript">
(function() {
var ga = document.createElement('script');
ga.src = ('https:' == document.location.protocol ?
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
ga.setAttribute('async', 'true');
var nodes = document.documentElement.childNodes;
var i = -1;
var node;
do {
i++;
node = nodes[i]
} while(node.nodeType !== Node.ELEMENT_NODE);
node.appendChild(ga);
})();
</script>
</div>
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>