blob: 0ed2cdcc7df906ebcf66bc5f05e5b6d10c81c758 [file] [log] [blame]
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>airflow.contrib.hooks.gcp_api_base_hook &mdash; Airflow Documentation</title>
<script type="text/javascript" src="../../../../../_static/js/modernizr.min.js"></script>
<script type="text/javascript" id="documentation_options" data-url_root="../../../../../" src="../../../../../_static/documentation_options.js"></script>
<script type="text/javascript" src="../../../../../_static/jquery.js"></script>
<script type="text/javascript" src="../../../../../_static/underscore.js"></script>
<script type="text/javascript" src="../../../../../_static/doctools.js"></script>
<script type="text/javascript" src="../../../../../_static/language_data.js"></script>
<script type="text/javascript" src="../../../../../_static/js/theme.js"></script>
<link rel="stylesheet" href="../../../../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../../../../_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="../../../../../genindex.html" />
<link rel="search" title="Search" href="../../../../../search.html" />
<link rel="next" title="airflow.contrib.hooks.gcp_bigtable_hook" href="../gcp_bigtable_hook/index.html" />
<link rel="prev" title="airflow.contrib.hooks.ftp_hook" href="../ftp_hook/index.html" />
<script>
document.addEventListener('DOMContentLoaded', function() {
var el = document.getElementById('changelog');
if (el !== null ) {
// [AIRFLOW-...]
el.innerHTML = el.innerHTML.replace(
/\[(AIRFLOW-[\d]+)\]/g,
`<a href="https://issues.apache.org/jira/browse/$1">[$1]</a>`
);
// (#...)
el.innerHTML = el.innerHTML.replace(
/\(#([\d]+)\)/g,
`<a href="https://github.com/apache/airflow/pull/$1">(#$1)</a>`
);
};
})
</script>
<style>
.example-header {
position: relative;
background: #9AAA7A;
padding: 8px 16px;
margin-bottom: 0;
}
.example-header--with-button {
padding-right: 166px;
}
.example-header:after{
content: '';
display: table;
clear: both;
}
.example-title {
display:block;
padding: 4px;
margin-right: 16px;
color: white;
overflow-x: auto;
}
.example-header-button {
top: 8px;
right: 16px;
position: absolute;
}
.example-header + .highlight-python {
margin-top: 0 !important;
}
.viewcode-button {
display: inline-block;
padding: 8px 16px;
border: 0;
margin: 0;
outline: 0;
border-radius: 2px;
-webkit-box-shadow: 0 3px 5px 0 rgba(0,0,0,.3);
box-shadow: 0 3px 6px 0 rgba(0,0,0,.3);
color: #404040;
background-color: #e7e7e7;
cursor: pointer;
font-size: 16px;
font-weight: 500;
line-height: 1;
text-decoration: none;
text-overflow: ellipsis;
overflow: hidden;
text-transform: uppercase;
-webkit-transition: background-color .2s;
transition: background-color .2s;
vertical-align: middle;
white-space: nowrap;
}
.viewcode-button:visited {
color: #404040;
}
.viewcode-button:hover, .viewcode-button:focus {
color: #404040;
background-color: #d6d6d6;
}
</style>
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="../../../../../index.html" class="icon icon-home"> Airflow
</a>
<div class="version">
1.10.4
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../../../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../../../../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../profiling.html">Data Profiling</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../cli.html">Command Line Interface</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../security.html">Security</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../metrics.html">Metrics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../kubernetes.html">Kubernetes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../changelog.html">Changelog</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../macros.html">Macros reference</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../../../../index.html">API Reference</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#operators">Operators</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="../../../../index.html#hooks">Hooks</a><ul class="current">
<li class="toctree-l3 current"><a class="reference internal" href="../../../../index.html#hooks-packages">Hooks packages</a><ul class="current">
<li class="toctree-l4"><a class="reference internal" href="../../../hooks/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.hooks</span></code></a></li>
<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.hooks</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#executors">Executors</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#models">Models</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#core-and-community-package">Core and community package</a></li>
</ul>
</li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../../index.html">Airflow</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../../../../index.html">Docs</a> &raquo;</li>
<li><a href="../../../../index.html">API Reference</a> &raquo;</li>
<li><a href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.hooks</span></code></a> &raquo;</li>
<li><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.hooks.gcp_api_base_hook</span></code></li>
<li class="wy-breadcrumbs-aside">
<a href="../../../../../_sources/_api/airflow/contrib/hooks/gcp_api_base_hook/index.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="module-airflow.contrib.hooks.gcp_api_base_hook">
<span id="airflow-contrib-hooks-gcp-api-base-hook"></span><h1><a class="reference internal" href="#module-airflow.contrib.hooks.gcp_api_base_hook" title="airflow.contrib.hooks.gcp_api_base_hook"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.hooks.gcp_api_base_hook</span></code></a><a class="headerlink" href="#module-airflow.contrib.hooks.gcp_api_base_hook" title="Permalink to this headline"></a></h1>
<div class="section" id="module-contents">
<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline"></a></h2>
<dl class="data">
<dt id="airflow.contrib.hooks.gcp_api_base_hook._DEFAULT_SCOPES">
<code class="sig-prename descclassname">airflow.contrib.hooks.gcp_api_base_hook.</code><code class="sig-name descname">_DEFAULT_SCOPES</code><em class="property"> = ['https://www.googleapis.com/auth/cloud-platform']</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#_DEFAULT_SCOPES"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook._DEFAULT_SCOPES" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="data">
<dt id="airflow.contrib.hooks.gcp_api_base_hook._G_APP_CRED_ENV_VAR">
<code class="sig-prename descclassname">airflow.contrib.hooks.gcp_api_base_hook.</code><code class="sig-name descname">_G_APP_CRED_ENV_VAR</code><em class="property"> = GOOGLE_APPLICATION_CREDENTIALS</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#_G_APP_CRED_ENV_VAR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook._G_APP_CRED_ENV_VAR" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="class">
<dt id="airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook">
<em class="property">class </em><code class="sig-prename descclassname">airflow.contrib.hooks.gcp_api_base_hook.</code><code class="sig-name descname">GoogleCloudBaseHook</code><span class="sig-paren">(</span><em class="sig-param">gcp_conn_id='google_cloud_default'</em>, <em class="sig-param">delegate_to=None</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#GoogleCloudBaseHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference internal" href="../../../hooks/base_hook/index.html#airflow.hooks.base_hook.BaseHook" title="airflow.hooks.base_hook.BaseHook"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.hooks.base_hook.BaseHook</span></code></a></p>
<p>A base hook for Google cloud-related hooks. Google cloud has a shared REST
API client that is built in the same way no matter which service you use.
This class helps construct and authorize the credentials needed to then
call googleapiclient.discovery.build() to actually discover and build a client
for a Google cloud service.</p>
<p>The class also contains some miscellaneous helper functions.</p>
<p>All hook derived from this base hook use the ‘Google Cloud Platform’ connection
type. Three ways of authentication are supported:</p>
<p>Default credentials: Only the ‘Project Id’ is required. You’ll need to
have set up default credentials, such as by the
<code class="docutils literal notranslate"><span class="pre">GOOGLE_APPLICATION_DEFAULT</span></code> environment variable or from the metadata
server on Google Compute Engine.</p>
<p>JSON key file: Specify ‘Project Id’, ‘Keyfile Path’ and ‘Scope’.</p>
<p>Legacy P12 key files are not supported.</p>
<p>JSON data provided in the UI: Specify ‘Keyfile JSON’.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>gcp_conn_id</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a>) – The connection ID to use when fetching connection info.</p></li>
<li><p><strong>delegate_to</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a>) – The account to impersonate, if any.
For this to work, the service account making the request must have
domain-wide delegation enabled.</p></li>
</ul>
</dd>
</dl>
<dl class="class">
<dt id="airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._Decorators">
<em class="property">class </em><code class="sig-name descname">_Decorators</code><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#GoogleCloudBaseHook._Decorators"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._Decorators" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.7)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p>
<p>A private inner class for keeping all decorator methods.</p>
<dl class="method">
<dt id="airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._Decorators.provide_gcp_credential_file">
<em class="property">static </em><code class="sig-name descname">provide_gcp_credential_file</code><span class="sig-paren">(</span><em class="sig-param">func</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#GoogleCloudBaseHook._Decorators.provide_gcp_credential_file"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._Decorators.provide_gcp_credential_file" title="Permalink to this definition"></a></dt>
<dd><p>Function decorator that provides a GOOGLE_APPLICATION_CREDENTIALS
environment variable, pointing to file path of a JSON file of service
account key.</p>
</dd></dl>
</dd></dl>
<dl class="attribute">
<dt id="airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.project_id">
<code class="sig-name descname">project_id</code><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#GoogleCloudBaseHook.project_id"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.project_id" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._get_credentials">
<code class="sig-name descname">_get_credentials</code><span class="sig-paren">(</span><em class="sig-param">self</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#GoogleCloudBaseHook._get_credentials"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._get_credentials" title="Permalink to this definition"></a></dt>
<dd><p>Returns the Credentials object for Google API</p>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._get_access_token">
<code class="sig-name descname">_get_access_token</code><span class="sig-paren">(</span><em class="sig-param">self</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#GoogleCloudBaseHook._get_access_token"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._get_access_token" title="Permalink to this definition"></a></dt>
<dd><p>Returns a valid access token from Google API Credentials</p>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._authorize">
<code class="sig-name descname">_authorize</code><span class="sig-paren">(</span><em class="sig-param">self</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#GoogleCloudBaseHook._authorize"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._authorize" title="Permalink to this definition"></a></dt>
<dd><p>Returns an authorized HTTP object to be used to build a Google cloud
service hook connection.</p>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._get_field">
<code class="sig-name descname">_get_field</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">f</em>, <em class="sig-param">default=None</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#GoogleCloudBaseHook._get_field"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._get_field" title="Permalink to this definition"></a></dt>
<dd><p>Fetches a field from extras, and returns it. This is some Airflow
magic. The google_cloud_platform hook type adds custom UI elements
to the hook page, which allow admins to specify service_account,
key_path, etc. They get formatted as shown below.</p>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.catch_http_exception">
<em class="property">static </em><code class="sig-name descname">catch_http_exception</code><span class="sig-paren">(</span><em class="sig-param">func</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#GoogleCloudBaseHook.catch_http_exception"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.catch_http_exception" title="Permalink to this definition"></a></dt>
<dd><p>Function decorator that intercepts HTTP Errors and raises AirflowException
with more informative message.</p>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.fallback_to_default_project_id">
<em class="property">static </em><code class="sig-name descname">fallback_to_default_project_id</code><span class="sig-paren">(</span><em class="sig-param">func</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#GoogleCloudBaseHook.fallback_to_default_project_id"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook.fallback_to_default_project_id" title="Permalink to this definition"></a></dt>
<dd><p>Decorator that provides fallback for Google Cloud Platform project id. If
the project is None it will be replaced with the project_id from the
service account the Hook is authenticated with. Project id can be specified
either via project_id kwarg or via first parameter in positional args.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>func</strong> – function to wrap</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>result of the function call</p>
</dd>
</dl>
</dd></dl>
<dl class="method">
<dt id="airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._get_project_id">
<code class="sig-name descname">_get_project_id</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">project_id</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/hooks/gcp_api_base_hook.html#GoogleCloudBaseHook._get_project_id"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook._get_project_id" title="Permalink to this definition"></a></dt>
<dd><p>In case project_id is None, overrides it with default project_id from
the service account that is authorized.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>project_id</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a>) – project id to</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>the project_id specified or default project id if project_id is None</p>
</dd>
</dl>
</dd></dl>
</dd></dl>
</div>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="../gcp_bigtable_hook/index.html" class="btn btn-neutral float-right" title="airflow.contrib.hooks.gcp_bigtable_hook" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="../ftp_hook/index.html" class="btn btn-neutral float-left" title="airflow.contrib.hooks.ftp_hook" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>