

<!--
Javascript to render AIRFLOW-XXX and PR references in text
as HTML links.

Overrides extrahead block from sphinx_rtd_theme
https://www.sphinx-doc.org/en/master/templating.html
-->


<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>airflow.contrib.operators.databricks_operator &mdash; Airflow Documentation</title>
  

    <link rel="shortcut icon" href="../../../../../_static/pin_32.png"/>
  
  
  <script type="text/javascript" src="../../../../../_static/js/modernizr.min.js"></script>
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../../../../../" src="../../../../../_static/documentation_options.js"></script>
        <script type="text/javascript" src="../../../../../_static/jquery.js"></script>
        <script type="text/javascript" src="../../../../../_static/underscore.js"></script>
        <script type="text/javascript" src="../../../../../_static/doctools.js"></script>
        <script type="text/javascript" src="../../../../../_static/language_data.js"></script>
        <script type="text/javascript" src="../../../../../_static/jira-links.js"></script>
    
    <script type="text/javascript" src="../../../../../_static/js/theme.js"></script>

    
  <link rel="stylesheet" href="../../../../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../../../../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../../../../../_static/graphviz.css" type="text/css" />
  <link rel="stylesheet" href="../../../../../_static/exampleinclude.css" type="text/css" />
    <link rel="index" title="Index" href="../../../../../genindex.html" />
    <link rel="search" title="Search" href="../../../../../search.html" />
    <link rel="next" title="airflow.contrib.operators.dataflow_operator" href="../dataflow_operator/index.html" />
    <link rel="prev" title="airflow.contrib.operators.cassandra_to_gcs" href="../cassandra_to_gcs/index.html" />
   
  <script>
  </script>
  <style>

  </style>

</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

            <a href="../../../../../index.html" class="icon icon-home"> Airflow
          

          </a>

          
              <div class="version">
                1.10.8
              </div>
            
          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
              <ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../profiling.html">Data Profiling</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../cli.html">Command Line Interface Reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../executor/index.html">Executor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../security.html">Security</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../api.html">REST API Reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../metrics.html">Metrics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../errors.html">Error Tracking</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../kubernetes.html">Kubernetes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../dag-serialization.html">DAG Serialization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../changelog.html">Changelog</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../best-practices.html">Best Practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../macros.html">Macros reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="../../../../index.html">Python API</a><ul class="current">
<li class="toctree-l2 current"><a class="reference internal" href="../../../../index.html#operators">Operators</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="../../../../index.html#baseoperator">BaseOperator</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../../../index.html#basesensoroperator">BaseSensorOperator</a></li>
<li class="toctree-l3 current"><a class="reference internal" href="../../../../index.html#operators-packages">Operators packages</a><ul class="current">
<li class="toctree-l4"><a class="reference internal" href="../../../operators/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.operators</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../../sensors/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.sensors</span></code></a></li>
<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.operators</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="../../sensors/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.sensors</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#hooks">Hooks</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#executors">Executors</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#models">Models</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../../index.html#core-and-community-package">Core and community package</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../../../configurations-ref.html">Configurations</a></li>
</ul>

            
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../../../../../index.html">Airflow</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          
<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../../../../index.html">Docs</a> &raquo;</li>
        
          <li><a href="../../../../index.html">Python API Reference</a> &raquo;</li>
        
          <li><a href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.operators</span></code></a> &raquo;</li>
        
      <li><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.operators.databricks_operator</span></code></li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../../../../../_sources/_api/airflow/contrib/operators/databricks_operator/index.rst.txt" rel="nofollow"> View page source</a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="module-airflow.contrib.operators.databricks_operator">
<span id="airflow-contrib-operators-databricks-operator"></span><h1><a class="reference internal" href="#module-airflow.contrib.operators.databricks_operator" title="airflow.contrib.operators.databricks_operator"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.contrib.operators.databricks_operator</span></code></a><a class="headerlink" href="#module-airflow.contrib.operators.databricks_operator" title="Permalink to this headline">¶</a></h1>
<div class="section" id="module-contents">
<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline">¶</a></h2>
<dl class="data">
<dt id="airflow.contrib.operators.databricks_operator.XCOM_RUN_ID_KEY">
<code class="sig-prename descclassname">airflow.contrib.operators.databricks_operator.</code><code class="sig-name descname">XCOM_RUN_ID_KEY</code><em class="property"> = run_id</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#XCOM_RUN_ID_KEY"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.XCOM_RUN_ID_KEY" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="data">
<dt id="airflow.contrib.operators.databricks_operator.XCOM_RUN_PAGE_URL_KEY">
<code class="sig-prename descclassname">airflow.contrib.operators.databricks_operator.</code><code class="sig-name descname">XCOM_RUN_PAGE_URL_KEY</code><em class="property"> = run_page_url</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#XCOM_RUN_PAGE_URL_KEY"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.XCOM_RUN_PAGE_URL_KEY" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="function">
<dt id="airflow.contrib.operators.databricks_operator._deep_string_coerce">
<code class="sig-prename descclassname">airflow.contrib.operators.databricks_operator.</code><code class="sig-name descname">_deep_string_coerce</code><span class="sig-paren">(</span><em class="sig-param">content</em>, <em class="sig-param">json_path='json'</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#_deep_string_coerce"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator._deep_string_coerce" title="Permalink to this definition">¶</a></dt>
<dt>
<code class="sig-name descname">Coerces content or all values of content if it is a dict to a string. The</code></dt>
<dt>
<code class="sig-name descname">function will throw if content contains non-string or non-numeric types.</code></dt>
<dd><p>The reason why we have this function is because the <code class="docutils literal notranslate"><span class="pre">self.json</span></code> field must be a
dict with only string values. This is because <code class="docutils literal notranslate"><span class="pre">render_template</span></code> will fail
for numerical values.</p>
</dd></dl>

<dl class="function">
<dt id="airflow.contrib.operators.databricks_operator._handle_databricks_operator_execution">
<code class="sig-prename descclassname">airflow.contrib.operators.databricks_operator.</code><code class="sig-name descname">_handle_databricks_operator_execution</code><span class="sig-paren">(</span><em class="sig-param">operator</em>, <em class="sig-param">hook</em>, <em class="sig-param">log</em>, <em class="sig-param">context</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#_handle_databricks_operator_execution"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator._handle_databricks_operator_execution" title="Permalink to this definition">¶</a></dt>
<dt>
<code class="sig-name descname">Handles the Airflow + Databricks lifecycle logic for a Databricks operator</code></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>operator</strong> – Databricks operator being handled</p></li>
<li><p><strong>context</strong> – Airflow context</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="class">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator">
<em class="property">class </em><code class="sig-prename descclassname">airflow.contrib.operators.databricks_operator.</code><code class="sig-name descname">DatabricksSubmitRunOperator</code><span class="sig-paren">(</span><em class="sig-param">json=None</em>, <em class="sig-param">spark_jar_task=None</em>, <em class="sig-param">notebook_task=None</em>, <em class="sig-param">new_cluster=None</em>, <em class="sig-param">existing_cluster_id=None</em>, <em class="sig-param">libraries=None</em>, <em class="sig-param">run_name=None</em>, <em class="sig-param">timeout_seconds=None</em>, <em class="sig-param">databricks_conn_id='databricks_default'</em>, <em class="sig-param">polling_period_seconds=30</em>, <em class="sig-param">databricks_retry_limit=3</em>, <em class="sig-param">databricks_retry_delay=1</em>, <em class="sig-param">do_xcom_push=False</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksSubmitRunOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="../../../models/index.html#airflow.models.BaseOperator" title="airflow.models.BaseOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.models.BaseOperator</span></code></a></p>
<p>Submits a Spark job run to Databricks using the
<a class="reference external" href="https://docs.databricks.com/api/latest/jobs.html#runs-submit">api/2.0/jobs/runs/submit</a>
API endpoint.</p>
<p>There are two ways to instantiate this operator.</p>
<p>In the first way, you can take the JSON payload that you typically use
to call the <code class="docutils literal notranslate"><span class="pre">api/2.0/jobs/runs/submit</span></code> endpoint and pass it directly
to our <code class="docutils literal notranslate"><span class="pre">DatabricksSubmitRunOperator</span></code> through the <code class="docutils literal notranslate"><span class="pre">json</span></code> parameter.
For example</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">json</span> <span class="o">=</span> <span class="p">{</span>
  <span class="s1">&#39;new_cluster&#39;</span><span class="p">:</span> <span class="p">{</span>
    <span class="s1">&#39;spark_version&#39;</span><span class="p">:</span> <span class="s1">&#39;2.1.0-db3-scala2.11&#39;</span><span class="p">,</span>
    <span class="s1">&#39;num_workers&#39;</span><span class="p">:</span> <span class="mi">2</span>
  <span class="p">},</span>
  <span class="s1">&#39;notebook_task&#39;</span><span class="p">:</span> <span class="p">{</span>
    <span class="s1">&#39;notebook_path&#39;</span><span class="p">:</span> <span class="s1">&#39;/Users/airflow@example.com/PrepareData&#39;</span><span class="p">,</span>
  <span class="p">},</span>
<span class="p">}</span>
<span class="n">notebook_run</span> <span class="o">=</span> <span class="n">DatabricksSubmitRunOperator</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;notebook_run&#39;</span><span class="p">,</span> <span class="n">json</span><span class="o">=</span><span class="n">json</span><span class="p">)</span>
</pre></div>
</div>
<p>Another way to accomplish the same thing is to use the named parameters
of the <code class="docutils literal notranslate"><span class="pre">DatabricksSubmitRunOperator</span></code> directly. Note that there is exactly
one named parameter for each top level parameter in the <code class="docutils literal notranslate"><span class="pre">runs/submit</span></code>
endpoint. In this method, your code would look like this:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">new_cluster</span> <span class="o">=</span> <span class="p">{</span>
  <span class="s1">&#39;spark_version&#39;</span><span class="p">:</span> <span class="s1">&#39;2.1.0-db3-scala2.11&#39;</span><span class="p">,</span>
  <span class="s1">&#39;num_workers&#39;</span><span class="p">:</span> <span class="mi">2</span>
<span class="p">}</span>
<span class="n">notebook_task</span> <span class="o">=</span> <span class="p">{</span>
  <span class="s1">&#39;notebook_path&#39;</span><span class="p">:</span> <span class="s1">&#39;/Users/airflow@example.com/PrepareData&#39;</span><span class="p">,</span>
<span class="p">}</span>
<span class="n">notebook_run</span> <span class="o">=</span> <span class="n">DatabricksSubmitRunOperator</span><span class="p">(</span>
    <span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;notebook_run&#39;</span><span class="p">,</span>
    <span class="n">new_cluster</span><span class="o">=</span><span class="n">new_cluster</span><span class="p">,</span>
    <span class="n">notebook_task</span><span class="o">=</span><span class="n">notebook_task</span><span class="p">)</span>
</pre></div>
</div>
<p>In the case where both the json parameter <strong>AND</strong> the named parameters
are provided, they will be merged together. If there are conflicts during the merge,
the named parameters will take precedence and override the top level <code class="docutils literal notranslate"><span class="pre">json</span></code> keys.</p>
<dl class="simple">
<dt>Currently the named parameters that <code class="docutils literal notranslate"><span class="pre">DatabricksSubmitRunOperator</span></code> supports are</dt><dd><ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">spark_jar_task</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">notebook_task</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">new_cluster</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">existing_cluster_id</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">libraries</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">run_name</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">timeout_seconds</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>json</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.8)"><em>dict</em></a>) – <p>A JSON object containing API parameters which will be passed
directly to the <code class="docutils literal notranslate"><span class="pre">api/2.0/jobs/runs/submit</span></code> endpoint. The other named parameters
(i.e. <code class="docutils literal notranslate"><span class="pre">spark_jar_task</span></code>, <code class="docutils literal notranslate"><span class="pre">notebook_task</span></code>..) to this operator will
be merged with this json dictionary if they are provided.
If there are conflicts during the merge, the named parameters will
take precedence and override the top level json keys. (templated)</p>
<div class="admonition seealso">
<p class="admonition-title">See also</p>
<p>For more information about templating see <a class="reference internal" href="../../../../../concepts.html#jinja-templating"><span class="std std-ref">Jinja Templating</span></a>.
<a class="reference external" href="https://docs.databricks.com/api/latest/jobs.html#runs-submit">https://docs.databricks.com/api/latest/jobs.html#runs-submit</a></p>
</div>
</p></li>
<li><p><strong>spark_jar_task</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.8)"><em>dict</em></a>) – <p>The main class and parameters for the JAR task. Note that
the actual JAR is specified in the <code class="docutils literal notranslate"><span class="pre">libraries</span></code>.
<em>EITHER</em> <code class="docutils literal notranslate"><span class="pre">spark_jar_task</span></code> <em>OR</em> <code class="docutils literal notranslate"><span class="pre">notebook_task</span></code> should be specified.
This field will be templated.</p>
<div class="admonition seealso">
<p class="admonition-title">See also</p>
<p><a class="reference external" href="https://docs.databricks.com/api/latest/jobs.html#jobssparkjartask">https://docs.databricks.com/api/latest/jobs.html#jobssparkjartask</a></p>
</div>
</p></li>
<li><p><strong>notebook_task</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.8)"><em>dict</em></a>) – <p>The notebook path and parameters for the notebook task.
<em>EITHER</em> <code class="docutils literal notranslate"><span class="pre">spark_jar_task</span></code> <em>OR</em> <code class="docutils literal notranslate"><span class="pre">notebook_task</span></code> should be specified.
This field will be templated.</p>
<div class="admonition seealso">
<p class="admonition-title">See also</p>
<p><a class="reference external" href="https://docs.databricks.com/api/latest/jobs.html#jobsnotebooktask">https://docs.databricks.com/api/latest/jobs.html#jobsnotebooktask</a></p>
</div>
</p></li>
<li><p><strong>new_cluster</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.8)"><em>dict</em></a>) – <p>Specs for a new cluster on which this task will be run.
<em>EITHER</em> <code class="docutils literal notranslate"><span class="pre">new_cluster</span></code> <em>OR</em> <code class="docutils literal notranslate"><span class="pre">existing_cluster_id</span></code> should be specified.
This field will be templated.</p>
<div class="admonition seealso">
<p class="admonition-title">See also</p>
<p><a class="reference external" href="https://docs.databricks.com/api/latest/jobs.html#jobsclusterspecnewcluster">https://docs.databricks.com/api/latest/jobs.html#jobsclusterspecnewcluster</a></p>
</div>
</p></li>
<li><p><strong>existing_cluster_id</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – ID for existing cluster on which to run this task.
<em>EITHER</em> <code class="docutils literal notranslate"><span class="pre">new_cluster</span></code> <em>OR</em> <code class="docutils literal notranslate"><span class="pre">existing_cluster_id</span></code> should be specified.
This field will be templated.</p></li>
<li><p><strong>libraries</strong> (<em>list of dicts</em>) – <p>Libraries which this run will use.
This field will be templated.</p>
<div class="admonition seealso">
<p class="admonition-title">See also</p>
<p><a class="reference external" href="https://docs.databricks.com/api/latest/libraries.html#managedlibrarieslibrary">https://docs.databricks.com/api/latest/libraries.html#managedlibrarieslibrary</a></p>
</div>
</p></li>
<li><p><strong>run_name</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – The run name used for this task.
By default this will be set to the Airflow <code class="docutils literal notranslate"><span class="pre">task_id</span></code>. This <code class="docutils literal notranslate"><span class="pre">task_id</span></code> is a
required parameter of the superclass <code class="docutils literal notranslate"><span class="pre">BaseOperator</span></code>.
This field will be templated.</p></li>
<li><p><strong>timeout_seconds</strong> (<em>int32</em>) – The timeout for this run. By default a value of 0 is used
which means to have no timeout.
This field will be templated.</p></li>
<li><p><strong>databricks_conn_id</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – The name of the Airflow connection to use.
By default and in the common case this will be <code class="docutils literal notranslate"><span class="pre">databricks_default</span></code>. To use
token based authentication, provide the key <code class="docutils literal notranslate"><span class="pre">token</span></code> in the extra field for the
connection and create the key <code class="docutils literal notranslate"><span class="pre">host</span></code> and leave the <code class="docutils literal notranslate"><span class="pre">host</span></code> field empty.</p></li>
<li><p><strong>polling_period_seconds</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a>) – Controls the rate which we poll for the result of
this run. By default the operator will poll every 30 seconds.</p></li>
<li><p><strong>databricks_retry_limit</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a>) – Amount of times retry if the Databricks backend is
unreachable. Its value must be greater than or equal to 1.</p></li>
<li><p><strong>databricks_retry_delay</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.8)"><em>float</em></a>) – Number of seconds to wait between retries (it
might be a floating point number).</p></li>
<li><p><strong>do_xcom_push</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a>) – Whether we should push run_id and run_page_url to xcom.</p></li>
</ul>
</dd>
</dl>
<dl class="attribute">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.template_fields">
<code class="sig-name descname">template_fields</code><em class="property"> = ['json']</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksSubmitRunOperator.template_fields"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.template_fields" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.ui_color">
<code class="sig-name descname">ui_color</code><em class="property"> = #1CB1C2</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksSubmitRunOperator.ui_color"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.ui_color" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.ui_fgcolor">
<code class="sig-name descname">ui_fgcolor</code><em class="property"> = #fff</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksSubmitRunOperator.ui_fgcolor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.ui_fgcolor" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.get_hook">
<code class="sig-name descname">get_hook</code><span class="sig-paren">(</span><em class="sig-param">self</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksSubmitRunOperator.get_hook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.get_hook" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.execute">
<code class="sig-name descname">execute</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">context</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksSubmitRunOperator.execute"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.execute" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.on_kill">
<code class="sig-name descname">on_kill</code><span class="sig-paren">(</span><em class="sig-param">self</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksSubmitRunOperator.on_kill"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.on_kill" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

</dd></dl>

<dl class="class">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator">
<em class="property">class </em><code class="sig-prename descclassname">airflow.contrib.operators.databricks_operator.</code><code class="sig-name descname">DatabricksRunNowOperator</code><span class="sig-paren">(</span><em class="sig-param">job_id</em>, <em class="sig-param">json=None</em>, <em class="sig-param">notebook_params=None</em>, <em class="sig-param">python_params=None</em>, <em class="sig-param">spark_submit_params=None</em>, <em class="sig-param">databricks_conn_id='databricks_default'</em>, <em class="sig-param">polling_period_seconds=30</em>, <em class="sig-param">databricks_retry_limit=3</em>, <em class="sig-param">databricks_retry_delay=1</em>, <em class="sig-param">do_xcom_push=False</em>, <em class="sig-param">**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksRunNowOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="../../../models/index.html#airflow.models.BaseOperator" title="airflow.models.BaseOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.models.BaseOperator</span></code></a></p>
<p>Runs an existing Spark job run to Databricks using the
<a class="reference external" href="https://docs.databricks.com/api/latest/jobs.html#run-now">api/2.0/jobs/run-now</a>
API endpoint.</p>
<p>There are two ways to instantiate this operator.</p>
<p>In the first way, you can take the JSON payload that you typically use
to call the <code class="docutils literal notranslate"><span class="pre">api/2.0/jobs/run-now</span></code> endpoint and pass it directly
to our <code class="docutils literal notranslate"><span class="pre">DatabricksRunNowOperator</span></code> through the <code class="docutils literal notranslate"><span class="pre">json</span></code> parameter.
For example</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">json</span> <span class="o">=</span> <span class="p">{</span>
  <span class="s2">&quot;job_id&quot;</span><span class="p">:</span> <span class="mi">42</span><span class="p">,</span>
  <span class="s2">&quot;notebook_params&quot;</span><span class="p">:</span> <span class="p">{</span>
    <span class="s2">&quot;dry-run&quot;</span><span class="p">:</span> <span class="s2">&quot;true&quot;</span><span class="p">,</span>
    <span class="s2">&quot;oldest-time-to-consider&quot;</span><span class="p">:</span> <span class="s2">&quot;1457570074236&quot;</span>
  <span class="p">}</span>
<span class="p">}</span>

<span class="n">notebook_run</span> <span class="o">=</span> <span class="n">DatabricksRunNowOperator</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="s1">&#39;notebook_run&#39;</span><span class="p">,</span> <span class="n">json</span><span class="o">=</span><span class="n">json</span><span class="p">)</span>
</pre></div>
</div>
<p>Another way to accomplish the same thing is to use the named parameters
of the <code class="docutils literal notranslate"><span class="pre">DatabricksRunNowOperator</span></code> directly. Note that there is exactly
one named parameter for each top level parameter in the <code class="docutils literal notranslate"><span class="pre">run-now</span></code>
endpoint. In this method, your code would look like this:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">job_id</span><span class="o">=</span><span class="mi">42</span>

<span class="n">notebook_params</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s2">&quot;dry-run&quot;</span><span class="p">:</span> <span class="s2">&quot;true&quot;</span><span class="p">,</span>
    <span class="s2">&quot;oldest-time-to-consider&quot;</span><span class="p">:</span> <span class="s2">&quot;1457570074236&quot;</span>
<span class="p">}</span>

<span class="n">python_params</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;douglas adams&quot;</span><span class="p">,</span> <span class="s2">&quot;42&quot;</span><span class="p">]</span>

<span class="n">spark_submit_params</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;--class&quot;</span><span class="p">,</span> <span class="s2">&quot;org.apache.spark.examples.SparkPi&quot;</span><span class="p">]</span>

<span class="n">notebook_run</span> <span class="o">=</span> <span class="n">DatabricksRunNowOperator</span><span class="p">(</span>
    <span class="n">job_id</span><span class="o">=</span><span class="n">job_id</span><span class="p">,</span>
    <span class="n">notebook_params</span><span class="o">=</span><span class="n">notebook_params</span><span class="p">,</span>
    <span class="n">python_params</span><span class="o">=</span><span class="n">python_params</span><span class="p">,</span>
    <span class="n">spark_submit_params</span><span class="o">=</span><span class="n">spark_submit_params</span>
<span class="p">)</span>
</pre></div>
</div>
<p>In the case where both the json parameter <strong>AND</strong> the named parameters
are provided, they will be merged together. If there are conflicts during the merge,
the named parameters will take precedence and override the top level <code class="docutils literal notranslate"><span class="pre">json</span></code> keys.</p>
<dl class="simple">
<dt>Currently the named parameters that <code class="docutils literal notranslate"><span class="pre">DatabricksRunNowOperator</span></code> supports are</dt><dd><ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">job_id</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">json</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">notebook_params</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">python_params</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">spark_submit_params</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>job_id</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – <p>the job_id of the existing Databricks job.
This field will be templated.</p>
<div class="admonition seealso">
<p class="admonition-title">See also</p>
<p><a class="reference external" href="https://docs.databricks.com/api/latest/jobs.html#run-now">https://docs.databricks.com/api/latest/jobs.html#run-now</a></p>
</div>
</p></li>
<li><p><strong>json</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.8)"><em>dict</em></a>) – <p>A JSON object containing API parameters which will be passed
directly to the <code class="docutils literal notranslate"><span class="pre">api/2.0/jobs/run-now</span></code> endpoint. The other named parameters
(i.e. <code class="docutils literal notranslate"><span class="pre">notebook_params</span></code>, <code class="docutils literal notranslate"><span class="pre">spark_submit_params</span></code>..) to this operator will
be merged with this json dictionary if they are provided.
If there are conflicts during the merge, the named parameters will
take precedence and override the top level json keys. (templated)</p>
<div class="admonition seealso">
<p class="admonition-title">See also</p>
<p>For more information about templating see <a class="reference internal" href="../../../../../concepts.html#jinja-templating"><span class="std std-ref">Jinja Templating</span></a>.
<a class="reference external" href="https://docs.databricks.com/api/latest/jobs.html#run-now">https://docs.databricks.com/api/latest/jobs.html#run-now</a></p>
</div>
</p></li>
<li><p><strong>notebook_params</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.8)"><em>dict</em></a>) – <p>A dict from keys to values for jobs with notebook task,
e.g. “notebook_params”: {“name”: “john doe”, “age”:  “35”}.
The map is passed to the notebook and will be accessible through the
dbutils.widgets.get function. See Widgets for more information.
If not specified upon run-now, the triggered run will use the
job’s base parameters. notebook_params cannot be
specified in conjunction with jar_params. The json representation
of this field (i.e. {“notebook_params”:{“name”:”john doe”,”age”:”35”}})
cannot exceed 10,000 bytes.
This field will be templated.</p>
<div class="admonition seealso">
<p class="admonition-title">See also</p>
<p><a class="reference external" href="https://docs.databricks.com/user-guide/notebooks/widgets.html">https://docs.databricks.com/user-guide/notebooks/widgets.html</a></p>
</div>
</p></li>
<li><p><strong>python_params</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em>) – <p>A list of parameters for jobs with python tasks,
e.g. “python_params”: [“john doe”, “35”].
The parameters will be passed to python file as command line parameters.
If specified upon run-now, it would overwrite the parameters specified in
job setting.
The json representation of this field (i.e. {“python_params”:[“john doe”,”35”]})
cannot exceed 10,000 bytes.
This field will be templated.</p>
<div class="admonition seealso">
<p class="admonition-title">See also</p>
<p><a class="reference external" href="https://docs.databricks.com/api/latest/jobs.html#run-now">https://docs.databricks.com/api/latest/jobs.html#run-now</a></p>
</div>
</p></li>
<li><p><strong>spark_submit_params</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em>) – <p>A list of parameters for jobs with spark submit task,
e.g. “spark_submit_params”: [“–class”, “org.apache.spark.examples.SparkPi”].
The parameters will be passed to spark-submit script as command line parameters.
If specified upon run-now, it would overwrite the parameters specified
in job setting.
The json representation of this field cannot exceed 10,000 bytes.
This field will be templated.</p>
<div class="admonition seealso">
<p class="admonition-title">See also</p>
<p><a class="reference external" href="https://docs.databricks.com/api/latest/jobs.html#run-now">https://docs.databricks.com/api/latest/jobs.html#run-now</a></p>
</div>
</p></li>
<li><p><strong>timeout_seconds</strong> (<em>int32</em>) – The timeout for this run. By default a value of 0 is used
which means to have no timeout.
This field will be templated.</p></li>
<li><p><strong>databricks_conn_id</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – The name of the Airflow connection to use.
By default and in the common case this will be <code class="docutils literal notranslate"><span class="pre">databricks_default</span></code>. To use
token based authentication, provide the key <code class="docutils literal notranslate"><span class="pre">token</span></code> in the extra field for the
connection and create the key <code class="docutils literal notranslate"><span class="pre">host</span></code> and leave the <code class="docutils literal notranslate"><span class="pre">host</span></code> field empty.</p></li>
<li><p><strong>polling_period_seconds</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a>) – Controls the rate which we poll for the result of
this run. By default the operator will poll every 30 seconds.</p></li>
<li><p><strong>databricks_retry_limit</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a>) – Amount of times retry if the Databricks backend is
unreachable. Its value must be greater than or equal to 1.</p></li>
<li><p><strong>do_xcom_push</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a>) – Whether we should push run_id and run_page_url to xcom.</p></li>
</ul>
</dd>
</dl>
<dl class="attribute">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator.template_fields">
<code class="sig-name descname">template_fields</code><em class="property"> = ['json']</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksRunNowOperator.template_fields"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator.template_fields" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator.ui_color">
<code class="sig-name descname">ui_color</code><em class="property"> = #1CB1C2</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksRunNowOperator.ui_color"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator.ui_color" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator.ui_fgcolor">
<code class="sig-name descname">ui_fgcolor</code><em class="property"> = #fff</em><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksRunNowOperator.ui_fgcolor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator.ui_fgcolor" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator.get_hook">
<code class="sig-name descname">get_hook</code><span class="sig-paren">(</span><em class="sig-param">self</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksRunNowOperator.get_hook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator.get_hook" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator.execute">
<code class="sig-name descname">execute</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">context</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksRunNowOperator.execute"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator.execute" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator.on_kill">
<code class="sig-name descname">on_kill</code><span class="sig-paren">(</span><em class="sig-param">self</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../../_modules/airflow/contrib/operators/databricks_operator.html#DatabricksRunNowOperator.on_kill"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator.on_kill" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

</dd></dl>

</div>
</div>


           </div>
           
          </div>
          

<footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="../dataflow_operator/index.html" class="btn btn-neutral float-right" title="airflow.contrib.operators.dataflow_operator" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="../cassandra_to_gcs/index.html" class="btn btn-neutral float-left" title="airflow.contrib.operators.cassandra_to_gcs" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
  <div class="footer">This page uses <a href="https://analytics.google.com/">
    Google Analytics</a> to collect statistics. You can disable it by blocking
    the JavaScript coming from www.google-analytics.com. Check our
    <a href="../../../../../privacy_notice.html">Privacy Policy</a>
    for more details.
  </div>


</footer>

        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
    <!-- Theme Analytics -->
    <script>
    (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
    })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

    ga('create', 'UA-140539454-1', 'auto');
    ga('send', 'pageview');
    </script>

    
</body>
</html>