Google Git
Sign in
apache / airflow-site / f81e48aadee354653679693fa54b04ac42d24bfd / . / docs-archive / 1.10.3 / _modules / airflow / models.html
blob: 19982846aa7dfaa89f2d00dabb2d0973d80f4226 [file] [log] [blame]
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>airflow.models &mdash; Airflow Documentation</title>
<script type="text/javascript" src="../../_static/js/modernizr.min.js"></script>
<script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
<script type="text/javascript" src="../../_static/jquery.js"></script>
<script type="text/javascript" src="../../_static/underscore.js"></script>
<script type="text/javascript" src="../../_static/doctools.js"></script>
<script type="text/javascript" src="../../_static/language_data.js"></script>
<script type="text/javascript" src="../../_static/js/theme.js"></script>
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<script>
document.addEventListener('DOMContentLoaded', function() {
var el = document.getElementById('changelog');
if (el !== null ) {
// [AIRFLOW-...]
el.innerHTML = el.innerHTML.replace(
/\[(AIRFLOW-[\d]+)\]/g,
`<a href="https://issues.apache.org/jira/browse/$1">[$1]</a>`
);
// (#...)
el.innerHTML = el.innerHTML.replace(
/\(#([\d]+)\)/g,
`<a href="https://github.com/apache/airflow/pull/$1">(#$1)</a>`
);
};
})
</script>
<style>
.example-header {
position: relative;
background: #9AAA7A;
padding: 8px 16px;
margin-bottom: 0;
}
.example-header--with-button {
padding-right: 166px;
}
.example-header:after{
content: '';
display: table;
clear: both;
}
.example-title {
display:block;
padding: 4px;
margin-right: 16px;
color: white;
overflow-x: auto;
}
.example-header-button {
top: 8px;
right: 16px;
position: absolute;
}
.example-header + .highlight-python {
margin-top: 0 !important;
}
.viewcode-button {
display: inline-block;
padding: 8px 16px;
border: 0;
margin: 0;
outline: 0;
border-radius: 2px;
-webkit-box-shadow: 0 3px 5px 0 rgba(0,0,0,.3);
box-shadow: 0 3px 6px 0 rgba(0,0,0,.3);
color: #404040;
background-color: #e7e7e7;
cursor: pointer;
font-size: 16px;
font-weight: 500;
line-height: 1;
text-decoration: none;
text-overflow: ellipsis;
overflow: hidden;
text-transform: uppercase;
-webkit-transition: background-color .2s;
transition: background-color .2s;
vertical-align: middle;
white-space: nowrap;
}
.viewcode-button:visited {
color: #404040;
}
.viewcode-button:hover, .viewcode-button:focus {
color: #404040;
background-color: #d6d6d6;
}
</style>
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="../../index.html" class="icon icon-home"> Airflow
</a>
<div class="version">
1.10.3
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../profiling.html">Data Profiling</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../cli.html">Command Line Interface</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../scheduler.html">Scheduling &amp; Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../security.html">Security</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../api.html">Experimental Rest API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../metrics.html">Metrics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../kubernetes.html">Kubernetes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../changelog.html">Changelog</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../macros.html">Macros reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../_api/index.html">API Reference</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../index.html">Airflow</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../index.html">Docs</a> &raquo;</li>
<li><a href="../index.html">Module code</a> &raquo;</li>
<li>airflow.models</li>
<li class="wy-breadcrumbs-aside">
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<h1>Source code for airflow.models</h1><div class="highlight"><pre>
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
<span class="c1">#</span>
<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
<span class="c1"># distributed with this work for additional information</span>
<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
<span class="c1"># KIND, either express or implied. See the License for the</span>
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">absolute_import</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">division</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">print_function</span>
<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">unicode_literals</span>
<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="ne">ImportError</span> <span class="k">as</span> <span class="n">BuiltinImportError</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="nb">str</span>
<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">defaultdict</span><span class="p">,</span> <span class="n">namedtuple</span><span class="p">,</span> <span class="n">OrderedDict</span>
<span class="kn">import</span> <span class="nn">copy</span>
<span class="kn">from</span> <span class="nn">typing</span> <span class="k">import</span> <span class="n">Iterable</span>
<span class="kn">from</span> <span class="nn">future.standard_library</span> <span class="k">import</span> <span class="n">install_aliases</span>
<span class="kn">from</span> <span class="nn">airflow.models.base</span> <span class="k">import</span> <span class="n">Base</span><span class="p">,</span> <span class="n">ID_LEN</span>
<span class="k">try</span><span class="p">:</span>
<span class="c1"># Fix Python &gt; 3.7 deprecation</span>
<span class="kn">from</span> <span class="nn">collections.abc</span> <span class="k">import</span> <span class="n">Hashable</span>
<span class="k">except</span> <span class="n">BuiltinImportError</span><span class="p">:</span>
<span class="c1"># Preserve Python &lt; 3.3 compatibility</span>
<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">Hashable</span>
<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">timedelta</span>
<span class="kn">import</span> <span class="nn">dill</span>
<span class="kn">import</span> <span class="nn">functools</span>
<span class="kn">import</span> <span class="nn">getpass</span>
<span class="kn">import</span> <span class="nn">imp</span>
<span class="kn">import</span> <span class="nn">importlib</span>
<span class="kn">import</span> <span class="nn">zipfile</span>
<span class="kn">import</span> <span class="nn">jinja2</span>
<span class="kn">import</span> <span class="nn">json</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">pendulum</span>
<span class="kn">import</span> <span class="nn">pickle</span>
<span class="kn">import</span> <span class="nn">re</span>
<span class="kn">import</span> <span class="nn">signal</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">time</span>
<span class="kn">import</span> <span class="nn">textwrap</span>
<span class="kn">import</span> <span class="nn">traceback</span>
<span class="kn">import</span> <span class="nn">warnings</span>
<span class="kn">import</span> <span class="nn">hashlib</span>
<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span>
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="k">import</span> <span class="n">quote</span>
<span class="kn">from</span> <span class="nn">sqlalchemy</span> <span class="k">import</span> <span class="p">(</span>
<span class="n">Boolean</span><span class="p">,</span> <span class="n">Column</span><span class="p">,</span> <span class="n">DateTime</span><span class="p">,</span> <span class="n">Float</span><span class="p">,</span> <span class="n">ForeignKey</span><span class="p">,</span> <span class="n">Index</span><span class="p">,</span>
<span class="n">Integer</span><span class="p">,</span> <span class="n">LargeBinary</span><span class="p">,</span> <span class="n">PickleType</span><span class="p">,</span> <span class="n">String</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">UniqueConstraint</span><span class="p">,</span> <span class="n">and_</span><span class="p">,</span>
<span class="n">func</span><span class="p">,</span> <span class="n">or_</span>
<span class="p">)</span>
<span class="kn">from</span> <span class="nn">sqlalchemy.ext.declarative</span> <span class="k">import</span> <span class="n">declared_attr</span>
<span class="kn">from</span> <span class="nn">sqlalchemy.orm</span> <span class="k">import</span> <span class="n">reconstructor</span><span class="p">,</span> <span class="n">relationship</span><span class="p">,</span> <span class="n">synonym</span>
<span class="kn">from</span> <span class="nn">croniter</span> <span class="k">import</span> <span class="p">(</span>
<span class="n">croniter</span><span class="p">,</span> <span class="n">CroniterBadCronError</span><span class="p">,</span> <span class="n">CroniterBadDateError</span><span class="p">,</span> <span class="n">CroniterNotAlphaError</span>
<span class="p">)</span>
<span class="kn">import</span> <span class="nn">six</span>
<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">settings</span><span class="p">,</span> <span class="n">utils</span>
<span class="kn">from</span> <span class="nn">airflow.executors</span> <span class="k">import</span> <span class="n">GetDefaultExecutor</span><span class="p">,</span> <span class="n">LocalExecutor</span>
<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="p">(</span>
<span class="n">AirflowDagCycleException</span><span class="p">,</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">AirflowSkipException</span><span class="p">,</span> <span class="n">AirflowTaskTimeout</span><span class="p">,</span>
<span class="n">AirflowRescheduleException</span>
<span class="p">)</span>
<span class="kn">from</span> <span class="nn">airflow.dag.base_dag</span> <span class="k">import</span> <span class="n">BaseDag</span><span class="p">,</span> <span class="n">BaseDagBag</span>
<span class="kn">from</span> <span class="nn">airflow.lineage</span> <span class="k">import</span> <span class="n">apply_lineage</span><span class="p">,</span> <span class="n">prepare_lineage</span>
<span class="kn">from</span> <span class="nn">airflow.models.dagpickle</span> <span class="k">import</span> <span class="n">DagPickle</span>
<span class="kn">from</span> <span class="nn">airflow.models.errors</span> <span class="k">import</span> <span class="ne">ImportError</span> <span class="c1"># noqa: F401</span>
<span class="kn">from</span> <span class="nn">airflow.models.slamiss</span> <span class="k">import</span> <span class="n">SlaMiss</span> <span class="c1"># noqa: F401</span>
<span class="kn">from</span> <span class="nn">airflow.models.kubernetes</span> <span class="k">import</span> <span class="n">KubeWorkerIdentifier</span><span class="p">,</span> <span class="n">KubeResourceVersion</span> <span class="c1"># noqa: F401</span>
<span class="kn">from</span> <span class="nn">airflow.models.log</span> <span class="k">import</span> <span class="n">Log</span>
<span class="kn">from</span> <span class="nn">airflow.models.taskfail</span> <span class="k">import</span> <span class="n">TaskFail</span>
<span class="kn">from</span> <span class="nn">airflow.models.taskreschedule</span> <span class="k">import</span> <span class="n">TaskReschedule</span>
<span class="kn">from</span> <span class="nn">airflow.ti_deps.deps.not_in_retry_period_dep</span> <span class="k">import</span> <span class="n">NotInRetryPeriodDep</span>
<span class="kn">from</span> <span class="nn">airflow.ti_deps.deps.prev_dagrun_dep</span> <span class="k">import</span> <span class="n">PrevDagrunDep</span>
<span class="kn">from</span> <span class="nn">airflow.ti_deps.deps.trigger_rule_dep</span> <span class="k">import</span> <span class="n">TriggerRuleDep</span>
<span class="kn">from</span> <span class="nn">airflow.ti_deps.dep_context</span> <span class="k">import</span> <span class="n">DepContext</span><span class="p">,</span> <span class="n">QUEUE_DEPS</span><span class="p">,</span> <span class="n">RUN_DEPS</span>
<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">timezone</span>
<span class="kn">from</span> <span class="nn">airflow.utils.dag_processing</span> <span class="k">import</span> <span class="n">list_py_file_paths</span>
<span class="kn">from</span> <span class="nn">airflow.utils.dates</span> <span class="k">import</span> <span class="n">cron_presets</span><span class="p">,</span> <span class="n">date_range</span> <span class="k">as</span> <span class="n">utils_date_range</span>
<span class="kn">from</span> <span class="nn">airflow.utils.db</span> <span class="k">import</span> <span class="n">provide_session</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
<span class="kn">from</span> <span class="nn">airflow.utils.email</span> <span class="k">import</span> <span class="n">send_email</span>
<span class="kn">from</span> <span class="nn">airflow.utils.helpers</span> <span class="k">import</span> <span class="p">(</span>
<span class="n">as_tuple</span><span class="p">,</span> <span class="n">is_container</span><span class="p">,</span> <span class="n">validate_key</span><span class="p">,</span> <span class="n">pprinttable</span><span class="p">)</span>
<span class="kn">from</span> <span class="nn">airflow.utils.operator_resources</span> <span class="k">import</span> <span class="n">Resources</span>
<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
<span class="kn">from</span> <span class="nn">airflow.utils.sqlalchemy</span> <span class="k">import</span> <span class="n">UtcDateTime</span><span class="p">,</span> <span class="n">Interval</span>
<span class="kn">from</span> <span class="nn">airflow.utils.timeout</span> <span class="k">import</span> <span class="n">timeout</span>
<span class="kn">from</span> <span class="nn">airflow.utils.trigger_rule</span> <span class="k">import</span> <span class="n">TriggerRule</span>
<span class="kn">from</span> <span class="nn">airflow.utils.weight_rule</span> <span class="k">import</span> <span class="n">WeightRule</span>
<span class="kn">from</span> <span class="nn">airflow.utils.net</span> <span class="k">import</span> <span class="n">get_hostname</span>
<span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
<span class="n">install_aliases</span><span class="p">()</span>
<div class="viewcode-block" id="XCOM_RETURN_KEY"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCOM_RETURN_KEY">[docs]</a><span class="n">XCOM_RETURN_KEY</span> <span class="o">=</span> <span class="s1">&#39;return_value&#39;</span></div>
<div class="viewcode-block" id="Stats"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Stats">[docs]</a><span class="n">Stats</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">Stats</span></div>
<div class="viewcode-block" id="InvalidFernetToken"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.InvalidFernetToken">[docs]</a><span class="k">class</span> <span class="nc">InvalidFernetToken</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
<span class="c1"># If Fernet isn&#39;t loaded we need a valid exception class to catch. If it is</span>
<span class="c1"># loaded this will get reset to the actual class once get_fernet() is called</span>
<span class="k">pass</span></div>
<div class="viewcode-block" id="NullFernet"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.NullFernet">[docs]</a><span class="k">class</span> <span class="nc">NullFernet</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> A &quot;Null&quot; encryptor class that doesn&#39;t encrypt or decrypt but that presents</span>
<span class="sd"> a similar interface to Fernet.</span>
<span class="sd"> The purpose of this is to make the rest of the code not have to know the</span>
<span class="sd"> difference, and to only display the message once, not 20 times when</span>
<span class="sd"> `airflow initdb` is ran.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<div class="viewcode-block" id="NullFernet.is_encrypted"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.NullFernet.is_encrypted">[docs]</a> <span class="n">is_encrypted</span> <span class="o">=</span> <span class="kc">False</span></div>
<div class="viewcode-block" id="NullFernet.decrpyt"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.NullFernet.decrpyt">[docs]</a> <span class="k">def</span> <span class="nf">decrpyt</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">b</span><span class="p">):</span>
<span class="k">return</span> <span class="n">b</span></div>
<div class="viewcode-block" id="NullFernet.encrypt"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.NullFernet.encrypt">[docs]</a> <span class="k">def</span> <span class="nf">encrypt</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">b</span><span class="p">):</span>
<span class="k">return</span> <span class="n">b</span></div></div>
<div class="viewcode-block" id="_fernet"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models._fernet">[docs]</a><span class="n">_fernet</span> <span class="o">=</span> <span class="kc">None</span></div>
<div class="viewcode-block" id="get_fernet"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.get_fernet">[docs]</a><span class="k">def</span> <span class="nf">get_fernet</span><span class="p">():</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Deferred load of Fernet key.</span>
<span class="sd"> This function could fail either because Cryptography is not installed</span>
<span class="sd"> or because the Fernet key is invalid.</span>
<span class="sd"> :return: Fernet object</span>
<span class="sd"> :raises: airflow.exceptions.AirflowException if there&#39;s a problem trying to load Fernet</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">global</span> <span class="n">_fernet</span>
<span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
<span class="k">if</span> <span class="n">_fernet</span><span class="p">:</span>
<span class="k">return</span> <span class="n">_fernet</span>
<span class="k">try</span><span class="p">:</span>
<span class="kn">from</span> <span class="nn">cryptography.fernet</span> <span class="k">import</span> <span class="n">Fernet</span><span class="p">,</span> <span class="n">MultiFernet</span><span class="p">,</span> <span class="n">InvalidToken</span>
<span class="k">global</span> <span class="n">InvalidFernetToken</span>
<span class="n">InvalidFernetToken</span> <span class="o">=</span> <span class="n">InvalidToken</span>
<span class="k">except</span> <span class="n">BuiltinImportError</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
<span class="s2">&quot;cryptography not found - values will not be stored encrypted.&quot;</span>
<span class="p">)</span>
<span class="n">_fernet</span> <span class="o">=</span> <span class="n">NullFernet</span><span class="p">()</span>
<span class="k">return</span> <span class="n">_fernet</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">fernet_key</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;FERNET_KEY&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">fernet_key</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
<span class="s2">&quot;empty cryptography key - values will not be stored encrypted.&quot;</span>
<span class="p">)</span>
<span class="n">_fernet</span> <span class="o">=</span> <span class="n">NullFernet</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">_fernet</span> <span class="o">=</span> <span class="n">MultiFernet</span><span class="p">([</span>
<span class="n">Fernet</span><span class="p">(</span><span class="n">fernet_part</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">))</span>
<span class="k">for</span> <span class="n">fernet_part</span> <span class="ow">in</span> <span class="n">fernet_key</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;,&#39;</span><span class="p">)</span>
<span class="p">])</span>
<span class="n">_fernet</span><span class="o">.</span><span class="n">is_encrypted</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">)</span> <span class="k">as</span> <span class="n">ve</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;Could not create Fernet object: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ve</span><span class="p">))</span>
<span class="k">return</span> <span class="n">_fernet</span></div>
<span class="c1"># Used by DAG context_managers</span>
<div class="viewcode-block" id="_CONTEXT_MANAGER_DAG"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models._CONTEXT_MANAGER_DAG">[docs]</a><span class="n">_CONTEXT_MANAGER_DAG</span> <span class="o">=</span> <span class="kc">None</span></div>
<div class="viewcode-block" id="clear_task_instances"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.clear_task_instances">[docs]</a><span class="k">def</span> <span class="nf">clear_task_instances</span><span class="p">(</span><span class="n">tis</span><span class="p">,</span>
<span class="n">session</span><span class="p">,</span>
<span class="n">activate_dag_runs</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">dag</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Clears a set of task instances, but makes sure the running ones</span>
<span class="sd"> get killed.</span>
<span class="sd"> :param tis: a list of task instances</span>
<span class="sd"> :param session: current session</span>
<span class="sd"> :param activate_dag_runs: flag to check for active dag run</span>
<span class="sd"> :param dag: DAG object</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">job_ids</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">ti</span> <span class="ow">in</span> <span class="n">tis</span><span class="p">:</span>
<span class="k">if</span> <span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">:</span>
<span class="k">if</span> <span class="n">ti</span><span class="o">.</span><span class="n">job_id</span><span class="p">:</span>
<span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SHUTDOWN</span>
<span class="n">job_ids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">job_id</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">task_id</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">task_id</span>
<span class="k">if</span> <span class="n">dag</span> <span class="ow">and</span> <span class="n">dag</span><span class="o">.</span><span class="n">has_task</span><span class="p">(</span><span class="n">task_id</span><span class="p">):</span>
<span class="n">task</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">task_id</span><span class="p">)</span>
<span class="n">task_retries</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">retries</span>
<span class="n">ti</span><span class="o">.</span><span class="n">max_tries</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">try_number</span> <span class="o">+</span> <span class="n">task_retries</span> <span class="o">-</span> <span class="mi">1</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># Ignore errors when updating max_tries if dag is None or</span>
<span class="c1"># task not found in dag since database records could be</span>
<span class="c1"># outdated. We make max_tries the maximum value of its</span>
<span class="c1"># original max_tries or the current task try number.</span>
<span class="n">ti</span><span class="o">.</span><span class="n">max_tries</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">max_tries</span><span class="p">,</span> <span class="n">ti</span><span class="o">.</span><span class="n">try_number</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">NONE</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span>
<span class="k">if</span> <span class="n">job_ids</span><span class="p">:</span>
<span class="kn">from</span> <span class="nn">airflow.jobs</span> <span class="k">import</span> <span class="n">BaseJob</span> <span class="k">as</span> <span class="n">BJ</span>
<span class="k">for</span> <span class="n">job</span> <span class="ow">in</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">BJ</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">BJ</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">job_ids</span><span class="p">))</span><span class="o">.</span><span class="n">all</span><span class="p">():</span>
<span class="n">job</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SHUTDOWN</span>
<span class="k">if</span> <span class="n">activate_dag_runs</span> <span class="ow">and</span> <span class="n">tis</span><span class="p">:</span>
<span class="n">drs</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">in_</span><span class="p">({</span><span class="n">ti</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">for</span> <span class="n">ti</span> <span class="ow">in</span> <span class="n">tis</span><span class="p">}),</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">in_</span><span class="p">({</span><span class="n">ti</span><span class="o">.</span><span class="n">execution_date</span> <span class="k">for</span> <span class="n">ti</span> <span class="ow">in</span> <span class="n">tis</span><span class="p">}),</span>
<span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
<span class="k">for</span> <span class="n">dr</span> <span class="ow">in</span> <span class="n">drs</span><span class="p">:</span>
<span class="n">dr</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span>
<span class="n">dr</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span></div>
<div class="viewcode-block" id="get_last_dagrun"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.get_last_dagrun">[docs]</a><span class="k">def</span> <span class="nf">get_last_dagrun</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">session</span><span class="p">,</span> <span class="n">include_externally_triggered</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the last dag run for a dag, None if there was none.</span>
<span class="sd"> Last dag run can be any type of run eg. scheduled or backfilled.</span>
<span class="sd"> Overridden DagRuns are ignored.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">DR</span> <span class="o">=</span> <span class="n">DagRun</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DR</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">include_externally_triggered</span><span class="p">:</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">external_trigger</span> <span class="o">==</span> <span class="kc">False</span><span class="p">)</span> <span class="c1"># noqa</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">desc</span><span class="p">())</span>
<span class="k">return</span> <span class="n">query</span><span class="o">.</span><span class="n">first</span><span class="p">()</span></div>
<div class="viewcode-block" id="DagBag"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagBag">[docs]</a><span class="k">class</span> <span class="nc">DagBag</span><span class="p">(</span><span class="n">BaseDagBag</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> A dagbag is a collection of dags, parsed out of a folder tree and has high</span>
<span class="sd"> level configuration settings, like what database to use as a backend and</span>
<span class="sd"> what executor to use to fire off tasks. This makes it easier to run</span>
<span class="sd"> distinct environments for say production and development, tests, or for</span>
<span class="sd"> different teams or security profiles. What would have been system level</span>
<span class="sd"> settings are now dagbag level so that one system can run multiple,</span>
<span class="sd"> independent settings sets.</span>
<span class="sd"> :param dag_folder: the folder to scan to find DAGs</span>
<span class="sd"> :type dag_folder: unicode</span>
<span class="sd"> :param executor: the executor to use when executing task instances</span>
<span class="sd"> in this DagBag</span>
<span class="sd"> :param include_examples: whether to include the examples that ship</span>
<span class="sd"> with airflow or not</span>
<span class="sd"> :type include_examples: bool</span>
<span class="sd"> :param has_logged: an instance boolean that gets flipped from False to True after a</span>
<span class="sd"> file has been skipped. This is to prevent overloading the user with logging</span>
<span class="sd"> messages about skipped files. Therefore only once per DagBag is a file logged</span>
<span class="sd"> being skipped.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># static class variables to detetct dag cycle</span>
<div class="viewcode-block" id="DagBag.CYCLE_NEW"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagBag.CYCLE_NEW">[docs]</a> <span class="n">CYCLE_NEW</span> <span class="o">=</span> <span class="mi">0</span></div>
<div class="viewcode-block" id="DagBag.CYCLE_IN_PROGRESS"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagBag.CYCLE_IN_PROGRESS">[docs]</a> <span class="n">CYCLE_IN_PROGRESS</span> <span class="o">=</span> <span class="mi">1</span></div>
<div class="viewcode-block" id="DagBag.CYCLE_DONE"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagBag.CYCLE_DONE">[docs]</a> <span class="n">CYCLE_DONE</span> <span class="o">=</span> <span class="mi">2</span></div>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">dag_folder</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">executor</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">include_examples</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;LOAD_EXAMPLES&#39;</span><span class="p">),</span>
<span class="n">safe_mode</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;DAG_DISCOVERY_SAFE_MODE&#39;</span><span class="p">)):</span>
<span class="c1"># do not use default arg in signature, to fix import cycle on plugin load</span>
<span class="k">if</span> <span class="n">executor</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">executor</span> <span class="o">=</span> <span class="n">GetDefaultExecutor</span><span class="p">()</span>
<span class="n">dag_folder</span> <span class="o">=</span> <span class="n">dag_folder</span> <span class="ow">or</span> <span class="n">settings</span><span class="o">.</span><span class="n">DAGS_FOLDER</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Filling up the DagBag from </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">dag_folder</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dag_folder</span> <span class="o">=</span> <span class="n">dag_folder</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dags</span> <span class="o">=</span> <span class="p">{}</span>
<span class="c1"># the file&#39;s last modified timestamp when we last read it</span>
<span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span> <span class="o">=</span> <span class="p">{}</span>
<span class="bp">self</span><span class="o">.</span><span class="n">executor</span> <span class="o">=</span> <span class="n">executor</span>
<span class="bp">self</span><span class="o">.</span><span class="n">import_errors</span> <span class="o">=</span> <span class="p">{}</span>
<span class="bp">self</span><span class="o">.</span><span class="n">has_logged</span> <span class="o">=</span> <span class="kc">False</span>
<span class="bp">self</span><span class="o">.</span><span class="n">collect_dags</span><span class="p">(</span>
<span class="n">dag_folder</span><span class="o">=</span><span class="n">dag_folder</span><span class="p">,</span>
<span class="n">include_examples</span><span class="o">=</span><span class="n">include_examples</span><span class="p">,</span>
<span class="n">safe_mode</span><span class="o">=</span><span class="n">safe_mode</span><span class="p">)</span>
<div class="viewcode-block" id="DagBag.size"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagBag.size">[docs]</a> <span class="k">def</span> <span class="nf">size</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> :return: the amount of dags contained in this dagbag</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">)</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DagBag.dag_ids"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagBag.dag_ids">[docs]</a> <span class="k">def</span> <span class="nf">dag_ids</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span></div>
<div class="viewcode-block" id="DagBag.get_dag"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagBag.get_dag">[docs]</a> <span class="k">def</span> <span class="nf">get_dag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Gets the DAG out of the dictionary, and refreshes it if expired</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># If asking for a known subdag, we want to refresh the parent</span>
<span class="n">root_dag_id</span> <span class="o">=</span> <span class="n">dag_id</span>
<span class="k">if</span> <span class="n">dag_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">:</span>
<span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">[</span><span class="n">dag_id</span><span class="p">]</span>
<span class="k">if</span> <span class="n">dag</span><span class="o">.</span><span class="n">is_subdag</span><span class="p">:</span>
<span class="n">root_dag_id</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">parent_dag</span><span class="o">.</span><span class="n">dag_id</span>
<span class="c1"># If the dag corresponding to root_dag_id is absent or expired</span>
<span class="n">orm_dag</span> <span class="o">=</span> <span class="n">DagModel</span><span class="o">.</span><span class="n">get_current</span><span class="p">(</span><span class="n">root_dag_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">orm_dag</span> <span class="ow">and</span> <span class="p">(</span>
<span class="n">root_dag_id</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span> <span class="ow">or</span>
<span class="p">(</span>
<span class="n">orm_dag</span><span class="o">.</span><span class="n">last_expired</span> <span class="ow">and</span>
<span class="n">dag</span><span class="o">.</span><span class="n">last_loaded</span> <span class="o">&lt;</span> <span class="n">orm_dag</span><span class="o">.</span><span class="n">last_expired</span>
<span class="p">)</span>
<span class="p">):</span>
<span class="c1"># Reprocess source file</span>
<span class="n">found_dags</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_file</span><span class="p">(</span>
<span class="n">filepath</span><span class="o">=</span><span class="n">orm_dag</span><span class="o">.</span><span class="n">fileloc</span><span class="p">,</span> <span class="n">only_if_updated</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="c1"># If the source file no longer exports `dag_id`, delete it from self.dags</span>
<span class="k">if</span> <span class="n">found_dags</span> <span class="ow">and</span> <span class="n">dag_id</span> <span class="ow">in</span> <span class="p">[</span><span class="n">found_dag</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">for</span> <span class="n">found_dag</span> <span class="ow">in</span> <span class="n">found_dags</span><span class="p">]:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">[</span><span class="n">dag_id</span><span class="p">]</span>
<span class="k">elif</span> <span class="n">dag_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">:</span>
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">[</span><span class="n">dag_id</span><span class="p">]</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">dag_id</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagBag.process_file"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagBag.process_file">[docs]</a> <span class="k">def</span> <span class="nf">process_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filepath</span><span class="p">,</span> <span class="n">only_if_updated</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">safe_mode</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Given a path to a python module or zip file, this method imports</span>
<span class="sd"> the module and look for dag objects within it.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">found_dags</span> <span class="o">=</span> <span class="p">[]</span>
<span class="c1"># if the source file no longer exists in the DB or in the filesystem,</span>
<span class="c1"># return an empty list</span>
<span class="c1"># todo: raise exception?</span>
<span class="k">if</span> <span class="n">filepath</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">filepath</span><span class="p">):</span>
<span class="k">return</span> <span class="n">found_dags</span>
<span class="k">try</span><span class="p">:</span>
<span class="c1"># This failed before in what may have been a git sync</span>
<span class="c1"># race condition</span>
<span class="n">file_last_changed_on_disk</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">getmtime</span><span class="p">(</span><span class="n">filepath</span><span class="p">))</span>
<span class="k">if</span> <span class="n">only_if_updated</span> \
<span class="ow">and</span> <span class="n">filepath</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span> \
<span class="ow">and</span> <span class="n">file_last_changed_on_disk</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]:</span>
<span class="k">return</span> <span class="n">found_dags</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
<span class="k">return</span> <span class="n">found_dags</span>
<span class="n">mods</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">is_zipfile</span> <span class="o">=</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">is_zipfile</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_zipfile</span><span class="p">:</span>
<span class="k">if</span> <span class="n">safe_mode</span> <span class="ow">and</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">filepath</span><span class="p">):</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filepath</span><span class="p">,</span> <span class="s1">&#39;rb&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">all</span><span class="p">([</span><span class="n">s</span> <span class="ow">in</span> <span class="n">content</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="p">(</span><span class="sa">b</span><span class="s1">&#39;DAG&#39;</span><span class="p">,</span> <span class="sa">b</span><span class="s1">&#39;airflow&#39;</span><span class="p">)]):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="n">file_last_changed_on_disk</span>
<span class="c1"># Don&#39;t want to spam user with skip messages</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_logged</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">has_logged</span> <span class="o">=</span> <span class="kc">True</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s2">&quot;File </span><span class="si">%s</span><span class="s2"> assumed to contain no DAGs. Skipping.&quot;</span><span class="p">,</span>
<span class="n">filepath</span><span class="p">)</span>
<span class="k">return</span> <span class="n">found_dags</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Importing </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">filepath</span><span class="p">)</span>
<span class="n">org_mod_name</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">splitext</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">filepath</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
<span class="n">mod_name</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;unusual_prefix_&#39;</span> <span class="o">+</span>
<span class="n">hashlib</span><span class="o">.</span><span class="n">sha1</span><span class="p">(</span><span class="n">filepath</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span> <span class="o">+</span>
<span class="s1">&#39;_&#39;</span> <span class="o">+</span> <span class="n">org_mod_name</span><span class="p">)</span>
<span class="k">if</span> <span class="n">mod_name</span> <span class="ow">in</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">:</span>
<span class="k">del</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">mod_name</span><span class="p">]</span>
<span class="k">with</span> <span class="n">timeout</span><span class="p">(</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s2">&quot;DAGBAG_IMPORT_TIMEOUT&quot;</span><span class="p">)):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">m</span> <span class="o">=</span> <span class="n">imp</span><span class="o">.</span><span class="n">load_source</span><span class="p">(</span><span class="n">mod_name</span><span class="p">,</span> <span class="n">filepath</span><span class="p">)</span>
<span class="n">mods</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">m</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">&quot;Failed to import: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">filepath</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">import_errors</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="n">file_last_changed_on_disk</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">zip_file</span> <span class="o">=</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
<span class="k">for</span> <span class="n">mod</span> <span class="ow">in</span> <span class="n">zip_file</span><span class="o">.</span><span class="n">infolist</span><span class="p">():</span>
<span class="n">head</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">mod</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
<span class="n">mod_name</span><span class="p">,</span> <span class="n">ext</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">splitext</span><span class="p">(</span><span class="n">mod</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">head</span> <span class="ow">and</span> <span class="p">(</span><span class="n">ext</span> <span class="o">==</span> <span class="s1">&#39;.py&#39;</span> <span class="ow">or</span> <span class="n">ext</span> <span class="o">==</span> <span class="s1">&#39;.pyc&#39;</span><span class="p">):</span>
<span class="k">if</span> <span class="n">mod_name</span> <span class="o">==</span> <span class="s1">&#39;__init__&#39;</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;Found __init__.</span><span class="si">%s</span><span class="s2"> at root of </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">ext</span><span class="p">,</span> <span class="n">filepath</span><span class="p">)</span>
<span class="k">if</span> <span class="n">safe_mode</span><span class="p">:</span>
<span class="k">with</span> <span class="n">zip_file</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">mod</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span> <span class="k">as</span> <span class="n">zf</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Reading </span><span class="si">%s</span><span class="s2"> from </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">mod</span><span class="o">.</span><span class="n">filename</span><span class="p">,</span> <span class="n">filepath</span><span class="p">)</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">zf</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">all</span><span class="p">([</span><span class="n">s</span> <span class="ow">in</span> <span class="n">content</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="p">(</span><span class="sa">b</span><span class="s1">&#39;DAG&#39;</span><span class="p">,</span> <span class="sa">b</span><span class="s1">&#39;airflow&#39;</span><span class="p">)]):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">file_last_changed_on_disk</span><span class="p">)</span>
<span class="c1"># todo: create ignore list</span>
<span class="c1"># Don&#39;t want to spam user with skip messages</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_logged</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">has_logged</span> <span class="o">=</span> <span class="kc">True</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s2">&quot;File </span><span class="si">%s</span><span class="s2"> assumed to contain no DAGs. Skipping.&quot;</span><span class="p">,</span>
<span class="n">filepath</span><span class="p">)</span>
<span class="k">if</span> <span class="n">mod_name</span> <span class="ow">in</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">:</span>
<span class="k">del</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">mod_name</span><span class="p">]</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">filepath</span><span class="p">)</span>
<span class="n">m</span> <span class="o">=</span> <span class="n">importlib</span><span class="o">.</span><span class="n">import_module</span><span class="p">(</span><span class="n">mod_name</span><span class="p">)</span>
<span class="n">mods</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">m</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">&quot;Failed to import: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">filepath</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">import_errors</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="n">file_last_changed_on_disk</span>
<span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">mods</span><span class="p">:</span>
<span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="n">m</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">values</span><span class="p">()):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dag</span><span class="p">,</span> <span class="n">DAG</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">:</span>
<span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span> <span class="o">=</span> <span class="n">filepath</span>
<span class="k">if</span> <span class="n">dag</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">!=</span> <span class="n">filepath</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">is_zipfile</span><span class="p">:</span>
<span class="n">dag</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">=</span> <span class="n">filepath</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">dag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="kc">False</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bag_dag</span><span class="p">(</span><span class="n">dag</span><span class="p">,</span> <span class="n">parent_dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> <span class="n">root_dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">_schedule_interval</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
<span class="n">croniter</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">_schedule_interval</span><span class="p">)</span>
<span class="n">found_dags</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dag</span><span class="p">)</span>
<span class="n">found_dags</span> <span class="o">+=</span> <span class="n">dag</span><span class="o">.</span><span class="n">subdags</span>
<span class="k">except</span> <span class="p">(</span><span class="n">CroniterBadCronError</span><span class="p">,</span>
<span class="n">CroniterBadDateError</span><span class="p">,</span>
<span class="n">CroniterNotAlphaError</span><span class="p">)</span> <span class="k">as</span> <span class="n">cron_e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">&quot;Failed to bag_dag: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">import_errors</span><span class="p">[</span><span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">]</span> <span class="o">=</span> \
<span class="s2">&quot;Invalid Cron expression: &quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">cron_e</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">]</span> <span class="o">=</span> \
<span class="n">file_last_changed_on_disk</span>
<span class="k">except</span> <span class="n">AirflowDagCycleException</span> <span class="k">as</span> <span class="n">cycle_exception</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">&quot;Failed to bag_dag: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">import_errors</span><span class="p">[</span><span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">cycle_exception</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">]</span> <span class="o">=</span> \
<span class="n">file_last_changed_on_disk</span>
<span class="bp">self</span><span class="o">.</span><span class="n">file_last_changed</span><span class="p">[</span><span class="n">filepath</span><span class="p">]</span> <span class="o">=</span> <span class="n">file_last_changed_on_disk</span>
<span class="k">return</span> <span class="n">found_dags</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagBag.kill_zombies"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagBag.kill_zombies">[docs]</a> <span class="k">def</span> <span class="nf">kill_zombies</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">zombies</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Fail given zombie tasks, which are tasks that haven&#39;t</span>
<span class="sd"> had a heartbeat for too long, in the current DagBag.</span>
<span class="sd"> :param zombies: zombie task instances to kill.</span>
<span class="sd"> :type zombies: airflow.utils.dag_processing.SimpleTaskInstance</span>
<span class="sd"> :param session: DB session.</span>
<span class="sd"> :type session: sqlalchemy.orm.session.Session</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">for</span> <span class="n">zombie</span> <span class="ow">in</span> <span class="n">zombies</span><span class="p">:</span>
<span class="k">if</span> <span class="n">zombie</span><span class="o">.</span><span class="n">dag_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">:</span>
<span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">[</span><span class="n">zombie</span><span class="o">.</span><span class="n">dag_id</span><span class="p">]</span>
<span class="k">if</span> <span class="n">zombie</span><span class="o">.</span><span class="n">task_id</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">task_ids</span><span class="p">:</span>
<span class="n">task</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">zombie</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
<span class="n">ti</span> <span class="o">=</span> <span class="n">TaskInstance</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="n">zombie</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span>
<span class="c1"># Get properties needed for failure handling from SimpleTaskInstance.</span>
<span class="n">ti</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">zombie</span><span class="o">.</span><span class="n">start_date</span>
<span class="n">ti</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">zombie</span><span class="o">.</span><span class="n">end_date</span>
<span class="n">ti</span><span class="o">.</span><span class="n">try_number</span> <span class="o">=</span> <span class="n">zombie</span><span class="o">.</span><span class="n">try_number</span>
<span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">zombie</span><span class="o">.</span><span class="n">state</span>
<span class="n">ti</span><span class="o">.</span><span class="n">test_mode</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;unit_test_mode&#39;</span><span class="p">)</span>
<span class="n">ti</span><span class="o">.</span><span class="n">handle_failure</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{}</span><span class="s2"> detected as zombie&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ti</span><span class="p">),</span>
<span class="n">ti</span><span class="o">.</span><span class="n">test_mode</span><span class="p">,</span> <span class="n">ti</span><span class="o">.</span><span class="n">get_template_context</span><span class="p">())</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s1">&#39;Marked zombie job </span><span class="si">%s</span><span class="s1"> as </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">ti</span><span class="p">,</span> <span class="n">ti</span><span class="o">.</span><span class="n">state</span><span class="p">)</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span><span class="s1">&#39;zombies_killed&#39;</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
<div class="viewcode-block" id="DagBag.bag_dag"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagBag.bag_dag">[docs]</a> <span class="k">def</span> <span class="nf">bag_dag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dag</span><span class="p">,</span> <span class="n">parent_dag</span><span class="p">,</span> <span class="n">root_dag</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Adds the DAG into the bag, recurses into sub dags.</span>
<span class="sd"> Throws AirflowDagCycleException if a cycle is detected in this dag or its subdags</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">dag</span><span class="o">.</span><span class="n">test_cycle</span><span class="p">()</span> <span class="c1"># throws if a task cycle is found</span>
<span class="n">dag</span><span class="o">.</span><span class="n">resolve_template_files</span><span class="p">()</span>
<span class="n">dag</span><span class="o">.</span><span class="n">last_loaded</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">tasks</span><span class="p">:</span>
<span class="n">settings</span><span class="o">.</span><span class="n">policy</span><span class="p">(</span><span class="n">task</span><span class="p">)</span>
<span class="n">subdags</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">subdags</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">for</span> <span class="n">subdag</span> <span class="ow">in</span> <span class="n">subdags</span><span class="p">:</span>
<span class="n">subdag</span><span class="o">.</span><span class="n">full_filepath</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span>
<span class="n">subdag</span><span class="o">.</span><span class="n">parent_dag</span> <span class="o">=</span> <span class="n">dag</span>
<span class="n">subdag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="kc">True</span>
<span class="bp">self</span><span class="o">.</span><span class="n">bag_dag</span><span class="p">(</span><span class="n">subdag</span><span class="p">,</span> <span class="n">parent_dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> <span class="n">root_dag</span><span class="o">=</span><span class="n">root_dag</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">[</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">dag</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;Loaded DAG </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">dag</span><span class="p">)</span>
<span class="k">except</span> <span class="n">AirflowDagCycleException</span> <span class="k">as</span> <span class="n">cycle_exception</span><span class="p">:</span>
<span class="c1"># There was an error in bagging the dag. Remove it from the list of dags</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">&#39;Exception bagging dag: </span><span class="si">{dag.dag_id}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
<span class="c1"># Only necessary at the root level since DAG.subdags automatically</span>
<span class="c1"># performs DFS to search through all subdags</span>
<span class="k">if</span> <span class="n">dag</span> <span class="o">==</span> <span class="n">root_dag</span><span class="p">:</span>
<span class="k">for</span> <span class="n">subdag</span> <span class="ow">in</span> <span class="n">subdags</span><span class="p">:</span>
<span class="k">if</span> <span class="n">subdag</span><span class="o">.</span><span class="n">dag_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">:</span>
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">[</span><span class="n">subdag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">]</span>
<span class="k">raise</span> <span class="n">cycle_exception</span></div>
<div class="viewcode-block" id="DagBag.collect_dags"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagBag.collect_dags">[docs]</a> <span class="k">def</span> <span class="nf">collect_dags</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">dag_folder</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">only_if_updated</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">include_examples</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;LOAD_EXAMPLES&#39;</span><span class="p">),</span>
<span class="n">safe_mode</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;DAG_DISCOVERY_SAFE_MODE&#39;</span><span class="p">)):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Given a file path or a folder, this method looks for python modules,</span>
<span class="sd"> imports them and adds them to the dagbag collection.</span>
<span class="sd"> Note that if a ``.airflowignore`` file is found while processing</span>
<span class="sd"> the directory, it will behave much like a ``.gitignore``,</span>
<span class="sd"> ignoring files that match any of the regex patterns specified</span>
<span class="sd"> in the file.</span>
<span class="sd"> **Note**: The patterns in .airflowignore are treated as</span>
<span class="sd"> un-anchored regexes, not shell-like glob patterns.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">start_dttm</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="n">dag_folder</span> <span class="o">=</span> <span class="n">dag_folder</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_folder</span>
<span class="c1"># Used to store stats around DagBag processing</span>
<span class="n">stats</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">FileLoadStat</span> <span class="o">=</span> <span class="n">namedtuple</span><span class="p">(</span>
<span class="s1">&#39;FileLoadStat&#39;</span><span class="p">,</span> <span class="s2">&quot;file duration dag_num task_num dags&quot;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">filepath</span> <span class="ow">in</span> <span class="n">list_py_file_paths</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">,</span> <span class="n">safe_mode</span><span class="o">=</span><span class="n">safe_mode</span><span class="p">,</span>
<span class="n">include_examples</span><span class="o">=</span><span class="n">include_examples</span><span class="p">):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">ts</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="n">found_dags</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_file</span><span class="p">(</span>
<span class="n">filepath</span><span class="p">,</span> <span class="n">only_if_updated</span><span class="o">=</span><span class="n">only_if_updated</span><span class="p">,</span>
<span class="n">safe_mode</span><span class="o">=</span><span class="n">safe_mode</span><span class="p">)</span>
<span class="n">td</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">ts</span>
<span class="n">td</span> <span class="o">=</span> <span class="n">td</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">+</span> <span class="p">(</span>
<span class="nb">float</span><span class="p">(</span><span class="n">td</span><span class="o">.</span><span class="n">microseconds</span><span class="p">)</span> <span class="o">/</span> <span class="mi">1000000</span><span class="p">)</span>
<span class="n">stats</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">FileLoadStat</span><span class="p">(</span>
<span class="n">filepath</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">dag_folder</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">),</span>
<span class="n">td</span><span class="p">,</span>
<span class="nb">len</span><span class="p">(</span><span class="n">found_dags</span><span class="p">),</span>
<span class="nb">sum</span><span class="p">([</span><span class="nb">len</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">tasks</span><span class="p">)</span> <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">found_dags</span><span class="p">]),</span>
<span class="nb">str</span><span class="p">([</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">found_dags</span><span class="p">]),</span>
<span class="p">))</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">gauge</span><span class="p">(</span>
<span class="s1">&#39;collect_dags&#39;</span><span class="p">,</span> <span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">start_dttm</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">(),</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">gauge</span><span class="p">(</span>
<span class="s1">&#39;dagbag_size&#39;</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dags</span><span class="p">),</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">gauge</span><span class="p">(</span>
<span class="s1">&#39;dagbag_import_errors&#39;</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">import_errors</span><span class="p">),</span> <span class="mi">1</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dagbag_stats</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span>
<span class="n">stats</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">duration</span><span class="p">,</span> <span class="n">reverse</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagBag.dagbag_report"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagBag.dagbag_report">[docs]</a> <span class="k">def</span> <span class="nf">dagbag_report</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Prints a report around DagBag loading stats&quot;&quot;&quot;</span>
<span class="n">report</span> <span class="o">=</span> <span class="n">textwrap</span><span class="o">.</span><span class="n">dedent</span><span class="p">(</span><span class="s2">&quot;&quot;&quot;</span><span class="se">\n</span><span class="s2"></span>
<span class="s2"> -------------------------------------------------------------------</span>
<span class="s2"> DagBag loading stats for </span><span class="si">{dag_folder}</span><span class="s2"></span>
<span class="s2"> -------------------------------------------------------------------</span>
<span class="s2"> Number of DAGs: </span><span class="si">{dag_num}</span><span class="s2"></span>
<span class="s2"> Total task number: </span><span class="si">{task_num}</span><span class="s2"></span>
<span class="s2"> DagBag parsing time: </span><span class="si">{duration}</span><span class="s2"></span>
<span class="s2"> </span><span class="si">{table}</span><span class="s2"></span>
<span class="s2"> &quot;&quot;&quot;</span><span class="p">)</span>
<span class="n">stats</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dagbag_stats</span>
<span class="k">return</span> <span class="n">report</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">dag_folder</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_folder</span><span class="p">,</span>
<span class="n">duration</span><span class="o">=</span><span class="nb">sum</span><span class="p">([</span><span class="n">o</span><span class="o">.</span><span class="n">duration</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">stats</span><span class="p">]),</span>
<span class="n">dag_num</span><span class="o">=</span><span class="nb">sum</span><span class="p">([</span><span class="n">o</span><span class="o">.</span><span class="n">dag_num</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">stats</span><span class="p">]),</span>
<span class="n">task_num</span><span class="o">=</span><span class="nb">sum</span><span class="p">([</span><span class="n">o</span><span class="o">.</span><span class="n">task_num</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">stats</span><span class="p">]),</span>
<span class="n">table</span><span class="o">=</span><span class="n">pprinttable</span><span class="p">(</span><span class="n">stats</span><span class="p">),</span></div></div>
<span class="p">)</span>
<div class="viewcode-block" id="User"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.User">[docs]</a><span class="k">class</span> <span class="nc">User</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span>
<div class="viewcode-block" id="User.__tablename__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.User.__tablename__">[docs]</a> <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">&quot;users&quot;</span></div>
<div class="viewcode-block" id="User.id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.User.id">[docs]</a> <span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="User.username"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.User.username">[docs]</a> <span class="n">username</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">unique</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="User.email"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.User.email">[docs]</a> <span class="n">email</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">500</span><span class="p">))</span></div>
<div class="viewcode-block" id="User.superuser"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.User.superuser">[docs]</a> <span class="n">superuser</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">(),</span> <span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<div class="viewcode-block" id="User.__repr__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.User.__repr__">[docs]</a> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span></div>
<div class="viewcode-block" id="User.get_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.User.get_id">[docs]</a> <span class="k">def</span> <span class="nf">get_id</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">)</span></div>
<div class="viewcode-block" id="User.is_superuser"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.User.is_superuser">[docs]</a> <span class="k">def</span> <span class="nf">is_superuser</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">superuser</span></div></div>
<div class="viewcode-block" id="TaskInstance"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance">[docs]</a><span class="k">class</span> <span class="nc">TaskInstance</span><span class="p">(</span><span class="n">Base</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Task instances store the state of a task instance. This table is the</span>
<span class="sd"> authority and single source of truth around what tasks have run and the</span>
<span class="sd"> state they are in.</span>
<span class="sd"> The SqlAlchemy model doesn&#39;t have a SqlAlchemy foreign key to the task or</span>
<span class="sd"> dag model deliberately to have more control over transactions.</span>
<span class="sd"> Database transactions on this table should insure double triggers and</span>
<span class="sd"> any confusion around what task instances are or aren&#39;t ready to run</span>
<span class="sd"> even while multiple schedulers may be firing task instances.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<div class="viewcode-block" id="TaskInstance.__tablename__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.__tablename__">[docs]</a> <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">&quot;task_instance&quot;</span></div>
<div class="viewcode-block" id="TaskInstance.task_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.task_id">[docs]</a> <span class="n">task_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.dag_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.dag_id">[docs]</a> <span class="n">dag_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.execution_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.execution_date">[docs]</a> <span class="n">execution_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.start_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.start_date">[docs]</a> <span class="n">start_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.end_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.end_date">[docs]</a> <span class="n">end_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.duration"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.duration">[docs]</a> <span class="n">duration</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Float</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.state"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.state">[docs]</a> <span class="n">state</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">20</span><span class="p">))</span></div>
<div class="viewcode-block" id="TaskInstance._try_number"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance._try_number">[docs]</a> <span class="n">_try_number</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="s1">&#39;try_number&#39;</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.max_tries"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.max_tries">[docs]</a> <span class="n">max_tries</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.hostname"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.hostname">[docs]</a> <span class="n">hostname</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">1000</span><span class="p">))</span></div>
<div class="viewcode-block" id="TaskInstance.unixname"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.unixname">[docs]</a> <span class="n">unixname</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">1000</span><span class="p">))</span></div>
<div class="viewcode-block" id="TaskInstance.job_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.job_id">[docs]</a> <span class="n">job_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.pool"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.pool">[docs]</a> <span class="n">pool</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">50</span><span class="p">))</span></div>
<div class="viewcode-block" id="TaskInstance.queue"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.queue">[docs]</a> <span class="n">queue</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">50</span><span class="p">))</span></div>
<div class="viewcode-block" id="TaskInstance.priority_weight"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.priority_weight">[docs]</a> <span class="n">priority_weight</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.operator"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.operator">[docs]</a> <span class="n">operator</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">1000</span><span class="p">))</span></div>
<div class="viewcode-block" id="TaskInstance.queued_dttm"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.queued_dttm">[docs]</a> <span class="n">queued_dttm</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.pid"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.pid">[docs]</a> <span class="n">pid</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.executor_config"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.executor_config">[docs]</a> <span class="n">executor_config</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">PickleType</span><span class="p">(</span><span class="n">pickler</span><span class="o">=</span><span class="n">dill</span><span class="p">))</span></div>
<div class="viewcode-block" id="TaskInstance.__table_args__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.__table_args__">[docs]</a> <span class="n">__table_args__</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_dag_state&#39;</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
<span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_dag_date&#39;</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">),</span>
<span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_state&#39;</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
<span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_state_lkp&#39;</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
<span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_pool&#39;</span><span class="p">,</span> <span class="n">pool</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">priority_weight</span><span class="p">),</span>
<span class="n">Index</span><span class="p">(</span><span class="s1">&#39;ti_job_id&#39;</span><span class="p">,</span> <span class="n">job_id</span><span class="p">),</span></div>
<span class="p">)</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">,</span> <span class="n">state</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">dag_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_id</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">task</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;airflow.task&quot;</span><span class="p">)</span>
<span class="c1"># make sure we have a localized execution_date stored in UTC</span>
<span class="k">if</span> <span class="n">execution_date</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">timezone</span><span class="o">.</span><span class="n">is_localized</span><span class="p">(</span><span class="n">execution_date</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;execution date </span><span class="si">%s</span><span class="s2"> has no timezone information. Using &quot;</span>
<span class="s2">&quot;default from dag or system&quot;</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">has_dag</span><span class="p">():</span>
<span class="n">execution_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">make_aware</span><span class="p">(</span><span class="n">execution_date</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">timezone</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">execution_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">make_aware</span><span class="p">(</span><span class="n">execution_date</span><span class="p">)</span>
<span class="n">execution_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="n">execution_date</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">=</span> <span class="n">execution_date</span>
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">queue</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pool</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">pool</span>
<span class="bp">self</span><span class="o">.</span><span class="n">priority_weight</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">priority_weight_total</span>
<span class="bp">self</span><span class="o">.</span><span class="n">try_number</span> <span class="o">=</span> <span class="mi">0</span>
<span class="bp">self</span><span class="o">.</span><span class="n">max_tries</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">retries</span>
<span class="bp">self</span><span class="o">.</span><span class="n">unixname</span> <span class="o">=</span> <span class="n">getpass</span><span class="o">.</span><span class="n">getuser</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">run_as_user</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">run_as_user</span>
<span class="k">if</span> <span class="n">state</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">state</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hostname</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">executor_config</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">executor_config</span>
<span class="bp">self</span><span class="o">.</span><span class="n">init_on_load</span><span class="p">()</span>
<span class="c1"># Is this TaskInstance being currently running within `airflow run --raw`.</span>
<span class="c1"># Not persisted to the database so only valid for the current process</span>
<span class="bp">self</span><span class="o">.</span><span class="n">raw</span> <span class="o">=</span> <span class="kc">False</span>
<span class="nd">@reconstructor</span>
<div class="viewcode-block" id="TaskInstance.init_on_load"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.init_on_load">[docs]</a> <span class="k">def</span> <span class="nf">init_on_load</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot; Initialize the attributes that aren&#39;t stored in the DB. &quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">test_mode</span> <span class="o">=</span> <span class="kc">False</span> <span class="c1"># can be changed when calling &#39;run&#39;</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="TaskInstance.try_number"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.try_number">[docs]</a> <span class="k">def</span> <span class="nf">try_number</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Return the try number that this task number will be when it is actually</span>
<span class="sd"> run.</span>
<span class="sd"> If the TI is currently running, this will match the column in the</span>
<span class="sd"> databse, in all othercases this will be incremenetd</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># This is designed so that task logs end up in the right file.</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_try_number</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_try_number</span> <span class="o">+</span> <span class="mi">1</span></div>
<span class="nd">@try_number</span><span class="o">.</span><span class="n">setter</span>
<span class="k">def</span> <span class="nf">try_number</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_try_number</span> <span class="o">=</span> <span class="n">value</span>
<span class="nd">@property</span>
<div class="viewcode-block" id="TaskInstance.next_try_number"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.next_try_number">[docs]</a> <span class="k">def</span> <span class="nf">next_try_number</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_try_number</span> <span class="o">+</span> <span class="mi">1</span></div>
<div class="viewcode-block" id="TaskInstance.command"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.command">[docs]</a> <span class="k">def</span> <span class="nf">command</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">mark_success</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_all_deps</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_task_deps</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_ti_state</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">local</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">pickle_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">raw</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">job_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">cfg_path</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a command that can be executed anywhere where airflow is</span>
<span class="sd"> installed. This command is part of the message sent to executors by</span>
<span class="sd"> the orchestrator.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">command_as_list</span><span class="p">(</span>
<span class="n">mark_success</span><span class="o">=</span><span class="n">mark_success</span><span class="p">,</span>
<span class="n">ignore_all_deps</span><span class="o">=</span><span class="n">ignore_all_deps</span><span class="p">,</span>
<span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="n">ignore_depends_on_past</span><span class="p">,</span>
<span class="n">ignore_task_deps</span><span class="o">=</span><span class="n">ignore_task_deps</span><span class="p">,</span>
<span class="n">ignore_ti_state</span><span class="o">=</span><span class="n">ignore_ti_state</span><span class="p">,</span>
<span class="n">local</span><span class="o">=</span><span class="n">local</span><span class="p">,</span>
<span class="n">pickle_id</span><span class="o">=</span><span class="n">pickle_id</span><span class="p">,</span>
<span class="n">raw</span><span class="o">=</span><span class="n">raw</span><span class="p">,</span>
<span class="n">job_id</span><span class="o">=</span><span class="n">job_id</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="n">pool</span><span class="p">,</span>
<span class="n">cfg_path</span><span class="o">=</span><span class="n">cfg_path</span><span class="p">))</span></div>
<div class="viewcode-block" id="TaskInstance.command_as_list"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.command_as_list">[docs]</a> <span class="k">def</span> <span class="nf">command_as_list</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">mark_success</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_all_deps</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_task_deps</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_ti_state</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">local</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">pickle_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">raw</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">job_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">cfg_path</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a command that can be executed anywhere where airflow is</span>
<span class="sd"> installed. This command is part of the message sent to executors by</span>
<span class="sd"> the orchestrator.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">dag</span>
<span class="n">should_pass_filepath</span> <span class="o">=</span> <span class="ow">not</span> <span class="n">pickle_id</span> <span class="ow">and</span> <span class="n">dag</span>
<span class="k">if</span> <span class="n">should_pass_filepath</span> <span class="ow">and</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span> <span class="o">!=</span> <span class="n">dag</span><span class="o">.</span><span class="n">filepath</span><span class="p">:</span>
<span class="n">path</span> <span class="o">=</span> <span class="s2">&quot;DAGS_FOLDER/</span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">filepath</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">should_pass_filepath</span> <span class="ow">and</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">:</span>
<span class="n">path</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">full_filepath</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">path</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">return</span> <span class="n">TaskInstance</span><span class="o">.</span><span class="n">generate_command</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span>
<span class="n">mark_success</span><span class="o">=</span><span class="n">mark_success</span><span class="p">,</span>
<span class="n">ignore_all_deps</span><span class="o">=</span><span class="n">ignore_all_deps</span><span class="p">,</span>
<span class="n">ignore_task_deps</span><span class="o">=</span><span class="n">ignore_task_deps</span><span class="p">,</span>
<span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="n">ignore_depends_on_past</span><span class="p">,</span>
<span class="n">ignore_ti_state</span><span class="o">=</span><span class="n">ignore_ti_state</span><span class="p">,</span>
<span class="n">local</span><span class="o">=</span><span class="n">local</span><span class="p">,</span>
<span class="n">pickle_id</span><span class="o">=</span><span class="n">pickle_id</span><span class="p">,</span>
<span class="n">file_path</span><span class="o">=</span><span class="n">path</span><span class="p">,</span>
<span class="n">raw</span><span class="o">=</span><span class="n">raw</span><span class="p">,</span>
<span class="n">job_id</span><span class="o">=</span><span class="n">job_id</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="n">pool</span><span class="p">,</span>
<span class="n">cfg_path</span><span class="o">=</span><span class="n">cfg_path</span><span class="p">)</span></div>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="TaskInstance.generate_command"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.generate_command">[docs]</a> <span class="k">def</span> <span class="nf">generate_command</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">task_id</span><span class="p">,</span>
<span class="n">execution_date</span><span class="p">,</span>
<span class="n">mark_success</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_all_deps</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_task_deps</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_ti_state</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">local</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">pickle_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">file_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">raw</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">job_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">cfg_path</span><span class="o">=</span><span class="kc">None</span>
<span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Generates the shell command required to execute this task instance.</span>
<span class="sd"> :param dag_id: DAG ID</span>
<span class="sd"> :type dag_id: unicode</span>
<span class="sd"> :param task_id: Task ID</span>
<span class="sd"> :type task_id: unicode</span>
<span class="sd"> :param execution_date: Execution date for the task</span>
<span class="sd"> :type execution_date: datetime</span>
<span class="sd"> :param mark_success: Whether to mark the task as successful</span>
<span class="sd"> :type mark_success: bool</span>
<span class="sd"> :param ignore_all_deps: Ignore all ignorable dependencies.</span>
<span class="sd"> Overrides the other ignore_* parameters.</span>
<span class="sd"> :type ignore_all_deps: bool</span>
<span class="sd"> :param ignore_depends_on_past: Ignore depends_on_past parameter of DAGs</span>
<span class="sd"> (e.g. for Backfills)</span>
<span class="sd"> :type ignore_depends_on_past: bool</span>
<span class="sd"> :param ignore_task_deps: Ignore task-specific dependencies such as depends_on_past</span>
<span class="sd"> and trigger rule</span>
<span class="sd"> :type ignore_task_deps: bool</span>
<span class="sd"> :param ignore_ti_state: Ignore the task instance&#39;s previous failure/success</span>
<span class="sd"> :type ignore_ti_state: bool</span>
<span class="sd"> :param local: Whether to run the task locally</span>
<span class="sd"> :type local: bool</span>
<span class="sd"> :param pickle_id: If the DAG was serialized to the DB, the ID</span>
<span class="sd"> associated with the pickled DAG</span>
<span class="sd"> :type pickle_id: unicode</span>
<span class="sd"> :param file_path: path to the file containing the DAG definition</span>
<span class="sd"> :param raw: raw mode (needs more details)</span>
<span class="sd"> :param job_id: job ID (needs more details)</span>
<span class="sd"> :param pool: the Airflow pool that the task should run in</span>
<span class="sd"> :type pool: unicode</span>
<span class="sd"> :param cfg_path: the Path to the configuration file</span>
<span class="sd"> :type cfg_path: basestring</span>
<span class="sd"> :return: shell command that can be used to run the task instance</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">iso</span> <span class="o">=</span> <span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
<span class="n">cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;airflow&quot;</span><span class="p">,</span> <span class="s2">&quot;run&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">dag_id</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_id</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">iso</span><span class="p">)]</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s2">&quot;--mark_success&quot;</span><span class="p">])</span> <span class="k">if</span> <span class="n">mark_success</span> <span class="k">else</span> <span class="kc">None</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s2">&quot;--pickle&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">pickle_id</span><span class="p">)])</span> <span class="k">if</span> <span class="n">pickle_id</span> <span class="k">else</span> <span class="kc">None</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s2">&quot;--job_id&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">job_id</span><span class="p">)])</span> <span class="k">if</span> <span class="n">job_id</span> <span class="k">else</span> <span class="kc">None</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s2">&quot;-A&quot;</span><span class="p">])</span> <span class="k">if</span> <span class="n">ignore_all_deps</span> <span class="k">else</span> <span class="kc">None</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s2">&quot;-i&quot;</span><span class="p">])</span> <span class="k">if</span> <span class="n">ignore_task_deps</span> <span class="k">else</span> <span class="kc">None</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s2">&quot;-I&quot;</span><span class="p">])</span> <span class="k">if</span> <span class="n">ignore_depends_on_past</span> <span class="k">else</span> <span class="kc">None</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s2">&quot;--force&quot;</span><span class="p">])</span> <span class="k">if</span> <span class="n">ignore_ti_state</span> <span class="k">else</span> <span class="kc">None</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s2">&quot;--local&quot;</span><span class="p">])</span> <span class="k">if</span> <span class="n">local</span> <span class="k">else</span> <span class="kc">None</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s2">&quot;--pool&quot;</span><span class="p">,</span> <span class="n">pool</span><span class="p">])</span> <span class="k">if</span> <span class="n">pool</span> <span class="k">else</span> <span class="kc">None</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s2">&quot;--raw&quot;</span><span class="p">])</span> <span class="k">if</span> <span class="n">raw</span> <span class="k">else</span> <span class="kc">None</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s2">&quot;-sd&quot;</span><span class="p">,</span> <span class="n">file_path</span><span class="p">])</span> <span class="k">if</span> <span class="n">file_path</span> <span class="k">else</span> <span class="kc">None</span>
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">([</span><span class="s2">&quot;--cfg_path&quot;</span><span class="p">,</span> <span class="n">cfg_path</span><span class="p">])</span> <span class="k">if</span> <span class="n">cfg_path</span> <span class="k">else</span> <span class="kc">None</span>
<span class="k">return</span> <span class="n">cmd</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="TaskInstance.log_filepath"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.log_filepath">[docs]</a> <span class="k">def</span> <span class="nf">log_filepath</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">iso</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
<span class="n">log</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;BASE_LOG_FOLDER&#39;</span><span class="p">))</span>
<span class="k">return</span> <span class="p">(</span>
<span class="s2">&quot;</span><span class="si">{log}</span><span class="s2">/</span><span class="si">{self.dag_id}</span><span class="s2">/</span><span class="si">{self.task_id}</span><span class="s2">/</span><span class="si">{iso}</span><span class="s2">.log&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="TaskInstance.log_url"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.log_url">[docs]</a> <span class="k">def</span> <span class="nf">log_url</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">iso</span> <span class="o">=</span> <span class="n">quote</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">())</span>
<span class="n">BASE_URL</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;webserver&#39;</span><span class="p">,</span> <span class="s1">&#39;BASE_URL&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">settings</span><span class="o">.</span><span class="n">RBAC</span><span class="p">:</span>
<span class="k">return</span> <span class="n">BASE_URL</span> <span class="o">+</span> <span class="p">(</span>
<span class="s2">&quot;/log?&quot;</span>
<span class="s2">&quot;execution_date=</span><span class="si">{iso}</span><span class="s2">&quot;</span>
<span class="s2">&quot;&amp;task_id=</span><span class="si">{self.task_id}</span><span class="s2">&quot;</span>
<span class="s2">&quot;&amp;dag_id=</span><span class="si">{self.dag_id}</span><span class="s2">&quot;</span>
<span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">BASE_URL</span> <span class="o">+</span> <span class="p">(</span>
<span class="s2">&quot;/admin/airflow/log&quot;</span>
<span class="s2">&quot;?dag_id=</span><span class="si">{self.dag_id}</span><span class="s2">&quot;</span>
<span class="s2">&quot;&amp;task_id=</span><span class="si">{self.task_id}</span><span class="s2">&quot;</span>
<span class="s2">&quot;&amp;execution_date=</span><span class="si">{iso}</span><span class="s2">&quot;</span>
<span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="TaskInstance.mark_success_url"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.mark_success_url">[docs]</a> <span class="k">def</span> <span class="nf">mark_success_url</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">iso</span> <span class="o">=</span> <span class="n">quote</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">())</span>
<span class="n">BASE_URL</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;webserver&#39;</span><span class="p">,</span> <span class="s1">&#39;BASE_URL&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">settings</span><span class="o">.</span><span class="n">RBAC</span><span class="p">:</span>
<span class="k">return</span> <span class="n">BASE_URL</span> <span class="o">+</span> <span class="p">(</span>
<span class="s2">&quot;/success&quot;</span>
<span class="s2">&quot;?task_id=</span><span class="si">{self.task_id}</span><span class="s2">&quot;</span>
<span class="s2">&quot;&amp;dag_id=</span><span class="si">{self.dag_id}</span><span class="s2">&quot;</span>
<span class="s2">&quot;&amp;execution_date=</span><span class="si">{iso}</span><span class="s2">&quot;</span>
<span class="s2">&quot;&amp;upstream=false&quot;</span>
<span class="s2">&quot;&amp;downstream=false&quot;</span>
<span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">BASE_URL</span> <span class="o">+</span> <span class="p">(</span>
<span class="s2">&quot;/admin/airflow/success&quot;</span>
<span class="s2">&quot;?task_id=</span><span class="si">{self.task_id}</span><span class="s2">&quot;</span>
<span class="s2">&quot;&amp;dag_id=</span><span class="si">{self.dag_id}</span><span class="s2">&quot;</span>
<span class="s2">&quot;&amp;execution_date=</span><span class="si">{iso}</span><span class="s2">&quot;</span>
<span class="s2">&quot;&amp;upstream=false&quot;</span>
<span class="s2">&quot;&amp;downstream=false&quot;</span>
<span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.current_state"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.current_state">[docs]</a> <span class="k">def</span> <span class="nf">current_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Get the very latest state from the database, if a session is passed,</span>
<span class="sd"> we use and looking up the state becomes part of the session, otherwise</span>
<span class="sd"> a new session is used.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
<span class="n">ti</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span>
<span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
<span class="k">if</span> <span class="n">ti</span><span class="p">:</span>
<span class="n">state</span> <span class="o">=</span> <span class="n">ti</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">state</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">state</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">return</span> <span class="n">state</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.error"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.error">[docs]</a> <span class="k">def</span> <span class="nf">error</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Forces the task instance&#39;s state to FAILED in the database.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Recording the task instance as FAILED&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.refresh_from_db"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.refresh_from_db">[docs]</a> <span class="k">def</span> <span class="nf">refresh_from_db</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">lock_for_update</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Refreshes the task instance from the database based on the primary key</span>
<span class="sd"> :param lock_for_update: if True, indicates that the database should</span>
<span class="sd"> lock the TaskInstance (issuing a FOR UPDATE clause) until the</span>
<span class="sd"> session is committed.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span>
<span class="k">if</span> <span class="n">lock_for_update</span><span class="p">:</span>
<span class="n">ti</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">with_for_update</span><span class="p">()</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">ti</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="k">if</span> <span class="n">ti</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">state</span>
<span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">start_date</span>
<span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">end_date</span>
<span class="c1"># Get the raw value of try_number column, don&#39;t read through the</span>
<span class="c1"># accessor here otherwise it will be incremeneted by one already.</span>
<span class="bp">self</span><span class="o">.</span><span class="n">try_number</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">_try_number</span>
<span class="bp">self</span><span class="o">.</span><span class="n">max_tries</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">max_tries</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hostname</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">hostname</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pid</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">pid</span>
<span class="bp">self</span><span class="o">.</span><span class="n">executor_config</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">executor_config</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="kc">None</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.clear_xcom_data"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.clear_xcom_data">[docs]</a> <span class="k">def</span> <span class="nf">clear_xcom_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Clears all XCom data from the database for the task instance</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">XCom</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">XCom</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">XCom</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
<span class="n">XCom</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span>
<span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="TaskInstance.key"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.key">[docs]</a> <span class="k">def</span> <span class="nf">key</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a tuple that identifies the task instance uniquely</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">try_number</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.set_state"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.set_state">[docs]</a> <span class="k">def</span> <span class="nf">set_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">state</span>
<span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="TaskInstance.is_premature"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.is_premature">[docs]</a> <span class="k">def</span> <span class="nf">is_premature</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns whether a task is in UP_FOR_RETRY state and its retry interval</span>
<span class="sd"> has elapsed.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># is the task still in the retry waiting period?</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">UP_FOR_RETRY</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">ready_for_retry</span><span class="p">()</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.are_dependents_done"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.are_dependents_done">[docs]</a> <span class="k">def</span> <span class="nf">are_dependents_done</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Checks whether the dependents of this task instance have all succeeded.</span>
<span class="sd"> This is meant to be used by wait_for_downstream.</span>
<span class="sd"> This is useful when you do not want to start processing the next</span>
<span class="sd"> schedule of a task until the dependents are done. For instance,</span>
<span class="sd"> if the task DROPs and recreates a table.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">task</span><span class="o">.</span><span class="n">downstream_task_ids</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="n">ti</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">func</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">task_id</span><span class="p">))</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">TaskInstance</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">TaskInstance</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">downstream_task_ids</span><span class="p">),</span>
<span class="n">TaskInstance</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span>
<span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">count</span> <span class="o">=</span> <span class="n">ti</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
<span class="k">return</span> <span class="n">count</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">downstream_task_ids</span><span class="p">)</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance._get_previous_ti"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance._get_previous_ti">[docs]</a> <span class="k">def</span> <span class="nf">_get_previous_ti</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">dag</span>
<span class="k">if</span> <span class="n">dag</span><span class="p">:</span>
<span class="n">dr</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_dagrun</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="c1"># LEGACY: most likely running from unit tests</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">dr</span><span class="p">:</span>
<span class="c1"># Means that this TI is NOT being run from a DR, but from a catchup</span>
<span class="n">previous_scheduled_date</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">previous_schedule</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">previous_scheduled_date</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">return</span> <span class="n">TaskInstance</span><span class="p">(</span><span class="n">task</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="n">previous_scheduled_date</span><span class="p">)</span>
<span class="n">dr</span><span class="o">.</span><span class="n">dag</span> <span class="o">=</span> <span class="n">dag</span>
<span class="k">if</span> <span class="n">dag</span><span class="o">.</span><span class="n">catchup</span><span class="p">:</span>
<span class="n">last_dagrun</span> <span class="o">=</span> <span class="n">dr</span><span class="o">.</span><span class="n">get_previous_scheduled_dagrun</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">last_dagrun</span> <span class="o">=</span> <span class="n">dr</span><span class="o">.</span><span class="n">get_previous_dagrun</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="k">if</span> <span class="n">last_dagrun</span><span class="p">:</span>
<span class="k">return</span> <span class="n">last_dagrun</span><span class="o">.</span><span class="n">get_task_instance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">None</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="TaskInstance.previous_ti"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.previous_ti">[docs]</a> <span class="k">def</span> <span class="nf">previous_ti</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;The task instance for the task that ran before this task instance.&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_previous_ti</span><span class="p">()</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.are_dependencies_met"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.are_dependencies_met">[docs]</a> <span class="k">def</span> <span class="nf">are_dependencies_met</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">dep_context</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns whether or not all the conditions are met for this task instance to be run</span>
<span class="sd"> given the context for the dependencies (e.g. a task instance being force run from</span>
<span class="sd"> the UI will ignore some dependencies).</span>
<span class="sd"> :param dep_context: The execution context that determines the dependencies that</span>
<span class="sd"> should be evaluated.</span>
<span class="sd"> :type dep_context: DepContext</span>
<span class="sd"> :param session: database session</span>
<span class="sd"> :type session: sqlalchemy.orm.session.Session</span>
<span class="sd"> :param verbose: whether log details on failed dependencies on</span>
<span class="sd"> info or debug log level</span>
<span class="sd"> :type verbose: bool</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">dep_context</span> <span class="o">=</span> <span class="n">dep_context</span> <span class="ow">or</span> <span class="n">DepContext</span><span class="p">()</span>
<span class="n">failed</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">verbose_aware_logger</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span> <span class="k">if</span> <span class="n">verbose</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span>
<span class="k">for</span> <span class="n">dep_status</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_failed_dep_statuses</span><span class="p">(</span>
<span class="n">dep_context</span><span class="o">=</span><span class="n">dep_context</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">):</span>
<span class="n">failed</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">verbose_aware_logger</span><span class="p">(</span>
<span class="s2">&quot;Dependencies not met for </span><span class="si">%s</span><span class="s2">, dependency &#39;</span><span class="si">%s</span><span class="s2">&#39; FAILED: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="bp">self</span><span class="p">,</span> <span class="n">dep_status</span><span class="o">.</span><span class="n">dep_name</span><span class="p">,</span> <span class="n">dep_status</span><span class="o">.</span><span class="n">reason</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">failed</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="n">verbose_aware_logger</span><span class="p">(</span><span class="s2">&quot;Dependencies all met for </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.get_failed_dep_statuses"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.get_failed_dep_statuses">[docs]</a> <span class="k">def</span> <span class="nf">get_failed_dep_statuses</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">dep_context</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">dep_context</span> <span class="o">=</span> <span class="n">dep_context</span> <span class="ow">or</span> <span class="n">DepContext</span><span class="p">()</span>
<span class="k">for</span> <span class="n">dep</span> <span class="ow">in</span> <span class="n">dep_context</span><span class="o">.</span><span class="n">deps</span> <span class="o">|</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">deps</span><span class="p">:</span>
<span class="k">for</span> <span class="n">dep_status</span> <span class="ow">in</span> <span class="n">dep</span><span class="o">.</span><span class="n">get_dep_statuses</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">session</span><span class="p">,</span>
<span class="n">dep_context</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
<span class="s2">&quot;</span><span class="si">%s</span><span class="s2"> dependency &#39;</span><span class="si">%s</span><span class="s2">&#39; PASSED: </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="bp">self</span><span class="p">,</span> <span class="n">dep_status</span><span class="o">.</span><span class="n">dep_name</span><span class="p">,</span> <span class="n">dep_status</span><span class="o">.</span><span class="n">passed</span><span class="p">,</span> <span class="n">dep_status</span><span class="o">.</span><span class="n">reason</span>
<span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">dep_status</span><span class="o">.</span><span class="n">passed</span><span class="p">:</span>
<span class="k">yield</span> <span class="n">dep_status</span></div>
<div class="viewcode-block" id="TaskInstance.__repr__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.__repr__">[docs]</a> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">(</span>
<span class="s2">&quot;&lt;TaskInstance: </span><span class="si">{ti.dag_id}</span><span class="s2">.</span><span class="si">{ti.task_id}</span><span class="s2"> &quot;</span>
<span class="s2">&quot;</span><span class="si">{ti.execution_date}</span><span class="s2"> [</span><span class="si">{ti.state}</span><span class="s2">]&gt;&quot;</span>
<span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ti</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.next_retry_datetime"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.next_retry_datetime">[docs]</a> <span class="k">def</span> <span class="nf">next_retry_datetime</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Get datetime of the next retry if the task instance fails. For exponential</span>
<span class="sd"> backoff, retry_delay is used as base and will be converted to seconds.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">delay</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">retry_delay</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">retry_exponential_backoff</span><span class="p">:</span>
<span class="n">min_backoff</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">delay</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">*</span> <span class="p">(</span><span class="mi">2</span> <span class="o">**</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">try_number</span> <span class="o">-</span> <span class="mi">2</span><span class="p">)))</span>
<span class="c1"># deterministic per task instance</span>
<span class="nb">hash</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">hashlib</span><span class="o">.</span><span class="n">sha1</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{}</span><span class="s2">#</span><span class="si">{}</span><span class="s2">#</span><span class="si">{}</span><span class="s2">#</span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">try_number</span><span class="p">)</span>
<span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">(),</span> <span class="mi">16</span><span class="p">)</span>
<span class="c1"># between 0.5 * delay * (2^retry_number) and 1.0 * delay * (2^retry_number)</span>
<span class="n">modded_hash</span> <span class="o">=</span> <span class="n">min_backoff</span> <span class="o">+</span> <span class="nb">hash</span> <span class="o">%</span> <span class="n">min_backoff</span>
<span class="c1"># timedelta has a maximum representable value. The exponentiation</span>
<span class="c1"># here means this value can be exceeded after a certain number</span>
<span class="c1"># of tries (around 50 if the initial delay is 1s, even fewer if</span>
<span class="c1"># the delay is larger). Cap the value here before creating a</span>
<span class="c1"># timedelta object so the operation doesn&#39;t fail.</span>
<span class="n">delay_backoff_in_seconds</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span>
<span class="n">modded_hash</span><span class="p">,</span>
<span class="n">timedelta</span><span class="o">.</span><span class="n">max</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">-</span> <span class="mi">1</span>
<span class="p">)</span>
<span class="n">delay</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="n">delay_backoff_in_seconds</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">max_retry_delay</span><span class="p">:</span>
<span class="n">delay</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">max_retry_delay</span><span class="p">,</span> <span class="n">delay</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">+</span> <span class="n">delay</span></div>
<div class="viewcode-block" id="TaskInstance.ready_for_retry"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.ready_for_retry">[docs]</a> <span class="k">def</span> <span class="nf">ready_for_retry</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Checks on whether the task instance is in the right state and timeframe</span>
<span class="sd"> to be retried.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">UP_FOR_RETRY</span> <span class="ow">and</span>
<span class="bp">self</span><span class="o">.</span><span class="n">next_retry_datetime</span><span class="p">()</span> <span class="o">&lt;</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">())</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.pool_full"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.pool_full">[docs]</a> <span class="k">def</span> <span class="nf">pool_full</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a boolean as to whether the slot pool has room for this</span>
<span class="sd"> task to run</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">pool</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="n">pool</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">session</span>
<span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Pool</span><span class="p">)</span>
<span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">Pool</span><span class="o">.</span><span class="n">pool</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">pool</span><span class="p">)</span>
<span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">pool</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="n">open_slots</span> <span class="o">=</span> <span class="n">pool</span><span class="o">.</span><span class="n">open_slots</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="k">return</span> <span class="n">open_slots</span> <span class="o">&lt;=</span> <span class="mi">0</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.get_dagrun"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.get_dagrun">[docs]</a> <span class="k">def</span> <span class="nf">get_dagrun</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the DagRun for this TaskInstance</span>
<span class="sd"> :param session:</span>
<span class="sd"> :return: DagRun</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">dr</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span>
<span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="k">return</span> <span class="n">dr</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance._check_and_change_state_before_execution"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance._check_and_change_state_before_execution">[docs]</a> <span class="k">def</span> <span class="nf">_check_and_change_state_before_execution</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">ignore_all_deps</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_task_deps</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_ti_state</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">mark_success</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">test_mode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">job_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Checks dependencies and then sets state to RUNNING if they are met. Returns</span>
<span class="sd"> True if and only if state is set to RUNNING, which implies that task should be</span>
<span class="sd"> executed, in preparation for _run_raw_task</span>
<span class="sd"> :param verbose: whether to turn on more verbose logging</span>
<span class="sd"> :type verbose: bool</span>
<span class="sd"> :param ignore_all_deps: Ignore all of the non-critical dependencies, just runs</span>
<span class="sd"> :type ignore_all_deps: bool</span>
<span class="sd"> :param ignore_depends_on_past: Ignore depends_on_past DAG attribute</span>
<span class="sd"> :type ignore_depends_on_past: bool</span>
<span class="sd"> :param ignore_task_deps: Don&#39;t check the dependencies of this TI&#39;s task</span>
<span class="sd"> :type ignore_task_deps: bool</span>
<span class="sd"> :param ignore_ti_state: Disregards previous task instance state</span>
<span class="sd"> :type ignore_ti_state: bool</span>
<span class="sd"> :param mark_success: Don&#39;t run the task, mark its state as success</span>
<span class="sd"> :type mark_success: bool</span>
<span class="sd"> :param test_mode: Doesn&#39;t record success or failure in the DB</span>
<span class="sd"> :type test_mode: bool</span>
<span class="sd"> :param pool: specifies the pool to use to run the task instance</span>
<span class="sd"> :type pool: str</span>
<span class="sd"> :return: whether the state was changed to running or not</span>
<span class="sd"> :rtype: bool</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pool</span> <span class="o">=</span> <span class="n">pool</span> <span class="ow">or</span> <span class="n">task</span><span class="o">.</span><span class="n">pool</span>
<span class="bp">self</span><span class="o">.</span><span class="n">test_mode</span> <span class="o">=</span> <span class="n">test_mode</span>
<span class="bp">self</span><span class="o">.</span><span class="n">refresh_from_db</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span> <span class="n">lock_for_update</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="n">job_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hostname</span> <span class="o">=</span> <span class="n">get_hostname</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">operator</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">ignore_all_deps</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">ignore_ti_state</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">:</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span><span class="s1">&#39;previously_succeeded&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">queue_dep_context</span> <span class="o">=</span> <span class="n">DepContext</span><span class="p">(</span>
<span class="n">deps</span><span class="o">=</span><span class="n">QUEUE_DEPS</span><span class="p">,</span>
<span class="n">ignore_all_deps</span><span class="o">=</span><span class="n">ignore_all_deps</span><span class="p">,</span>
<span class="n">ignore_ti_state</span><span class="o">=</span><span class="n">ignore_ti_state</span><span class="p">,</span>
<span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="n">ignore_depends_on_past</span><span class="p">,</span>
<span class="n">ignore_task_deps</span><span class="o">=</span><span class="n">ignore_task_deps</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">are_dependencies_met</span><span class="p">(</span>
<span class="n">dep_context</span><span class="o">=</span><span class="n">queue_dep_context</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="c1"># TODO: Logging needs cleanup, not clear what is being printed</span>
<span class="n">hr</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="p">(</span><span class="s2">&quot;-&quot;</span> <span class="o">*</span> <span class="mi">80</span><span class="p">)</span> <span class="c1"># Line break</span>
<span class="c1"># For reporting purposes, we report based on 1-indexed,</span>
<span class="c1"># not 0-indexed lists (i.e. Attempt 1 instead of</span>
<span class="c1"># Attempt 0 for the first attempt).</span>
<span class="c1"># Set the task start date. In case it was re-scheduled use the initial</span>
<span class="c1"># start date that is recorded in task_reschedule table</span>
<span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="n">task_reschedules</span> <span class="o">=</span> <span class="n">TaskReschedule</span><span class="o">.</span><span class="n">find_for_task_instance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">)</span>
<span class="k">if</span> <span class="n">task_reschedules</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">task_reschedules</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">start_date</span>
<span class="n">dep_context</span> <span class="o">=</span> <span class="n">DepContext</span><span class="p">(</span>
<span class="n">deps</span><span class="o">=</span><span class="n">RUN_DEPS</span> <span class="o">-</span> <span class="n">QUEUE_DEPS</span><span class="p">,</span>
<span class="n">ignore_all_deps</span><span class="o">=</span><span class="n">ignore_all_deps</span><span class="p">,</span>
<span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="n">ignore_depends_on_past</span><span class="p">,</span>
<span class="n">ignore_task_deps</span><span class="o">=</span><span class="n">ignore_task_deps</span><span class="p">,</span>
<span class="n">ignore_ti_state</span><span class="o">=</span><span class="n">ignore_ti_state</span><span class="p">)</span>
<span class="n">runnable</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">are_dependencies_met</span><span class="p">(</span>
<span class="n">dep_context</span><span class="o">=</span><span class="n">dep_context</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">runnable</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">mark_success</span><span class="p">:</span>
<span class="c1"># FIXME: we might have hit concurrency limits, which means we probably</span>
<span class="c1"># have been running prematurely. This should be handled in the</span>
<span class="c1"># scheduling mechanism.</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">NONE</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">hr</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
<span class="s2">&quot;FIXME: Rescheduling due to concurrency limits reached at task runtime. Attempt </span><span class="si">%s</span><span class="s2"> of &quot;</span>
<span class="s2">&quot;</span><span class="si">%s</span><span class="s2">. State set to NONE.&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">try_number</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_tries</span> <span class="o">+</span> <span class="mi">1</span>
<span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">hr</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">queued_dttm</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Queuing into pool </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">pool</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="c1"># Another worker might have started running this task instance while</span>
<span class="c1"># the current worker process was blocked on refresh_from_db</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;Task Instance already running </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="c1"># print status message</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">hr</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Starting attempt </span><span class="si">%s</span><span class="s2"> of </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">try_number</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_tries</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">hr</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_try_number</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">test_mode</span><span class="p">:</span>
<span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">Log</span><span class="p">(</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">,</span> <span class="bp">self</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pid</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">getpid</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">test_mode</span><span class="p">:</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="c1"># Closing all pooled connections to prevent</span>
<span class="c1"># &quot;max number of connections reached&quot;</span>
<span class="n">settings</span><span class="o">.</span><span class="n">engine</span><span class="o">.</span><span class="n">dispose</span><span class="p">()</span>
<span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
<span class="k">if</span> <span class="n">mark_success</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Marking success for </span><span class="si">%s</span><span class="s2"> on </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Executing </span><span class="si">%s</span><span class="s2"> on </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">True</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance._run_raw_task"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance._run_raw_task">[docs]</a> <span class="k">def</span> <span class="nf">_run_raw_task</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">mark_success</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">test_mode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">job_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Immediately runs the task (without checking or changing db state</span>
<span class="sd"> before execution) and then sets the appropriate final state after</span>
<span class="sd"> completion and runs any post-execute callbacks. Meant to be called</span>
<span class="sd"> only after another function changes the state to running.</span>
<span class="sd"> :param mark_success: Don&#39;t run the task, mark its state as success</span>
<span class="sd"> :type mark_success: bool</span>
<span class="sd"> :param test_mode: Doesn&#39;t record success or failure in the DB</span>
<span class="sd"> :type test_mode: bool</span>
<span class="sd"> :param pool: specifies the pool to use to run the task instance</span>
<span class="sd"> :type pool: str</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pool</span> <span class="o">=</span> <span class="n">pool</span> <span class="ow">or</span> <span class="n">task</span><span class="o">.</span><span class="n">pool</span>
<span class="bp">self</span><span class="o">.</span><span class="n">test_mode</span> <span class="o">=</span> <span class="n">test_mode</span>
<span class="bp">self</span><span class="o">.</span><span class="n">refresh_from_db</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">job_id</span> <span class="o">=</span> <span class="n">job_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hostname</span> <span class="o">=</span> <span class="n">get_hostname</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">operator</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span>
<span class="n">context</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">actual_start_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">mark_success</span><span class="p">:</span>
<span class="n">context</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_template_context</span><span class="p">()</span>
<span class="n">task_copy</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">task</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">task_copy</span>
<span class="k">def</span> <span class="nf">signal_handler</span><span class="p">(</span><span class="n">signum</span><span class="p">,</span> <span class="n">frame</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Received SIGTERM. Terminating subprocesses.&quot;</span><span class="p">)</span>
<span class="n">task_copy</span><span class="o">.</span><span class="n">on_kill</span><span class="p">()</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;Task received SIGTERM signal&quot;</span><span class="p">)</span>
<span class="n">signal</span><span class="o">.</span><span class="n">signal</span><span class="p">(</span><span class="n">signal</span><span class="o">.</span><span class="n">SIGTERM</span><span class="p">,</span> <span class="n">signal_handler</span><span class="p">)</span>
<span class="c1"># Don&#39;t clear Xcom until the task is certain to execute</span>
<span class="bp">self</span><span class="o">.</span><span class="n">clear_xcom_data</span><span class="p">()</span>
<span class="n">start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">render_templates</span><span class="p">()</span>
<span class="n">task_copy</span><span class="o">.</span><span class="n">pre_execute</span><span class="p">(</span><span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">)</span>
<span class="c1"># If a timeout is specified for the task, make it fail</span>
<span class="c1"># if it goes beyond</span>
<span class="n">result</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">task_copy</span><span class="o">.</span><span class="n">execution_timeout</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">with</span> <span class="n">timeout</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span>
<span class="n">task_copy</span><span class="o">.</span><span class="n">execution_timeout</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">())):</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">task_copy</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">)</span>
<span class="k">except</span> <span class="n">AirflowTaskTimeout</span><span class="p">:</span>
<span class="n">task_copy</span><span class="o">.</span><span class="n">on_kill</span><span class="p">()</span>
<span class="k">raise</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">task_copy</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">)</span>
<span class="c1"># If the task returns a result, push an XCom containing it</span>
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">XCOM_RETURN_KEY</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
<span class="c1"># TODO remove deprecated behavior in Airflow 2.0</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">task_copy</span><span class="o">.</span><span class="n">post_execute</span><span class="p">(</span><span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">TypeError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">if</span> <span class="s1">&#39;unexpected keyword argument&#39;</span> <span class="ow">in</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">):</span>
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
<span class="s1">&#39;BaseOperator.post_execute() now takes two &#39;</span>
<span class="s1">&#39;arguments, `context` and `result`, but &quot;</span><span class="si">{}</span><span class="s1">&quot; only &#39;</span>
<span class="s1">&#39;expected one. This behavior is deprecated and &#39;</span>
<span class="s1">&#39;will be removed in a future version of &#39;</span>
<span class="s1">&#39;Airflow.&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">),</span>
<span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">)</span>
<span class="n">task_copy</span><span class="o">.</span><span class="n">post_execute</span><span class="p">(</span><span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span>
<span class="n">end_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
<span class="n">duration</span> <span class="o">=</span> <span class="n">end_time</span> <span class="o">-</span> <span class="n">start_time</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">timing</span><span class="p">(</span>
<span class="s1">&#39;dag.</span><span class="si">{dag_id}</span><span class="s1">.</span><span class="si">{task_id}</span><span class="s1">.duration&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">dag_id</span><span class="o">=</span><span class="n">task_copy</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="n">task_copy</span><span class="o">.</span><span class="n">task_id</span><span class="p">),</span>
<span class="n">duration</span><span class="p">)</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span><span class="s1">&#39;operator_successes_</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span><span class="s1">&#39;ti_successes&#39;</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">refresh_from_db</span><span class="p">(</span><span class="n">lock_for_update</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span>
<span class="k">except</span> <span class="n">AirflowSkipException</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">refresh_from_db</span><span class="p">(</span><span class="n">lock_for_update</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SKIPPED</span>
<span class="k">except</span> <span class="n">AirflowRescheduleException</span> <span class="k">as</span> <span class="n">reschedule_exception</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">refresh_from_db</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_handle_reschedule</span><span class="p">(</span><span class="n">actual_start_date</span><span class="p">,</span> <span class="n">reschedule_exception</span><span class="p">,</span> <span class="n">test_mode</span><span class="p">,</span> <span class="n">context</span><span class="p">)</span>
<span class="k">return</span>
<span class="k">except</span> <span class="n">AirflowException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">refresh_from_db</span><span class="p">()</span>
<span class="c1"># for case when task is marked as success/failed externally</span>
<span class="c1"># current behavior doesn&#39;t hit the success callback</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="ow">in</span> <span class="p">{</span><span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">}:</span>
<span class="k">return</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">handle_failure</span><span class="p">(</span><span class="n">e</span><span class="p">,</span> <span class="n">test_mode</span><span class="p">,</span> <span class="n">context</span><span class="p">)</span>
<span class="k">raise</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">Exception</span><span class="p">,</span> <span class="ne">KeyboardInterrupt</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">handle_failure</span><span class="p">(</span><span class="n">e</span><span class="p">,</span> <span class="n">test_mode</span><span class="p">,</span> <span class="n">context</span><span class="p">)</span>
<span class="k">raise</span>
<span class="c1"># Success callback</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">on_success_callback</span><span class="p">:</span>
<span class="n">task</span><span class="o">.</span><span class="n">on_success_callback</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e3</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Failed when executing success callback&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e3</span><span class="p">)</span>
<span class="c1"># Recording SUCCESS</span>
<span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">set_duration</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">test_mode</span><span class="p">:</span>
<span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">Log</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">state</span><span class="p">,</span> <span class="bp">self</span><span class="p">))</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.run"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.run">[docs]</a> <span class="k">def</span> <span class="nf">run</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">ignore_all_deps</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_task_deps</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_ti_state</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">mark_success</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">test_mode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">job_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_check_and_change_state_before_execution</span><span class="p">(</span>
<span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
<span class="n">ignore_all_deps</span><span class="o">=</span><span class="n">ignore_all_deps</span><span class="p">,</span>
<span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="n">ignore_depends_on_past</span><span class="p">,</span>
<span class="n">ignore_task_deps</span><span class="o">=</span><span class="n">ignore_task_deps</span><span class="p">,</span>
<span class="n">ignore_ti_state</span><span class="o">=</span><span class="n">ignore_ti_state</span><span class="p">,</span>
<span class="n">mark_success</span><span class="o">=</span><span class="n">mark_success</span><span class="p">,</span>
<span class="n">test_mode</span><span class="o">=</span><span class="n">test_mode</span><span class="p">,</span>
<span class="n">job_id</span><span class="o">=</span><span class="n">job_id</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="n">pool</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="k">if</span> <span class="n">res</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_run_raw_task</span><span class="p">(</span>
<span class="n">mark_success</span><span class="o">=</span><span class="n">mark_success</span><span class="p">,</span>
<span class="n">test_mode</span><span class="o">=</span><span class="n">test_mode</span><span class="p">,</span>
<span class="n">job_id</span><span class="o">=</span><span class="n">job_id</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="n">pool</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.dry_run"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.dry_run">[docs]</a> <span class="k">def</span> <span class="nf">dry_run</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span>
<span class="n">task_copy</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">task</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">task_copy</span>
<span class="bp">self</span><span class="o">.</span><span class="n">render_templates</span><span class="p">()</span>
<span class="n">task_copy</span><span class="o">.</span><span class="n">dry_run</span><span class="p">()</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance._handle_reschedule"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance._handle_reschedule">[docs]</a> <span class="k">def</span> <span class="nf">_handle_reschedule</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">actual_start_date</span><span class="p">,</span> <span class="n">reschedule_exception</span><span class="p">,</span> <span class="n">test_mode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="c1"># Don&#39;t record reschedule request in test mode</span>
<span class="k">if</span> <span class="n">test_mode</span><span class="p">:</span>
<span class="k">return</span>
<span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">set_duration</span><span class="p">()</span>
<span class="c1"># Log reschedule request</span>
<span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">TaskReschedule</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_try_number</span><span class="p">,</span>
<span class="n">actual_start_date</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span><span class="p">,</span>
<span class="n">reschedule_exception</span><span class="o">.</span><span class="n">reschedule_date</span><span class="p">))</span>
<span class="c1"># set state</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">UP_FOR_RESCHEDULE</span>
<span class="c1"># Decrement try_number so subsequent runs will use the same try number and write</span>
<span class="c1"># to same log file.</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_try_number</span> <span class="o">-=</span> <span class="mi">1</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Rescheduling task, marking task as UP_FOR_RESCHEDULE&#39;</span><span class="p">)</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.handle_failure"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.handle_failure">[docs]</a> <span class="k">def</span> <span class="nf">handle_failure</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">error</span><span class="p">,</span> <span class="n">test_mode</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
<span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span>
<span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">set_duration</span><span class="p">()</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span><span class="s1">&#39;operator_failures_</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span><span class="s1">&#39;ti_failures&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">test_mode</span><span class="p">:</span>
<span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">Log</span><span class="p">(</span><span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">,</span> <span class="bp">self</span><span class="p">))</span>
<span class="c1"># Log failure duration</span>
<span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">TaskFail</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span><span class="p">))</span>
<span class="k">if</span> <span class="n">context</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">context</span><span class="p">[</span><span class="s1">&#39;exception&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">error</span>
<span class="c1"># Let&#39;s go deeper</span>
<span class="k">try</span><span class="p">:</span>
<span class="c1"># Since this function is called only when the TI state is running,</span>
<span class="c1"># try_number contains the current try_number (not the next). We</span>
<span class="c1"># only mark task instance as FAILED if the next task instance</span>
<span class="c1"># try_number exceeds the max_tries.</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_eligible_to_retry</span><span class="p">():</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">UP_FOR_RETRY</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Marking task as UP_FOR_RETRY&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">email_on_retry</span> <span class="ow">and</span> <span class="n">task</span><span class="o">.</span><span class="n">email</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">email_alert</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span>
<span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">retries</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;All retries failed; marking task as FAILED&#39;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Marking task as FAILED.&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">email_on_failure</span> <span class="ow">and</span> <span class="n">task</span><span class="o">.</span><span class="n">email</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">email_alert</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e2</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;Failed to send email to: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">task</span><span class="o">.</span><span class="n">email</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e2</span><span class="p">)</span>
<span class="c1"># Handling callbacks pessimistically</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">UP_FOR_RETRY</span> <span class="ow">and</span> <span class="n">task</span><span class="o">.</span><span class="n">on_retry_callback</span><span class="p">:</span>
<span class="n">task</span><span class="o">.</span><span class="n">on_retry_callback</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span> <span class="ow">and</span> <span class="n">task</span><span class="o">.</span><span class="n">on_failure_callback</span><span class="p">:</span>
<span class="n">task</span><span class="o">.</span><span class="n">on_failure_callback</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e3</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Failed at executing callback&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e3</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">test_mode</span><span class="p">:</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
<div class="viewcode-block" id="TaskInstance.is_eligible_to_retry"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.is_eligible_to_retry">[docs]</a> <span class="k">def</span> <span class="nf">is_eligible_to_retry</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Is task instance is eligible for retry&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">retries</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">try_number</span> <span class="o">&lt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_tries</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.get_template_context"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.get_template_context">[docs]</a> <span class="k">def</span> <span class="nf">get_template_context</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span>
<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">macros</span>
<span class="n">tables</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="s1">&#39;tables&#39;</span> <span class="ow">in</span> <span class="n">task</span><span class="o">.</span><span class="n">params</span><span class="p">:</span>
<span class="n">tables</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="s1">&#39;tables&#39;</span><span class="p">]</span>
<span class="n">params</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">run_id</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span>
<span class="n">dag_run</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="s1">&#39;dag&#39;</span><span class="p">):</span>
<span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">params</span><span class="p">:</span>
<span class="n">params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">params</span><span class="p">)</span>
<span class="n">dag_run</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span>
<span class="o">.</span><span class="n">filter_by</span><span class="p">(</span>
<span class="n">dag_id</span><span class="o">=</span><span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span>
<span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="p">)</span>
<span class="n">run_id</span> <span class="o">=</span> <span class="n">dag_run</span><span class="o">.</span><span class="n">run_id</span> <span class="k">if</span> <span class="n">dag_run</span> <span class="k">else</span> <span class="kc">None</span>
<span class="n">session</span><span class="o">.</span><span class="n">expunge_all</span><span class="p">()</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="n">ds</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="n">ts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
<span class="n">yesterday_ds</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="n">tomorrow_ds</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">+</span> <span class="n">timedelta</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="c1"># For manually triggered dagruns that aren&#39;t run on a schedule, next/previous</span>
<span class="c1"># schedule dates don&#39;t make sense, and should be set to execution date for</span>
<span class="c1"># consistency with how execution_date is set for manually triggered tasks, i.e.</span>
<span class="c1"># triggered_date == execution_date.</span>
<span class="k">if</span> <span class="n">dag_run</span> <span class="ow">and</span> <span class="n">dag_run</span><span class="o">.</span><span class="n">external_trigger</span><span class="p">:</span>
<span class="n">prev_execution_date</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span>
<span class="n">next_execution_date</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">prev_execution_date</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">previous_schedule</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span>
<span class="n">next_execution_date</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">following_schedule</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span>
<span class="n">next_ds</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">next_ds_nodash</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">next_execution_date</span><span class="p">:</span>
<span class="n">next_ds</span> <span class="o">=</span> <span class="n">next_execution_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="n">next_ds_nodash</span> <span class="o">=</span> <span class="n">next_ds</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">prev_ds</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">prev_ds_nodash</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">prev_execution_date</span><span class="p">:</span>
<span class="n">prev_ds</span> <span class="o">=</span> <span class="n">prev_execution_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="n">prev_ds_nodash</span> <span class="o">=</span> <span class="n">prev_ds</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">ds_nodash</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">ts_nodash</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">&#39;%Y%m</span><span class="si">%d</span><span class="s1">T%H%M%S&#39;</span><span class="p">)</span>
<span class="n">ts_nodash_with_tz</span> <span class="o">=</span> <span class="n">ts</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;:&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">yesterday_ds_nodash</span> <span class="o">=</span> <span class="n">yesterday_ds</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">tomorrow_ds_nodash</span> <span class="o">=</span> <span class="n">tomorrow_ds</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">ti_key_str</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{task.dag_id}</span><span class="s2">__</span><span class="si">{task.task_id}</span><span class="s2">__</span><span class="si">{ds_nodash}</span><span class="s2">&quot;</span>
<span class="n">ti_key_str</span> <span class="o">=</span> <span class="n">ti_key_str</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
<span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">params</span><span class="p">:</span>
<span class="n">params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">params</span><span class="p">)</span>
<span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;dag_run_conf_overrides_params&#39;</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">overwrite_params_with_dag_run_conf</span><span class="p">(</span><span class="n">params</span><span class="o">=</span><span class="n">params</span><span class="p">,</span> <span class="n">dag_run</span><span class="o">=</span><span class="n">dag_run</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">VariableAccessor</span><span class="p">:</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Wrapper around Variable. This way you can get variables in templates by using</span>
<span class="sd"> {var.value.your_variable_name}.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">var</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">var</span> <span class="o">=</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">var</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">var</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">VariableJsonAccessor</span><span class="p">:</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Wrapper around deserialized Variables. This way you can get variables</span>
<span class="sd"> in templates by using {var.json.your_variable_name}.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">var</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">var</span> <span class="o">=</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="n">deserialize_json</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">var</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">var</span><span class="p">)</span>
<span class="k">return</span> <span class="p">{</span>
<span class="s1">&#39;dag&#39;</span><span class="p">:</span> <span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="p">,</span>
<span class="s1">&#39;ds&#39;</span><span class="p">:</span> <span class="n">ds</span><span class="p">,</span>
<span class="s1">&#39;next_ds&#39;</span><span class="p">:</span> <span class="n">next_ds</span><span class="p">,</span>
<span class="s1">&#39;next_ds_nodash&#39;</span><span class="p">:</span> <span class="n">next_ds_nodash</span><span class="p">,</span>
<span class="s1">&#39;prev_ds&#39;</span><span class="p">:</span> <span class="n">prev_ds</span><span class="p">,</span>
<span class="s1">&#39;prev_ds_nodash&#39;</span><span class="p">:</span> <span class="n">prev_ds_nodash</span><span class="p">,</span>
<span class="s1">&#39;ds_nodash&#39;</span><span class="p">:</span> <span class="n">ds_nodash</span><span class="p">,</span>
<span class="s1">&#39;ts&#39;</span><span class="p">:</span> <span class="n">ts</span><span class="p">,</span>
<span class="s1">&#39;ts_nodash&#39;</span><span class="p">:</span> <span class="n">ts_nodash</span><span class="p">,</span>
<span class="s1">&#39;ts_nodash_with_tz&#39;</span><span class="p">:</span> <span class="n">ts_nodash_with_tz</span><span class="p">,</span>
<span class="s1">&#39;yesterday_ds&#39;</span><span class="p">:</span> <span class="n">yesterday_ds</span><span class="p">,</span>
<span class="s1">&#39;yesterday_ds_nodash&#39;</span><span class="p">:</span> <span class="n">yesterday_ds_nodash</span><span class="p">,</span>
<span class="s1">&#39;tomorrow_ds&#39;</span><span class="p">:</span> <span class="n">tomorrow_ds</span><span class="p">,</span>
<span class="s1">&#39;tomorrow_ds_nodash&#39;</span><span class="p">:</span> <span class="n">tomorrow_ds_nodash</span><span class="p">,</span>
<span class="s1">&#39;END_DATE&#39;</span><span class="p">:</span> <span class="n">ds</span><span class="p">,</span>
<span class="s1">&#39;end_date&#39;</span><span class="p">:</span> <span class="n">ds</span><span class="p">,</span>
<span class="s1">&#39;dag_run&#39;</span><span class="p">:</span> <span class="n">dag_run</span><span class="p">,</span>
<span class="s1">&#39;run_id&#39;</span><span class="p">:</span> <span class="n">run_id</span><span class="p">,</span>
<span class="s1">&#39;execution_date&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span>
<span class="s1">&#39;prev_execution_date&#39;</span><span class="p">:</span> <span class="n">prev_execution_date</span><span class="p">,</span>
<span class="s1">&#39;next_execution_date&#39;</span><span class="p">:</span> <span class="n">next_execution_date</span><span class="p">,</span>
<span class="s1">&#39;latest_date&#39;</span><span class="p">:</span> <span class="n">ds</span><span class="p">,</span>
<span class="s1">&#39;macros&#39;</span><span class="p">:</span> <span class="n">macros</span><span class="p">,</span>
<span class="s1">&#39;params&#39;</span><span class="p">:</span> <span class="n">params</span><span class="p">,</span>
<span class="s1">&#39;tables&#39;</span><span class="p">:</span> <span class="n">tables</span><span class="p">,</span>
<span class="s1">&#39;task&#39;</span><span class="p">:</span> <span class="n">task</span><span class="p">,</span>
<span class="s1">&#39;task_instance&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="p">,</span>
<span class="s1">&#39;ti&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="p">,</span>
<span class="s1">&#39;task_instance_key_str&#39;</span><span class="p">:</span> <span class="n">ti_key_str</span><span class="p">,</span>
<span class="s1">&#39;conf&#39;</span><span class="p">:</span> <span class="n">configuration</span><span class="p">,</span>
<span class="s1">&#39;test_mode&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_mode</span><span class="p">,</span>
<span class="s1">&#39;var&#39;</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">&#39;value&#39;</span><span class="p">:</span> <span class="n">VariableAccessor</span><span class="p">(),</span>
<span class="s1">&#39;json&#39;</span><span class="p">:</span> <span class="n">VariableJsonAccessor</span><span class="p">()</span>
<span class="p">},</span>
<span class="s1">&#39;inlets&#39;</span><span class="p">:</span> <span class="n">task</span><span class="o">.</span><span class="n">inlets</span><span class="p">,</span>
<span class="s1">&#39;outlets&#39;</span><span class="p">:</span> <span class="n">task</span><span class="o">.</span><span class="n">outlets</span><span class="p">,</span></div>
<span class="p">}</span>
<div class="viewcode-block" id="TaskInstance.overwrite_params_with_dag_run_conf"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.overwrite_params_with_dag_run_conf">[docs]</a> <span class="k">def</span> <span class="nf">overwrite_params_with_dag_run_conf</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">params</span><span class="p">,</span> <span class="n">dag_run</span><span class="p">):</span>
<span class="k">if</span> <span class="n">dag_run</span> <span class="ow">and</span> <span class="n">dag_run</span><span class="o">.</span><span class="n">conf</span><span class="p">:</span>
<span class="n">params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">dag_run</span><span class="o">.</span><span class="n">conf</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.render_templates"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.render_templates">[docs]</a> <span class="k">def</span> <span class="nf">render_templates</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span>
<span class="n">jinja_context</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_template_context</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;task&#39;</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="p">,</span> <span class="s1">&#39;dag&#39;</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">user_defined_macros</span><span class="p">:</span>
<span class="n">jinja_context</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">user_defined_macros</span><span class="p">)</span>
<span class="n">rt</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">render_template</span> <span class="c1"># shortcut to method</span>
<span class="k">for</span> <span class="n">attr</span> <span class="ow">in</span> <span class="n">task</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="n">template_fields</span><span class="p">:</span>
<span class="n">content</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="n">attr</span><span class="p">)</span>
<span class="k">if</span> <span class="n">content</span><span class="p">:</span>
<span class="n">rendered_content</span> <span class="o">=</span> <span class="n">rt</span><span class="p">(</span><span class="n">attr</span><span class="p">,</span> <span class="n">content</span><span class="p">,</span> <span class="n">jinja_context</span><span class="p">)</span>
<span class="nb">setattr</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="n">attr</span><span class="p">,</span> <span class="n">rendered_content</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.email_alert"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.email_alert">[docs]</a> <span class="k">def</span> <span class="nf">email_alert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">exception</span><span class="p">):</span>
<span class="n">exception_html</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">exception</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">,</span> <span class="s1">&#39;&lt;br&gt;&#39;</span><span class="p">)</span>
<span class="n">jinja_context</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_template_context</span><span class="p">()</span>
<span class="c1"># This function is called after changing the state</span>
<span class="c1"># from State.RUNNING so need to subtract 1 from self.try_number.</span>
<span class="n">jinja_context</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span>
<span class="n">exception</span><span class="o">=</span><span class="n">exception</span><span class="p">,</span>
<span class="n">exception_html</span><span class="o">=</span><span class="n">exception_html</span><span class="p">,</span>
<span class="n">try_number</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">try_number</span> <span class="o">-</span> <span class="mi">1</span><span class="p">,</span>
<span class="n">max_tries</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_tries</span><span class="p">))</span>
<span class="n">jinja_env</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">get_template_env</span><span class="p">()</span>
<span class="n">default_subject</span> <span class="o">=</span> <span class="s1">&#39;Airflow alert: {{ti}}&#39;</span>
<span class="c1"># For reporting purposes, we report based on 1-indexed,</span>
<span class="c1"># not 0-indexed lists (i.e. Try 1 instead of</span>
<span class="c1"># Try 0 for the first attempt).</span>
<span class="n">default_html_content</span> <span class="o">=</span> <span class="p">(</span>
<span class="s1">&#39;Try {{try_number}} out of {{max_tries + 1}}&lt;br&gt;&#39;</span>
<span class="s1">&#39;Exception:&lt;br&gt;{{exception_html}}&lt;br&gt;&#39;</span>
<span class="s1">&#39;Log: &lt;a href=&quot;{{ti.log_url}}&quot;&gt;Link&lt;/a&gt;&lt;br&gt;&#39;</span>
<span class="s1">&#39;Host: {{ti.hostname}}&lt;br&gt;&#39;</span>
<span class="s1">&#39;Log file: {{ti.log_filepath}}&lt;br&gt;&#39;</span>
<span class="s1">&#39;Mark success: &lt;a href=&quot;{{ti.mark_success_url}}&quot;&gt;Link&lt;/a&gt;&lt;br&gt;&#39;</span>
<span class="p">)</span>
<span class="k">def</span> <span class="nf">render</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">content</span><span class="p">):</span>
<span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">has_option</span><span class="p">(</span><span class="s1">&#39;email&#39;</span><span class="p">,</span> <span class="n">key</span><span class="p">):</span>
<span class="n">path</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;email&#39;</span><span class="p">,</span> <span class="n">key</span><span class="p">)</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="n">content</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="k">return</span> <span class="n">jinja_env</span><span class="o">.</span><span class="n">from_string</span><span class="p">(</span><span class="n">content</span><span class="p">)</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="o">**</span><span class="n">jinja_context</span><span class="p">)</span>
<span class="n">subject</span> <span class="o">=</span> <span class="n">render</span><span class="p">(</span><span class="s1">&#39;subject_template&#39;</span><span class="p">,</span> <span class="n">default_subject</span><span class="p">)</span>
<span class="n">html_content</span> <span class="o">=</span> <span class="n">render</span><span class="p">(</span><span class="s1">&#39;html_content_template&#39;</span><span class="p">,</span> <span class="n">default_html_content</span><span class="p">)</span>
<span class="n">send_email</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">email</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">html_content</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.set_duration"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.set_duration">[docs]</a> <span class="k">def</span> <span class="nf">set_duration</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">duration</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">duration</span> <span class="o">=</span> <span class="kc">None</span></div>
<div class="viewcode-block" id="TaskInstance.xcom_push"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.xcom_push">[docs]</a> <span class="k">def</span> <span class="nf">xcom_push</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">key</span><span class="p">,</span>
<span class="n">value</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Make an XCom available for tasks to pull.</span>
<span class="sd"> :param key: A key for the XCom</span>
<span class="sd"> :type key: str</span>
<span class="sd"> :param value: A value for the XCom. The value is pickled and stored</span>
<span class="sd"> in the database.</span>
<span class="sd"> :type value: any pickleable object</span>
<span class="sd"> :param execution_date: if provided, the XCom will not be visible until</span>
<span class="sd"> this date. This can be used, for example, to send a message to a</span>
<span class="sd"> task on a future date without it being immediately visible.</span>
<span class="sd"> :type execution_date: datetime</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">execution_date</span> <span class="ow">and</span> <span class="n">execution_date</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
<span class="s1">&#39;execution_date can not be in the past (current &#39;</span>
<span class="s1">&#39;execution_date is </span><span class="si">{}</span><span class="s1">; received </span><span class="si">{}</span><span class="s1">)&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">))</span>
<span class="n">XCom</span><span class="o">.</span><span class="n">set</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span>
<span class="n">value</span><span class="o">=</span><span class="n">value</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
<span class="n">dag_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="n">execution_date</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span></div>
<div class="viewcode-block" id="TaskInstance.xcom_pull"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.xcom_pull">[docs]</a> <span class="k">def</span> <span class="nf">xcom_pull</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">task_ids</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">dag_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">key</span><span class="o">=</span><span class="n">XCOM_RETURN_KEY</span><span class="p">,</span>
<span class="n">include_prior_dates</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Pull XComs that optionally meet certain criteria.</span>
<span class="sd"> The default value for `key` limits the search to XComs</span>
<span class="sd"> that were returned by other tasks (as opposed to those that were pushed</span>
<span class="sd"> manually). To remove this filter, pass key=None (or any desired value).</span>
<span class="sd"> If a single task_id string is provided, the result is the value of the</span>
<span class="sd"> most recent matching XCom from that task_id. If multiple task_ids are</span>
<span class="sd"> provided, a tuple of matching values is returned. None is returned</span>
<span class="sd"> whenever no matches are found.</span>
<span class="sd"> :param key: A key for the XCom. If provided, only XComs with matching</span>
<span class="sd"> keys will be returned. The default key is &#39;return_value&#39;, also</span>
<span class="sd"> available as a constant XCOM_RETURN_KEY. This key is automatically</span>
<span class="sd"> given to XComs returned by tasks (as opposed to being pushed</span>
<span class="sd"> manually). To remove the filter, pass key=None.</span>
<span class="sd"> :type key: str</span>
<span class="sd"> :param task_ids: Only XComs from tasks with matching ids will be</span>
<span class="sd"> pulled. Can pass None to remove the filter.</span>
<span class="sd"> :type task_ids: str or iterable of strings (representing task_ids)</span>
<span class="sd"> :param dag_id: If provided, only pulls XComs from this DAG.</span>
<span class="sd"> If None (default), the DAG of the calling task is used.</span>
<span class="sd"> :type dag_id: str</span>
<span class="sd"> :param include_prior_dates: If False, only XComs from the current</span>
<span class="sd"> execution_date are returned. If True, XComs from previous dates</span>
<span class="sd"> are returned as well.</span>
<span class="sd"> :type include_prior_dates: bool</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">dag_id</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">dag_id</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span>
<span class="n">pull_fn</span> <span class="o">=</span> <span class="n">functools</span><span class="o">.</span><span class="n">partial</span><span class="p">(</span>
<span class="n">XCom</span><span class="o">.</span><span class="n">get_one</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span>
<span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span>
<span class="n">dag_id</span><span class="o">=</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">include_prior_dates</span><span class="o">=</span><span class="n">include_prior_dates</span><span class="p">)</span>
<span class="k">if</span> <span class="n">is_container</span><span class="p">(</span><span class="n">task_ids</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">pull_fn</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="n">t</span><span class="p">)</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">task_ids</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">pull_fn</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="n">task_ids</span><span class="p">)</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="TaskInstance.get_num_running_task_instances"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.get_num_running_task_instances">[docs]</a> <span class="k">def</span> <span class="nf">get_num_running_task_instances</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">):</span>
<span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
<span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span></div>
<span class="p">)</span><span class="o">.</span><span class="n">count</span><span class="p">()</span>
<div class="viewcode-block" id="TaskInstance.init_run_context"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.TaskInstance.init_run_context">[docs]</a> <span class="k">def</span> <span class="nf">init_run_context</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">raw</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Sets the log context.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">raw</span> <span class="o">=</span> <span class="n">raw</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_set_context</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="BaseOperator"><a class="viewcode-back" href="../../_api/airflow/operators/index.html#airflow.models.BaseOperator">[docs]</a><span class="nd">@functools</span><span class="o">.</span><span class="n">total_ordering</span>
<span class="k">class</span> <span class="nc">BaseOperator</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Abstract base class for all operators. Since operators create objects that</span>
<span class="sd"> become nodes in the dag, BaseOperator contains many recursive methods for</span>
<span class="sd"> dag crawling behavior. To derive this class, you are expected to override</span>
<span class="sd"> the constructor as well as the &#39;execute&#39; method.</span>
<span class="sd"> Operators derived from this class should perform or trigger certain tasks</span>
<span class="sd"> synchronously (wait for completion). Example of operators could be an</span>
<span class="sd"> operator that runs a Pig job (PigOperator), a sensor operator that</span>
<span class="sd"> waits for a partition to land in Hive (HiveSensorOperator), or one that</span>
<span class="sd"> moves data from Hive to MySQL (Hive2MySqlOperator). Instances of these</span>
<span class="sd"> operators (tasks) target specific operations, running specific scripts,</span>
<span class="sd"> functions or data transfers.</span>
<span class="sd"> This class is abstract and shouldn&#39;t be instantiated. Instantiating a</span>
<span class="sd"> class derived from this one results in the creation of a task object,</span>
<span class="sd"> which ultimately becomes a node in DAG objects. Task dependencies should</span>
<span class="sd"> be set by using the set_upstream and/or set_downstream methods.</span>
<span class="sd"> :param task_id: a unique, meaningful id for the task</span>
<span class="sd"> :type task_id: str</span>
<span class="sd"> :param owner: the owner of the task, using the unix username is recommended</span>
<span class="sd"> :type owner: str</span>
<span class="sd"> :param retries: the number of retries that should be performed before</span>
<span class="sd"> failing the task</span>
<span class="sd"> :type retries: int</span>
<span class="sd"> :param retry_delay: delay between retries</span>
<span class="sd"> :type retry_delay: datetime.timedelta</span>
<span class="sd"> :param retry_exponential_backoff: allow progressive longer waits between</span>
<span class="sd"> retries by using exponential backoff algorithm on retry delay (delay</span>
<span class="sd"> will be converted into seconds)</span>
<span class="sd"> :type retry_exponential_backoff: bool</span>
<span class="sd"> :param max_retry_delay: maximum delay interval between retries</span>
<span class="sd"> :type max_retry_delay: datetime.timedelta</span>
<span class="sd"> :param start_date: The ``start_date`` for the task, determines</span>
<span class="sd"> the ``execution_date`` for the first task instance. The best practice</span>
<span class="sd"> is to have the start_date rounded</span>
<span class="sd"> to your DAG&#39;s ``schedule_interval``. Daily jobs have their start_date</span>
<span class="sd"> some day at 00:00:00, hourly jobs have their start_date at 00:00</span>
<span class="sd"> of a specific hour. Note that Airflow simply looks at the latest</span>
<span class="sd"> ``execution_date`` and adds the ``schedule_interval`` to determine</span>
<span class="sd"> the next ``execution_date``. It is also very important</span>
<span class="sd"> to note that different tasks&#39; dependencies</span>
<span class="sd"> need to line up in time. If task A depends on task B and their</span>
<span class="sd"> start_date are offset in a way that their execution_date don&#39;t line</span>
<span class="sd"> up, A&#39;s dependencies will never be met. If you are looking to delay</span>
<span class="sd"> a task, for example running a daily task at 2AM, look into the</span>
<span class="sd"> ``TimeSensor`` and ``TimeDeltaSensor``. We advise against using</span>
<span class="sd"> dynamic ``start_date`` and recommend using fixed ones. Read the</span>
<span class="sd"> FAQ entry about start_date for more information.</span>
<span class="sd"> :type start_date: datetime.datetime</span>
<span class="sd"> :param end_date: if specified, the scheduler won&#39;t go beyond this date</span>
<span class="sd"> :type end_date: datetime.datetime</span>
<span class="sd"> :param depends_on_past: when set to true, task instances will run</span>
<span class="sd"> sequentially while relying on the previous task&#39;s schedule to</span>
<span class="sd"> succeed. The task instance for the start_date is allowed to run.</span>
<span class="sd"> :type depends_on_past: bool</span>
<span class="sd"> :param wait_for_downstream: when set to true, an instance of task</span>
<span class="sd"> X will wait for tasks immediately downstream of the previous instance</span>
<span class="sd"> of task X to finish successfully before it runs. This is useful if the</span>
<span class="sd"> different instances of a task X alter the same asset, and this asset</span>
<span class="sd"> is used by tasks downstream of task X. Note that depends_on_past</span>
<span class="sd"> is forced to True wherever wait_for_downstream is used.</span>
<span class="sd"> :type wait_for_downstream: bool</span>
<span class="sd"> :param queue: which queue to target when running this job. Not</span>
<span class="sd"> all executors implement queue management, the CeleryExecutor</span>
<span class="sd"> does support targeting specific queues.</span>
<span class="sd"> :type queue: str</span>
<span class="sd"> :param dag: a reference to the dag the task is attached to (if any)</span>
<span class="sd"> :type dag: airflow.models.DAG</span>
<span class="sd"> :param priority_weight: priority weight of this task against other task.</span>
<span class="sd"> This allows the executor to trigger higher priority tasks before</span>
<span class="sd"> others when things get backed up.</span>
<span class="sd"> :type priority_weight: int</span>
<span class="sd"> :param weight_rule: weighting method used for the effective total</span>
<span class="sd"> priority weight of the task. Options are:</span>
<span class="sd"> ``{ downstream | upstream | absolute }`` default is ``downstream``</span>
<span class="sd"> When set to ``downstream`` the effective weight of the task is the</span>
<span class="sd"> aggregate sum of all downstream descendants. As a result, upstream</span>
<span class="sd"> tasks will have higher weight and will be scheduled more aggressively</span>
<span class="sd"> when using positive weight values. This is useful when you have</span>
<span class="sd"> multiple dag run instances and desire to have all upstream tasks to</span>
<span class="sd"> complete for all runs before each dag can continue processing</span>
<span class="sd"> downstream tasks. When set to ``upstream`` the effective weight is the</span>
<span class="sd"> aggregate sum of all upstream ancestors. This is the opposite where</span>
<span class="sd"> downtream tasks have higher weight and will be scheduled more</span>
<span class="sd"> aggressively when using positive weight values. This is useful when you</span>
<span class="sd"> have multiple dag run instances and prefer to have each dag complete</span>
<span class="sd"> before starting upstream tasks of other dags. When set to</span>
<span class="sd"> ``absolute``, the effective weight is the exact ``priority_weight``</span>
<span class="sd"> specified without additional weighting. You may want to do this when</span>
<span class="sd"> you know exactly what priority weight each task should have.</span>
<span class="sd"> Additionally, when set to ``absolute``, there is bonus effect of</span>
<span class="sd"> significantly speeding up the task creation process as for very large</span>
<span class="sd"> DAGS. Options can be set as string or using the constants defined in</span>
<span class="sd"> the static class ``airflow.utils.WeightRule``</span>
<span class="sd"> :type weight_rule: str</span>
<span class="sd"> :param pool: the slot pool this task should run in, slot pools are a</span>
<span class="sd"> way to limit concurrency for certain tasks</span>
<span class="sd"> :type pool: str</span>
<span class="sd"> :param sla: time by which the job is expected to succeed. Note that</span>
<span class="sd"> this represents the ``timedelta`` after the period is closed. For</span>
<span class="sd"> example if you set an SLA of 1 hour, the scheduler would send an email</span>
<span class="sd"> soon after 1:00AM on the ``2016-01-02`` if the ``2016-01-01`` instance</span>
<span class="sd"> has not succeeded yet.</span>
<span class="sd"> The scheduler pays special attention for jobs with an SLA and</span>
<span class="sd"> sends alert</span>
<span class="sd"> emails for sla misses. SLA misses are also recorded in the database</span>
<span class="sd"> for future reference. All tasks that share the same SLA time</span>
<span class="sd"> get bundled in a single email, sent soon after that time. SLA</span>
<span class="sd"> notification are sent once and only once for each task instance.</span>
<span class="sd"> :type sla: datetime.timedelta</span>
<span class="sd"> :param execution_timeout: max time allowed for the execution of</span>
<span class="sd"> this task instance, if it goes beyond it will raise and fail.</span>
<span class="sd"> :type execution_timeout: datetime.timedelta</span>
<span class="sd"> :param on_failure_callback: a function to be called when a task instance</span>
<span class="sd"> of this task fails. a context dictionary is passed as a single</span>
<span class="sd"> parameter to this function. Context contains references to related</span>
<span class="sd"> objects to the task instance and is documented under the macros</span>
<span class="sd"> section of the API.</span>
<span class="sd"> :type on_failure_callback: callable</span>
<span class="sd"> :param on_retry_callback: much like the ``on_failure_callback`` except</span>
<span class="sd"> that it is executed when retries occur.</span>
<span class="sd"> :type on_retry_callback: callable</span>
<span class="sd"> :param on_success_callback: much like the ``on_failure_callback`` except</span>
<span class="sd"> that it is executed when the task succeeds.</span>
<span class="sd"> :type on_success_callback: callable</span>
<span class="sd"> :param trigger_rule: defines the rule by which dependencies are applied</span>
<span class="sd"> for the task to get triggered. Options are:</span>
<span class="sd"> ``{ all_success | all_failed | all_done | one_success |</span>
<span class="sd"> one_failed | none_failed | none_skipped | dummy}``</span>
<span class="sd"> default is ``all_success``. Options can be set as string or</span>
<span class="sd"> using the constants defined in the static class</span>
<span class="sd"> ``airflow.utils.TriggerRule``</span>
<span class="sd"> :type trigger_rule: str</span>
<span class="sd"> :param resources: A map of resource parameter names (the argument names of the</span>
<span class="sd"> Resources constructor) to their values.</span>
<span class="sd"> :type resources: dict</span>
<span class="sd"> :param run_as_user: unix username to impersonate while running the task</span>
<span class="sd"> :type run_as_user: str</span>
<span class="sd"> :param task_concurrency: When set, a task will be able to limit the concurrent</span>
<span class="sd"> runs across execution_dates</span>
<span class="sd"> :type task_concurrency: int</span>
<span class="sd"> :param executor_config: Additional task-level configuration parameters that are</span>
<span class="sd"> interpreted by a specific executor. Parameters are namespaced by the name of</span>
<span class="sd"> executor.</span>
<span class="sd"> **Example**: to run this task in a specific docker container through</span>
<span class="sd"> the KubernetesExecutor ::</span>
<span class="sd"> MyOperator(...,</span>
<span class="sd"> executor_config={</span>
<span class="sd"> &quot;KubernetesExecutor&quot;:</span>
<span class="sd"> {&quot;image&quot;: &quot;myCustomDockerImage&quot;}</span>
<span class="sd"> }</span>
<span class="sd"> )</span>
<span class="sd"> :type executor_config: dict</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># For derived classes to define which fields will get jinjaified</span>
<div class="viewcode-block" id="BaseOperator.template_fields"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.template_fields">[docs]</a> <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[]</span> <span class="c1"># type: Iterable[str]</span></div>
<span class="c1"># Defines which files extensions to look for in the templated fields</span>
<div class="viewcode-block" id="BaseOperator.template_ext"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.template_ext">[docs]</a> <span class="n">template_ext</span> <span class="o">=</span> <span class="p">[]</span> <span class="c1"># type: Iterable[str]</span></div>
<span class="c1"># Defines the color in the UI</span>
<div class="viewcode-block" id="BaseOperator.ui_color"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.ui_color">[docs]</a> <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">&#39;#fff&#39;</span></div>
<div class="viewcode-block" id="BaseOperator.ui_fgcolor"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.ui_fgcolor">[docs]</a> <span class="n">ui_fgcolor</span> <span class="o">=</span> <span class="s1">&#39;#000&#39;</span></div>
<span class="c1"># base list which includes all the attrs that don&#39;t need deep copy.</span>
<div class="viewcode-block" id="BaseOperator._base_operator_shallow_copy_attrs"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator._base_operator_shallow_copy_attrs">[docs]</a> <span class="n">_base_operator_shallow_copy_attrs</span> <span class="o">=</span> <span class="p">(</span><span class="s1">&#39;user_defined_macros&#39;</span><span class="p">,</span>
<span class="s1">&#39;user_defined_filters&#39;</span><span class="p">,</span>
<span class="s1">&#39;params&#39;</span><span class="p">,</span>
<span class="s1">&#39;_log&#39;</span><span class="p">,)</span></div>
<span class="c1"># each operator should override this class attr for shallow copy attrs.</span>
<div class="viewcode-block" id="BaseOperator.shallow_copy_attrs"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.shallow_copy_attrs">[docs]</a> <span class="n">shallow_copy_attrs</span> <span class="o">=</span> <span class="p">()</span> <span class="c1"># type: Iterable[str]</span></div>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">task_id</span><span class="p">,</span>
<span class="n">owner</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;operators&#39;</span><span class="p">,</span> <span class="s1">&#39;DEFAULT_OWNER&#39;</span><span class="p">),</span>
<span class="n">email</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">email_on_retry</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">email_on_failure</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">retries</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
<span class="n">retry_delay</span><span class="o">=</span><span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="mi">300</span><span class="p">),</span>
<span class="n">retry_exponential_backoff</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">max_retry_delay</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">schedule_interval</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="c1"># not hooked as of now</span>
<span class="n">depends_on_past</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">wait_for_downstream</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">dag</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">default_args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">priority_weight</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
<span class="n">weight_rule</span><span class="o">=</span><span class="n">WeightRule</span><span class="o">.</span><span class="n">DOWNSTREAM</span><span class="p">,</span>
<span class="n">queue</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;celery&#39;</span><span class="p">,</span> <span class="s1">&#39;default_queue&#39;</span><span class="p">),</span>
<span class="n">pool</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">sla</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">execution_timeout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">on_failure_callback</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">on_success_callback</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">on_retry_callback</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">trigger_rule</span><span class="o">=</span><span class="n">TriggerRule</span><span class="o">.</span><span class="n">ALL_SUCCESS</span><span class="p">,</span>
<span class="n">resources</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">run_as_user</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">task_concurrency</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">executor_config</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">inlets</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">outlets</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
<span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="k">if</span> <span class="n">args</span> <span class="ow">or</span> <span class="n">kwargs</span><span class="p">:</span>
<span class="c1"># TODO remove *args and **kwargs in Airflow 2.0</span>
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
<span class="s1">&#39;Invalid arguments were passed to </span><span class="si">{c}</span><span class="s1"> (task_id: </span><span class="si">{t}</span><span class="s1">). &#39;</span>
<span class="s1">&#39;Support for passing such arguments will be dropped in &#39;</span>
<span class="s1">&#39;Airflow 2.0. Invalid arguments were:&#39;</span>
<span class="s1">&#39;</span><span class="se">\n</span><span class="s1">*args: </span><span class="si">{a}</span><span class="se">\n</span><span class="s1">**kwargs: </span><span class="si">{k}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">c</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span> <span class="n">a</span><span class="o">=</span><span class="n">args</span><span class="p">,</span> <span class="n">k</span><span class="o">=</span><span class="n">kwargs</span><span class="p">,</span> <span class="n">t</span><span class="o">=</span><span class="n">task_id</span><span class="p">),</span>
<span class="n">category</span><span class="o">=</span><span class="ne">PendingDeprecationWarning</span><span class="p">,</span>
<span class="n">stacklevel</span><span class="o">=</span><span class="mi">3</span>
<span class="p">)</span>
<span class="n">validate_key</span><span class="p">(</span><span class="n">task_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_id</span> <span class="o">=</span> <span class="n">task_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">owner</span> <span class="o">=</span> <span class="n">owner</span>
<span class="bp">self</span><span class="o">.</span><span class="n">email</span> <span class="o">=</span> <span class="n">email</span>
<span class="bp">self</span><span class="o">.</span><span class="n">email_on_retry</span> <span class="o">=</span> <span class="n">email_on_retry</span>
<span class="bp">self</span><span class="o">.</span><span class="n">email_on_failure</span> <span class="o">=</span> <span class="n">email_on_failure</span>
<span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">start_date</span>
<span class="k">if</span> <span class="n">start_date</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">start_date</span><span class="p">,</span> <span class="n">datetime</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;start_date for </span><span class="si">%s</span><span class="s2"> isn&#39;t datetime.datetime&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">start_date</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="n">start_date</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">end_date</span>
<span class="k">if</span> <span class="n">end_date</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="n">end_date</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">TriggerRule</span><span class="o">.</span><span class="n">is_valid</span><span class="p">(</span><span class="n">trigger_rule</span><span class="p">):</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
<span class="s2">&quot;The trigger_rule must be one of </span><span class="si">{all_triggers}</span><span class="s2">,&quot;</span>
<span class="s2">&quot;&#39;</span><span class="si">{d}</span><span class="s2">.</span><span class="si">{t}</span><span class="s2">&#39;; received &#39;</span><span class="si">{tr}</span><span class="s2">&#39;.&quot;</span>
<span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">all_triggers</span><span class="o">=</span><span class="n">TriggerRule</span><span class="o">.</span><span class="n">all_triggers</span><span class="p">(),</span>
<span class="n">d</span><span class="o">=</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">if</span> <span class="n">dag</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="n">t</span><span class="o">=</span><span class="n">task_id</span><span class="p">,</span> <span class="n">tr</span><span class="o">=</span><span class="n">trigger_rule</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">trigger_rule</span> <span class="o">=</span> <span class="n">trigger_rule</span>
<span class="bp">self</span><span class="o">.</span><span class="n">depends_on_past</span> <span class="o">=</span> <span class="n">depends_on_past</span>
<span class="bp">self</span><span class="o">.</span><span class="n">wait_for_downstream</span> <span class="o">=</span> <span class="n">wait_for_downstream</span>
<span class="k">if</span> <span class="n">wait_for_downstream</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">depends_on_past</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">if</span> <span class="n">schedule_interval</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
<span class="s2">&quot;schedule_interval is used for </span><span class="si">%s</span><span class="s2">, though it has &quot;</span>
<span class="s2">&quot;been deprecated as a task parameter, you need to &quot;</span>
<span class="s2">&quot;specify it as a DAG parameter instead&quot;</span><span class="p">,</span>
<span class="bp">self</span>
<span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span> <span class="o">=</span> <span class="n">schedule_interval</span>
<span class="bp">self</span><span class="o">.</span><span class="n">retries</span> <span class="o">=</span> <span class="n">retries</span>
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="n">queue</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pool</span> <span class="o">=</span> <span class="n">pool</span>
<span class="bp">self</span><span class="o">.</span><span class="n">sla</span> <span class="o">=</span> <span class="n">sla</span>
<span class="bp">self</span><span class="o">.</span><span class="n">execution_timeout</span> <span class="o">=</span> <span class="n">execution_timeout</span>
<span class="bp">self</span><span class="o">.</span><span class="n">on_failure_callback</span> <span class="o">=</span> <span class="n">on_failure_callback</span>
<span class="bp">self</span><span class="o">.</span><span class="n">on_success_callback</span> <span class="o">=</span> <span class="n">on_success_callback</span>
<span class="bp">self</span><span class="o">.</span><span class="n">on_retry_callback</span> <span class="o">=</span> <span class="n">on_retry_callback</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">retry_delay</span><span class="p">,</span> <span class="n">timedelta</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">retry_delay</span> <span class="o">=</span> <span class="n">retry_delay</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Retry_delay isn&#39;t timedelta object, assuming secs&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">retry_delay</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="n">retry_delay</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">retry_exponential_backoff</span> <span class="o">=</span> <span class="n">retry_exponential_backoff</span>
<span class="bp">self</span><span class="o">.</span><span class="n">max_retry_delay</span> <span class="o">=</span> <span class="n">max_retry_delay</span>
<span class="bp">self</span><span class="o">.</span><span class="n">params</span> <span class="o">=</span> <span class="n">params</span> <span class="ow">or</span> <span class="p">{}</span> <span class="c1"># Available in templates!</span>
<span class="bp">self</span><span class="o">.</span><span class="n">priority_weight</span> <span class="o">=</span> <span class="n">priority_weight</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">WeightRule</span><span class="o">.</span><span class="n">is_valid</span><span class="p">(</span><span class="n">weight_rule</span><span class="p">):</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
<span class="s2">&quot;The weight_rule must be one of </span><span class="si">{all_weight_rules}</span><span class="s2">,&quot;</span>
<span class="s2">&quot;&#39;</span><span class="si">{d}</span><span class="s2">.</span><span class="si">{t}</span><span class="s2">&#39;; received &#39;</span><span class="si">{tr}</span><span class="s2">&#39;.&quot;</span>
<span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">all_weight_rules</span><span class="o">=</span><span class="n">WeightRule</span><span class="o">.</span><span class="n">all_weight_rules</span><span class="p">,</span>
<span class="n">d</span><span class="o">=</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">if</span> <span class="n">dag</span> <span class="k">else</span> <span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="n">t</span><span class="o">=</span><span class="n">task_id</span><span class="p">,</span> <span class="n">tr</span><span class="o">=</span><span class="n">weight_rule</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">weight_rule</span> <span class="o">=</span> <span class="n">weight_rule</span>
<span class="bp">self</span><span class="o">.</span><span class="n">resources</span> <span class="o">=</span> <span class="n">Resources</span><span class="p">(</span><span class="o">**</span><span class="p">(</span><span class="n">resources</span> <span class="ow">or</span> <span class="p">{}))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">run_as_user</span> <span class="o">=</span> <span class="n">run_as_user</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_concurrency</span> <span class="o">=</span> <span class="n">task_concurrency</span>
<span class="bp">self</span><span class="o">.</span><span class="n">executor_config</span> <span class="o">=</span> <span class="n">executor_config</span> <span class="ow">or</span> <span class="p">{}</span>
<span class="c1"># Private attributes</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_upstream_task_ids</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_downstream_task_ids</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">dag</span> <span class="ow">and</span> <span class="n">_CONTEXT_MANAGER_DAG</span><span class="p">:</span>
<span class="n">dag</span> <span class="o">=</span> <span class="n">_CONTEXT_MANAGER_DAG</span>
<span class="k">if</span> <span class="n">dag</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dag</span> <span class="o">=</span> <span class="n">dag</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;airflow.task.operators&quot;</span><span class="p">)</span>
<span class="c1"># lineage</span>
<span class="bp">self</span><span class="o">.</span><span class="n">inlets</span> <span class="o">=</span> <span class="p">[]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">outlets</span> <span class="o">=</span> <span class="p">[]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">lineage_data</span> <span class="o">=</span> <span class="kc">None</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_inlets</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;auto&quot;</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
<span class="s2">&quot;task_ids&quot;</span><span class="p">:</span> <span class="p">[],</span>
<span class="s2">&quot;datasets&quot;</span><span class="p">:</span> <span class="p">[],</span>
<span class="p">}</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_outlets</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;datasets&quot;</span><span class="p">:</span> <span class="p">[],</span>
<span class="p">}</span>
<span class="k">if</span> <span class="n">inlets</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_inlets</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">inlets</span><span class="p">)</span>
<span class="k">if</span> <span class="n">outlets</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_outlets</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">outlets</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_comps</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;task_id&#39;</span><span class="p">,</span>
<span class="s1">&#39;dag_id&#39;</span><span class="p">,</span>
<span class="s1">&#39;owner&#39;</span><span class="p">,</span>
<span class="s1">&#39;email&#39;</span><span class="p">,</span>
<span class="s1">&#39;email_on_retry&#39;</span><span class="p">,</span>
<span class="s1">&#39;retry_delay&#39;</span><span class="p">,</span>
<span class="s1">&#39;retry_exponential_backoff&#39;</span><span class="p">,</span>
<span class="s1">&#39;max_retry_delay&#39;</span><span class="p">,</span>
<span class="s1">&#39;start_date&#39;</span><span class="p">,</span>
<span class="s1">&#39;schedule_interval&#39;</span><span class="p">,</span>
<span class="s1">&#39;depends_on_past&#39;</span><span class="p">,</span>
<span class="s1">&#39;wait_for_downstream&#39;</span><span class="p">,</span>
<span class="s1">&#39;priority_weight&#39;</span><span class="p">,</span>
<span class="s1">&#39;sla&#39;</span><span class="p">,</span>
<span class="s1">&#39;execution_timeout&#39;</span><span class="p">,</span>
<span class="s1">&#39;on_failure_callback&#39;</span><span class="p">,</span>
<span class="s1">&#39;on_success_callback&#39;</span><span class="p">,</span>
<span class="s1">&#39;on_retry_callback&#39;</span><span class="p">,</span>
<span class="p">}</span>
<div class="viewcode-block" id="BaseOperator.__eq__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.__eq__">[docs]</a> <span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">if</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span> <span class="ow">and</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">task_id</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">all</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">c</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">c</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_comps</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span></div>
<div class="viewcode-block" id="BaseOperator.__ne__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.__ne__">[docs]</a> <span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span> <span class="o">==</span> <span class="n">other</span></div>
<div class="viewcode-block" id="BaseOperator.__lt__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.__lt__">[docs]</a> <span class="k">def</span> <span class="nf">__lt__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span> <span class="o">&lt;</span> <span class="n">other</span><span class="o">.</span><span class="n">task_id</span></div>
<div class="viewcode-block" id="BaseOperator.__hash__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.__hash__">[docs]</a> <span class="k">def</span> <span class="nf">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">hash_components</span> <span class="o">=</span> <span class="p">[</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)]</span>
<span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_comps</span><span class="p">:</span>
<span class="n">val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">c</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="nb">hash</span><span class="p">(</span><span class="n">val</span><span class="p">)</span>
<span class="n">hash_components</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">val</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">TypeError</span><span class="p">:</span>
<span class="n">hash_components</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">repr</span><span class="p">(</span><span class="n">val</span><span class="p">))</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">tuple</span><span class="p">(</span><span class="n">hash_components</span><span class="p">))</span></div>
<span class="c1"># Composing Operators -----------------------------------------------</span>
<div class="viewcode-block" id="BaseOperator.__rshift__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.__rshift__">[docs]</a> <span class="k">def</span> <span class="nf">__rshift__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Implements Self &gt;&gt; Other == self.set_downstream(other)</span>
<span class="sd"> If &quot;Other&quot; is a DAG, the DAG is assigned to the Operator.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">DAG</span><span class="p">):</span>
<span class="c1"># if this dag is already assigned, do nothing</span>
<span class="c1"># otherwise, do normal dag assignment</span>
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">has_dag</span><span class="p">()</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span> <span class="ow">is</span> <span class="n">other</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dag</span> <span class="o">=</span> <span class="n">other</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">set_downstream</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="k">return</span> <span class="n">other</span></div>
<div class="viewcode-block" id="BaseOperator.__lshift__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.__lshift__">[docs]</a> <span class="k">def</span> <span class="nf">__lshift__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Implements Self &lt;&lt; Other == self.set_upstream(other)</span>
<span class="sd"> If &quot;Other&quot; is a DAG, the DAG is assigned to the Operator.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">DAG</span><span class="p">):</span>
<span class="c1"># if this dag is already assigned, do nothing</span>
<span class="c1"># otherwise, do normal dag assignment</span>
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">has_dag</span><span class="p">()</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span> <span class="ow">is</span> <span class="n">other</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dag</span> <span class="o">=</span> <span class="n">other</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">set_upstream</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="k">return</span> <span class="n">other</span></div>
<div class="viewcode-block" id="BaseOperator.__rrshift__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.__rrshift__">[docs]</a> <span class="k">def</span> <span class="nf">__rrshift__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Called for [DAG] &gt;&gt; [Operator] because DAGs don&#39;t have</span>
<span class="sd"> __rshift__ operators.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="fm">__lshift__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span></div>
<div class="viewcode-block" id="BaseOperator.__rlshift__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.__rlshift__">[docs]</a> <span class="k">def</span> <span class="nf">__rlshift__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Called for [DAG] &lt;&lt; [Operator] because DAGs don&#39;t have</span>
<span class="sd"> __lshift__ operators.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="fm">__rshift__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span></div>
<span class="c1"># /Composing Operators ---------------------------------------------</span>
<span class="nd">@property</span>
<div class="viewcode-block" id="BaseOperator.dag"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.dag">[docs]</a> <span class="k">def</span> <span class="nf">dag</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the Operator&#39;s DAG if set, otherwise raises an error</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_dag</span><span class="p">():</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dag</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
<span class="s1">&#39;Operator </span><span class="si">{}</span><span class="s1"> has not been assigned to a DAG yet&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span></div>
<span class="nd">@dag</span><span class="o">.</span><span class="n">setter</span>
<span class="k">def</span> <span class="nf">dag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dag</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Operators can be assigned to one DAG, one time. Repeat assignments to</span>
<span class="sd"> that same DAG are ok.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dag</span><span class="p">,</span> <span class="n">DAG</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span>
<span class="s1">&#39;Expected DAG; received </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">))</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_dag</span><span class="p">()</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">dag</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
<span class="s2">&quot;The DAG assigned to </span><span class="si">{}</span><span class="s2"> can not be changed.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">:</span>
<span class="n">dag</span><span class="o">.</span><span class="n">add_task</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_dag</span> <span class="o">=</span> <span class="n">dag</span>
<div class="viewcode-block" id="BaseOperator.has_dag"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.has_dag">[docs]</a> <span class="k">def</span> <span class="nf">has_dag</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns True if the Operator has been assigned to a DAG.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;_dag&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="BaseOperator.dag_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.dag_id">[docs]</a> <span class="k">def</span> <span class="nf">dag_id</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_dag</span><span class="p">():</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="s1">&#39;adhoc_&#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">owner</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="BaseOperator.deps"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.deps">[docs]</a> <span class="k">def</span> <span class="nf">deps</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the list of dependencies for the operator. These differ from execution</span>
<span class="sd"> context dependencies in that they are specific to tasks and can be</span>
<span class="sd"> extended/overridden by subclasses.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="p">{</span>
<span class="n">NotInRetryPeriodDep</span><span class="p">(),</span>
<span class="n">PrevDagrunDep</span><span class="p">(),</span>
<span class="n">TriggerRuleDep</span><span class="p">(),</span></div>
<span class="p">}</span>
<span class="nd">@property</span>
<div class="viewcode-block" id="BaseOperator.schedule_interval"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.schedule_interval">[docs]</a> <span class="k">def</span> <span class="nf">schedule_interval</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> The schedule interval of the DAG always wins over individual tasks so</span>
<span class="sd"> that tasks within a DAG always line up. The task still needs a</span>
<span class="sd"> schedule_interval as it may not be attached to a DAG.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_dag</span><span class="p">():</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">_schedule_interval</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="BaseOperator.priority_weight_total"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.priority_weight_total">[docs]</a> <span class="k">def</span> <span class="nf">priority_weight_total</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight_rule</span> <span class="o">==</span> <span class="n">WeightRule</span><span class="o">.</span><span class="n">ABSOLUTE</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">priority_weight</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight_rule</span> <span class="o">==</span> <span class="n">WeightRule</span><span class="o">.</span><span class="n">DOWNSTREAM</span><span class="p">:</span>
<span class="n">upstream</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight_rule</span> <span class="o">==</span> <span class="n">WeightRule</span><span class="o">.</span><span class="n">UPSTREAM</span><span class="p">:</span>
<span class="n">upstream</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">upstream</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">priority_weight</span> <span class="o">+</span> <span class="nb">sum</span><span class="p">(</span>
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">task_id</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">[</span><span class="n">task_id</span><span class="p">]</span><span class="o">.</span><span class="n">priority_weight</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">get_flat_relative_ids</span><span class="p">(</span><span class="n">upstream</span><span class="o">=</span><span class="n">upstream</span><span class="p">))</span></div>
<span class="p">)</span>
<span class="nd">@prepare_lineage</span>
<div class="viewcode-block" id="BaseOperator.pre_execute"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.pre_execute">[docs]</a> <span class="k">def</span> <span class="nf">pre_execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> This hook is triggered right before self.execute() is called.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">pass</span></div>
<div class="viewcode-block" id="BaseOperator.execute"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> This is the main method to derive when creating an operator.</span>
<span class="sd"> Context is the same dictionary used as when rendering jinja templates.</span>
<span class="sd"> Refer to get_template_context for more context.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
<span class="nd">@apply_lineage</span>
<div class="viewcode-block" id="BaseOperator.post_execute"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.post_execute">[docs]</a> <span class="k">def</span> <span class="nf">post_execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> This hook is triggered right after self.execute() is called.</span>
<span class="sd"> It is passed the execution context and any results returned by the</span>
<span class="sd"> operator.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">pass</span></div>
<div class="viewcode-block" id="BaseOperator.on_kill"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.on_kill">[docs]</a> <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Override this method to cleanup subprocesses when a task instance</span>
<span class="sd"> gets killed. Any use of the threading, subprocess or multiprocessing</span>
<span class="sd"> module within an operator needs to be cleaned up or it will leave</span>
<span class="sd"> ghost processes behind.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">pass</span></div>
<div class="viewcode-block" id="BaseOperator.__deepcopy__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.__deepcopy__">[docs]</a> <span class="k">def</span> <span class="nf">__deepcopy__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">memo</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Hack sorting double chained task lists by task_id to avoid hitting</span>
<span class="sd"> max_depth on deepcopy operations.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">sys</span><span class="o">.</span><span class="n">setrecursionlimit</span><span class="p">(</span><span class="mi">5000</span><span class="p">)</span> <span class="c1"># TODO fix this in a better way</span>
<span class="bp">cls</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span>
<span class="n">result</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span>
<span class="n">memo</span><span class="p">[</span><span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="p">)]</span> <span class="o">=</span> <span class="n">result</span>
<span class="n">shallow_copy</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">shallow_copy_attrs</span> <span class="o">+</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_base_operator_shallow_copy_attrs</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">()):</span>
<span class="k">if</span> <span class="n">k</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">shallow_copy</span><span class="p">:</span>
<span class="nb">setattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">memo</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="nb">setattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">v</span><span class="p">))</span>
<span class="k">return</span> <span class="n">result</span></div>
<div class="viewcode-block" id="BaseOperator.__getstate__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.__getstate__">[docs]</a> <span class="k">def</span> <span class="nf">__getstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">state</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span>
<span class="k">del</span> <span class="n">state</span><span class="p">[</span><span class="s1">&#39;_log&#39;</span><span class="p">]</span>
<span class="k">return</span> <span class="n">state</span></div>
<div class="viewcode-block" id="BaseOperator.__setstate__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.__setstate__">[docs]</a> <span class="k">def</span> <span class="nf">__setstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span> <span class="o">=</span> <span class="n">state</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;airflow.task.operators&quot;</span><span class="p">)</span></div>
<div class="viewcode-block" id="BaseOperator.render_template_from_field"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.render_template_from_field">[docs]</a> <span class="k">def</span> <span class="nf">render_template_from_field</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attr</span><span class="p">,</span> <span class="n">content</span><span class="p">,</span> <span class="n">context</span><span class="p">,</span> <span class="n">jinja_env</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Renders a template from a field. If the field is a string, it will</span>
<span class="sd"> simply render the string and return the result. If it is a collection or</span>
<span class="sd"> nested set of collections, it will traverse the structure and render</span>
<span class="sd"> all elements in it. If the field has another type, it will return it as it is.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">rt</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">render_template</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">jinja_env</span><span class="o">.</span><span class="n">from_string</span><span class="p">(</span><span class="n">content</span><span class="p">)</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="o">**</span><span class="n">context</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
<span class="n">result</span> <span class="o">=</span> <span class="p">[</span><span class="n">rt</span><span class="p">(</span><span class="n">attr</span><span class="p">,</span> <span class="n">e</span><span class="p">,</span> <span class="n">context</span><span class="p">)</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="n">content</span><span class="p">]</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="n">result</span> <span class="o">=</span> <span class="p">{</span>
<span class="n">k</span><span class="p">:</span> <span class="n">rt</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{}</span><span class="s2">[</span><span class="si">{}</span><span class="s2">]&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">attr</span><span class="p">,</span> <span class="n">k</span><span class="p">),</span> <span class="n">v</span><span class="p">,</span> <span class="n">context</span><span class="p">)</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="n">content</span><span class="o">.</span><span class="n">items</span><span class="p">())}</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">content</span>
<span class="k">return</span> <span class="n">result</span></div>
<div class="viewcode-block" id="BaseOperator.render_template"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.render_template">[docs]</a> <span class="k">def</span> <span class="nf">render_template</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attr</span><span class="p">,</span> <span class="n">content</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Renders a template either from a file or directly in a field, and returns</span>
<span class="sd"> the rendered result.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">jinja_env</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_template_env</span><span class="p">()</span>
<span class="n">exts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="n">template_ext</span>
<span class="k">if</span> <span class="p">(</span>
<span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">)</span> <span class="ow">and</span>
<span class="nb">any</span><span class="p">([</span><span class="n">content</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="n">ext</span><span class="p">)</span> <span class="k">for</span> <span class="n">ext</span> <span class="ow">in</span> <span class="n">exts</span><span class="p">])):</span>
<span class="k">return</span> <span class="n">jinja_env</span><span class="o">.</span><span class="n">get_template</span><span class="p">(</span><span class="n">content</span><span class="p">)</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="o">**</span><span class="n">context</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">render_template_from_field</span><span class="p">(</span><span class="n">attr</span><span class="p">,</span> <span class="n">content</span><span class="p">,</span> <span class="n">context</span><span class="p">,</span> <span class="n">jinja_env</span><span class="p">)</span></div>
<div class="viewcode-block" id="BaseOperator.get_template_env"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.get_template_env">[docs]</a> <span class="k">def</span> <span class="nf">get_template_env</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">get_template_env</span><span class="p">()</span> \
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;dag&#39;</span><span class="p">)</span> \
<span class="k">else</span> <span class="n">jinja2</span><span class="o">.</span><span class="n">Environment</span><span class="p">(</span><span class="n">cache_size</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span></div>
<div class="viewcode-block" id="BaseOperator.prepare_template"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.prepare_template">[docs]</a> <span class="k">def</span> <span class="nf">prepare_template</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Hook that is triggered after the templated fields get replaced</span>
<span class="sd"> by their content. If you need your operator to alter the</span>
<span class="sd"> content of the file before the template is rendered,</span>
<span class="sd"> it should override this method to do so.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">pass</span></div>
<div class="viewcode-block" id="BaseOperator.resolve_template_files"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.resolve_template_files">[docs]</a> <span class="k">def</span> <span class="nf">resolve_template_files</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="c1"># Getting the content of files for template_field / template_ext</span>
<span class="k">for</span> <span class="n">attr</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">template_fields</span><span class="p">:</span>
<span class="n">content</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attr</span><span class="p">)</span>
<span class="k">if</span> <span class="n">content</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">continue</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">)</span> <span class="ow">and</span> \
<span class="nb">any</span><span class="p">([</span><span class="n">content</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="n">ext</span><span class="p">)</span> <span class="k">for</span> <span class="n">ext</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">template_ext</span><span class="p">]):</span>
<span class="n">env</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_template_env</span><span class="p">()</span>
<span class="k">try</span><span class="p">:</span>
<span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attr</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">loader</span><span class="o">.</span><span class="n">get_source</span><span class="p">(</span><span class="n">env</span><span class="p">,</span> <span class="n">content</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
<span class="n">env</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">get_template_env</span><span class="p">()</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">)</span> <span class="ow">and</span> \
<span class="nb">any</span><span class="p">([</span><span class="n">content</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="n">ext</span><span class="p">)</span> <span class="k">for</span> <span class="n">ext</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">template_ext</span><span class="p">]):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">content</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="n">loader</span><span class="o">.</span><span class="n">get_source</span><span class="p">(</span><span class="n">env</span><span class="p">,</span> <span class="n">content</span><span class="p">[</span><span class="n">i</span><span class="p">])[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">prepare_template</span><span class="p">()</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="BaseOperator.upstream_list"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.upstream_list">[docs]</a> <span class="k">def</span> <span class="nf">upstream_list</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;@property: list of tasks directly upstream&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">tid</span><span class="p">)</span> <span class="k">for</span> <span class="n">tid</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_upstream_task_ids</span><span class="p">]</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="BaseOperator.upstream_task_ids"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.upstream_task_ids">[docs]</a> <span class="k">def</span> <span class="nf">upstream_task_ids</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_upstream_task_ids</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="BaseOperator.downstream_list"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.downstream_list">[docs]</a> <span class="k">def</span> <span class="nf">downstream_list</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;@property: list of tasks directly downstream&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">tid</span><span class="p">)</span> <span class="k">for</span> <span class="n">tid</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_downstream_task_ids</span><span class="p">]</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="BaseOperator.downstream_task_ids"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.downstream_task_ids">[docs]</a> <span class="k">def</span> <span class="nf">downstream_task_ids</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_downstream_task_ids</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="BaseOperator.clear"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.clear">[docs]</a> <span class="k">def</span> <span class="nf">clear</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">downstream</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Clears the state of task instances associated with the task, following</span>
<span class="sd"> the parameters specified.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">start_date</span><span class="p">:</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&gt;=</span> <span class="n">start_date</span><span class="p">)</span>
<span class="k">if</span> <span class="n">end_date</span><span class="p">:</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&lt;=</span> <span class="n">end_date</span><span class="p">)</span>
<span class="n">tasks</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">]</span>
<span class="k">if</span> <span class="n">upstream</span><span class="p">:</span>
<span class="n">tasks</span> <span class="o">+=</span> <span class="p">[</span>
<span class="n">t</span><span class="o">.</span><span class="n">task_id</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_flat_relatives</span><span class="p">(</span><span class="n">upstream</span><span class="o">=</span><span class="kc">True</span><span class="p">)]</span>
<span class="k">if</span> <span class="n">downstream</span><span class="p">:</span>
<span class="n">tasks</span> <span class="o">+=</span> <span class="p">[</span>
<span class="n">t</span><span class="o">.</span><span class="n">task_id</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_flat_relatives</span><span class="p">(</span><span class="n">upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">)]</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">tasks</span><span class="p">))</span>
<span class="n">count</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">count</span><span class="p">()</span>
<span class="n">clear_task_instances</span><span class="p">(</span><span class="n">qry</span><span class="o">.</span><span class="n">all</span><span class="p">(),</span> <span class="n">session</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="k">return</span> <span class="n">count</span></div>
<div class="viewcode-block" id="BaseOperator.get_task_instances"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.get_task_instances">[docs]</a> <span class="k">def</span> <span class="nf">get_task_instances</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">,</span> <span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Get a set of task instance related to this task for a specific date</span>
<span class="sd"> range.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
<span class="n">end_date</span> <span class="o">=</span> <span class="n">end_date</span> <span class="ow">or</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&gt;=</span> <span class="n">start_date</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&lt;=</span> <span class="n">end_date</span><span class="p">,</span>
<span class="p">)</span><span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">()</span></div>
<div class="viewcode-block" id="BaseOperator.get_flat_relative_ids"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.get_flat_relative_ids">[docs]</a> <span class="k">def</span> <span class="nf">get_flat_relative_ids</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">found_descendants</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Get a flat list of relatives&#39; ids, either upstream or downstream.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">found_descendants</span><span class="p">:</span>
<span class="n">found_descendants</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
<span class="n">relative_ids</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_direct_relative_ids</span><span class="p">(</span><span class="n">upstream</span><span class="p">)</span>
<span class="k">for</span> <span class="n">relative_id</span> <span class="ow">in</span> <span class="n">relative_ids</span><span class="p">:</span>
<span class="k">if</span> <span class="n">relative_id</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">found_descendants</span><span class="p">:</span>
<span class="n">found_descendants</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">relative_id</span><span class="p">)</span>
<span class="n">relative_task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">[</span><span class="n">relative_id</span><span class="p">]</span>
<span class="n">relative_task</span><span class="o">.</span><span class="n">get_flat_relative_ids</span><span class="p">(</span><span class="n">upstream</span><span class="p">,</span>
<span class="n">found_descendants</span><span class="p">)</span>
<span class="k">return</span> <span class="n">found_descendants</span></div>
<div class="viewcode-block" id="BaseOperator.get_flat_relatives"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.get_flat_relatives">[docs]</a> <span class="k">def</span> <span class="nf">get_flat_relatives</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Get a flat list of relatives, either upstream or downstream.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">task_id</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">[</span><span class="n">task_id</span><span class="p">],</span>
<span class="bp">self</span><span class="o">.</span><span class="n">get_flat_relative_ids</span><span class="p">(</span><span class="n">upstream</span><span class="p">)))</span></div>
<div class="viewcode-block" id="BaseOperator.run"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.run">[docs]</a> <span class="k">def</span> <span class="nf">run</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">ignore_first_depends_on_past</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_ti_state</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">mark_success</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Run a set of task instances for a date range.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">start_date</span> <span class="o">=</span> <span class="n">start_date</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span>
<span class="n">end_date</span> <span class="o">=</span> <span class="n">end_date</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="ow">or</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="k">for</span> <span class="n">dt</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">date_range</span><span class="p">(</span><span class="n">start_date</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">):</span>
<span class="n">TaskInstance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dt</span><span class="p">)</span><span class="o">.</span><span class="n">run</span><span class="p">(</span>
<span class="n">mark_success</span><span class="o">=</span><span class="n">mark_success</span><span class="p">,</span>
<span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="p">(</span>
<span class="n">dt</span> <span class="o">==</span> <span class="n">start_date</span> <span class="ow">and</span> <span class="n">ignore_first_depends_on_past</span><span class="p">),</span>
<span class="n">ignore_ti_state</span><span class="o">=</span><span class="n">ignore_ti_state</span><span class="p">)</span></div>
<div class="viewcode-block" id="BaseOperator.dry_run"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.dry_run">[docs]</a> <span class="k">def</span> <span class="nf">dry_run</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Dry run&#39;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">attr</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">template_fields</span><span class="p">:</span>
<span class="n">content</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attr</span><span class="p">)</span>
<span class="k">if</span> <span class="n">content</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Rendering template for </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="n">attr</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">content</span><span class="p">)</span></div>
<div class="viewcode-block" id="BaseOperator.get_direct_relative_ids"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.get_direct_relative_ids">[docs]</a> <span class="k">def</span> <span class="nf">get_direct_relative_ids</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Get the direct relative ids to the current task, upstream or</span>
<span class="sd"> downstream.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">upstream</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_upstream_task_ids</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_downstream_task_ids</span></div>
<div class="viewcode-block" id="BaseOperator.get_direct_relatives"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.get_direct_relatives">[docs]</a> <span class="k">def</span> <span class="nf">get_direct_relatives</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Get the direct relatives to the current task, upstream or</span>
<span class="sd"> downstream.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">upstream</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">upstream_list</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">downstream_list</span></div>
<div class="viewcode-block" id="BaseOperator.__repr__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.__repr__">[docs]</a> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;&lt;Task(</span><span class="si">{self.__class__.__name__}</span><span class="s2">): </span><span class="si">{self.task_id}</span><span class="s2">&gt;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="bp">self</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="BaseOperator.task_type"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.task_type">[docs]</a> <span class="k">def</span> <span class="nf">task_type</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span></div>
<div class="viewcode-block" id="BaseOperator.add_only_new"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.add_only_new">[docs]</a> <span class="k">def</span> <span class="nf">add_only_new</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item_set</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
<span class="k">if</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">item_set</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
<span class="s1">&#39;Dependency </span><span class="si">{self}</span><span class="s1">, </span><span class="si">{item}</span><span class="s1"> already registered&#39;</span>
<span class="s1">&#39;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">item_set</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span></div>
<div class="viewcode-block" id="BaseOperator._set_relatives"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator._set_relatives">[docs]</a> <span class="k">def</span> <span class="nf">_set_relatives</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_or_task_list</span><span class="p">,</span> <span class="n">upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">task_list</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">task_or_task_list</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">TypeError</span><span class="p">:</span>
<span class="n">task_list</span> <span class="o">=</span> <span class="p">[</span><span class="n">task_or_task_list</span><span class="p">]</span>
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">task_list</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">BaseOperator</span><span class="p">):</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
<span class="s2">&quot;Relationships can only be set between &quot;</span>
<span class="s2">&quot;Operators; received </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">))</span>
<span class="c1"># relationships can only be set if the tasks share a single DAG. Tasks</span>
<span class="c1"># without a DAG are assigned to that DAG.</span>
<span class="n">dags</span> <span class="o">=</span> <span class="p">{</span><span class="n">t</span><span class="o">.</span><span class="n">_dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">_dag</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="p">[</span><span class="bp">self</span><span class="p">]</span> <span class="o">+</span> <span class="n">task_list</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">has_dag</span><span class="p">()}</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">dags</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
<span class="s1">&#39;Tried to set relationships between tasks in &#39;</span>
<span class="s1">&#39;more than one DAG: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dags</span><span class="o">.</span><span class="n">values</span><span class="p">()))</span>
<span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">dags</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">dag</span> <span class="o">=</span> <span class="n">dags</span><span class="o">.</span><span class="n">popitem</span><span class="p">()[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span>
<span class="s2">&quot;Tried to create relationships between tasks that don&#39;t have &quot;</span>
<span class="s2">&quot;DAGs yet. Set the DAG for at least one &quot;</span>
<span class="s2">&quot;task and try again: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">([</span><span class="bp">self</span><span class="p">]</span> <span class="o">+</span> <span class="n">task_list</span><span class="p">))</span>
<span class="k">if</span> <span class="n">dag</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_dag</span><span class="p">():</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dag</span> <span class="o">=</span> <span class="n">dag</span>
<span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">task_list</span><span class="p">:</span>
<span class="k">if</span> <span class="n">dag</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">task</span><span class="o">.</span><span class="n">has_dag</span><span class="p">():</span>
<span class="n">task</span><span class="o">.</span><span class="n">dag</span> <span class="o">=</span> <span class="n">dag</span>
<span class="k">if</span> <span class="n">upstream</span><span class="p">:</span>
<span class="n">task</span><span class="o">.</span><span class="n">add_only_new</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">_downstream_task_ids</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">add_only_new</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_upstream_task_ids</span><span class="p">,</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">add_only_new</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_downstream_task_ids</span><span class="p">,</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
<span class="n">task</span><span class="o">.</span><span class="n">add_only_new</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">_upstream_task_ids</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span></div>
<div class="viewcode-block" id="BaseOperator.set_downstream"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.set_downstream">[docs]</a> <span class="k">def</span> <span class="nf">set_downstream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_or_task_list</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Set a task or a task list to be directly downstream from the current</span>
<span class="sd"> task.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_set_relatives</span><span class="p">(</span><span class="n">task_or_task_list</span><span class="p">,</span> <span class="n">upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<div class="viewcode-block" id="BaseOperator.set_upstream"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.set_upstream">[docs]</a> <span class="k">def</span> <span class="nf">set_upstream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_or_task_list</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Set a task or a task list to be directly upstream from the current</span>
<span class="sd"> task.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_set_relatives</span><span class="p">(</span><span class="n">task_or_task_list</span><span class="p">,</span> <span class="n">upstream</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="BaseOperator.xcom_push"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.xcom_push">[docs]</a> <span class="k">def</span> <span class="nf">xcom_push</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">context</span><span class="p">,</span>
<span class="n">key</span><span class="p">,</span>
<span class="n">value</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> See TaskInstance.xcom_push()</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">context</span><span class="p">[</span><span class="s1">&#39;ti&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span>
<span class="n">value</span><span class="o">=</span><span class="n">value</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="n">execution_date</span><span class="p">)</span></div>
<div class="viewcode-block" id="BaseOperator.xcom_pull"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.BaseOperator.xcom_pull">[docs]</a> <span class="k">def</span> <span class="nf">xcom_pull</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">context</span><span class="p">,</span>
<span class="n">task_ids</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">dag_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">key</span><span class="o">=</span><span class="n">XCOM_RETURN_KEY</span><span class="p">,</span>
<span class="n">include_prior_dates</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> See TaskInstance.xcom_pull()</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">context</span><span class="p">[</span><span class="s1">&#39;ti&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">xcom_pull</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span>
<span class="n">task_ids</span><span class="o">=</span><span class="n">task_ids</span><span class="p">,</span>
<span class="n">dag_id</span><span class="o">=</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">include_prior_dates</span><span class="o">=</span><span class="n">include_prior_dates</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DagModel"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel">[docs]</a><span class="k">class</span> <span class="nc">DagModel</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span>
<div class="viewcode-block" id="DagModel.__tablename__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.__tablename__">[docs]</a> <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">&quot;dag&quot;</span></div>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> These items are stored in the database for state related information</span>
<span class="sd"> &quot;&quot;&quot;</span>
<div class="viewcode-block" id="DagModel.dag_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.dag_id">[docs]</a> <span class="n">dag_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<span class="c1"># A DAG can be paused from the UI / DB</span>
<span class="c1"># Set this default value of is_paused based on a configuration value!</span>
<div class="viewcode-block" id="DagModel.is_paused_at_creation"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.is_paused_at_creation">[docs]</a> <span class="n">is_paused_at_creation</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span>\
<span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span>
<span class="s1">&#39;dags_are_paused_at_creation&#39;</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagModel.is_paused"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.is_paused">[docs]</a> <span class="n">is_paused</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">is_paused_at_creation</span><span class="p">)</span></div>
<span class="c1"># Whether the DAG is a subdag</span>
<div class="viewcode-block" id="DagModel.is_subdag"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.is_subdag">[docs]</a> <span class="n">is_subdag</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<span class="c1"># Whether that DAG was seen on the last DagBag load</span>
<div class="viewcode-block" id="DagModel.is_active"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.is_active">[docs]</a> <span class="n">is_active</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<span class="c1"># Last time the scheduler started</span>
<div class="viewcode-block" id="DagModel.last_scheduler_run"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.last_scheduler_run">[docs]</a> <span class="n">last_scheduler_run</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div>
<span class="c1"># Last time this DAG was pickled</span>
<div class="viewcode-block" id="DagModel.last_pickled"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.last_pickled">[docs]</a> <span class="n">last_pickled</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div>
<span class="c1"># Time when the DAG last received a refresh signal</span>
<span class="c1"># (e.g. the DAG&#39;s &quot;refresh&quot; button was clicked in the web UI)</span>
<div class="viewcode-block" id="DagModel.last_expired"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.last_expired">[docs]</a> <span class="n">last_expired</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div>
<span class="c1"># Whether (one of) the scheduler is scheduling this DAG at the moment</span>
<div class="viewcode-block" id="DagModel.scheduler_lock"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.scheduler_lock">[docs]</a> <span class="n">scheduler_lock</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">)</span></div>
<span class="c1"># Foreign key to the latest pickle_id</span>
<div class="viewcode-block" id="DagModel.pickle_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.pickle_id">[docs]</a> <span class="n">pickle_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">)</span></div>
<span class="c1"># The location of the file containing the DAG object</span>
<div class="viewcode-block" id="DagModel.fileloc"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.fileloc">[docs]</a> <span class="n">fileloc</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">2000</span><span class="p">))</span></div>
<span class="c1"># String representing the owners</span>
<div class="viewcode-block" id="DagModel.owners"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.owners">[docs]</a> <span class="n">owners</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">2000</span><span class="p">))</span></div>
<span class="c1"># Description of the dag</span>
<div class="viewcode-block" id="DagModel.description"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.description">[docs]</a> <span class="n">description</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Text</span><span class="p">)</span></div>
<span class="c1"># Default view of the inside the webserver</span>
<div class="viewcode-block" id="DagModel.default_view"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.default_view">[docs]</a> <span class="n">default_view</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">25</span><span class="p">))</span></div>
<span class="c1"># Schedule interval</span>
<div class="viewcode-block" id="DagModel.schedule_interval"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.schedule_interval">[docs]</a> <span class="n">schedule_interval</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Interval</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagModel.__repr__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.__repr__">[docs]</a> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;&lt;DAG: </span><span class="si">{self.dag_id}</span><span class="s2">&gt;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DagModel.timezone"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.timezone">[docs]</a> <span class="k">def</span> <span class="nf">timezone</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">settings</span><span class="o">.</span><span class="n">TIMEZONE</span></div>
<span class="nd">@staticmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagModel.get_dagmodel"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.get_dagmodel">[docs]</a> <span class="k">def</span> <span class="nf">get_dagmodel</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span></div>
<span class="nd">@classmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagModel.get_current"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.get_current">[docs]</a> <span class="k">def</span> <span class="nf">get_current</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span></div>
<div class="viewcode-block" id="DagModel.get_default_view"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.get_default_view">[docs]</a> <span class="k">def</span> <span class="nf">get_default_view</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_view</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;webserver&#39;</span><span class="p">,</span> <span class="s1">&#39;dag_default_view&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_view</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagModel.get_last_dagrun"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.get_last_dagrun">[docs]</a> <span class="k">def</span> <span class="nf">get_last_dagrun</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">include_externally_triggered</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="k">return</span> <span class="n">get_last_dagrun</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span>
<span class="n">include_externally_triggered</span><span class="o">=</span><span class="n">include_externally_triggered</span><span class="p">)</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DagModel.safe_dag_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.safe_dag_id">[docs]</a> <span class="k">def</span> <span class="nf">safe_dag_id</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="s1">&#39;__dot__&#39;</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagModel.get_dag"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.get_dag">[docs]</a> <span class="k">def</span> <span class="nf">get_dag</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">DagBag</span><span class="p">(</span><span class="n">dag_folder</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">fileloc</span><span class="p">)</span><span class="o">.</span><span class="n">get_dag</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagModel.create_dagrun"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagModel.create_dagrun">[docs]</a> <span class="k">def</span> <span class="nf">create_dagrun</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">run_id</span><span class="p">,</span>
<span class="n">state</span><span class="p">,</span>
<span class="n">execution_date</span><span class="p">,</span>
<span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">external_trigger</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">conf</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Creates a dag run from this dag including the tasks associated with this dag.</span>
<span class="sd"> Returns the dag run.</span>
<span class="sd"> :param run_id: defines the the run id for this dag run</span>
<span class="sd"> :type run_id: str</span>
<span class="sd"> :param execution_date: the execution date of this dag run</span>
<span class="sd"> :type execution_date: datetime.datetime</span>
<span class="sd"> :param state: the state of the dag run</span>
<span class="sd"> :type state: airflow.utils.state.State</span>
<span class="sd"> :param start_date: the date this dag run should be evaluated</span>
<span class="sd"> :type start_date: datetime.datetime</span>
<span class="sd"> :param external_trigger: whether this dag run is externally triggered</span>
<span class="sd"> :type external_trigger: bool</span>
<span class="sd"> :param session: database session</span>
<span class="sd"> :type session: sqlalchemy.orm.session.Session</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_dag</span><span class="p">()</span><span class="o">.</span><span class="n">create_dagrun</span><span class="p">(</span><span class="n">run_id</span><span class="o">=</span><span class="n">run_id</span><span class="p">,</span>
<span class="n">state</span><span class="o">=</span><span class="n">state</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="n">execution_date</span><span class="p">,</span>
<span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span>
<span class="n">external_trigger</span><span class="o">=</span><span class="n">external_trigger</span><span class="p">,</span>
<span class="n">conf</span><span class="o">=</span><span class="n">conf</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span></div></div>
<div class="viewcode-block" id="DAG"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG">[docs]</a><span class="nd">@functools</span><span class="o">.</span><span class="n">total_ordering</span>
<span class="k">class</span> <span class="nc">DAG</span><span class="p">(</span><span class="n">BaseDag</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> A dag (directed acyclic graph) is a collection of tasks with directional</span>
<span class="sd"> dependencies. A dag also has a schedule, a start end an end date</span>
<span class="sd"> (optional). For each schedule, (say daily or hourly), the DAG needs to run</span>
<span class="sd"> each individual tasks as their dependencies are met. Certain tasks have</span>
<span class="sd"> the property of depending on their own past, meaning that they can&#39;t run</span>
<span class="sd"> until their previous schedule (and upstream tasks) are completed.</span>
<span class="sd"> DAGs essentially act as namespaces for tasks. A task_id can only be</span>
<span class="sd"> added once to a DAG.</span>
<span class="sd"> :param dag_id: The id of the DAG</span>
<span class="sd"> :type dag_id: str</span>
<span class="sd"> :param description: The description for the DAG to e.g. be shown on the webserver</span>
<span class="sd"> :type description: str</span>
<span class="sd"> :param schedule_interval: Defines how often that DAG runs, this</span>
<span class="sd"> timedelta object gets added to your latest task instance&#39;s</span>
<span class="sd"> execution_date to figure out the next schedule</span>
<span class="sd"> :type schedule_interval: datetime.timedelta or</span>
<span class="sd"> dateutil.relativedelta.relativedelta or str that acts as a cron</span>
<span class="sd"> expression</span>
<span class="sd"> :param start_date: The timestamp from which the scheduler will</span>
<span class="sd"> attempt to backfill</span>
<span class="sd"> :type start_date: datetime.datetime</span>
<span class="sd"> :param end_date: A date beyond which your DAG won&#39;t run, leave to None</span>
<span class="sd"> for open ended scheduling</span>
<span class="sd"> :type end_date: datetime.datetime</span>
<span class="sd"> :param template_searchpath: This list of folders (non relative)</span>
<span class="sd"> defines where jinja will look for your templates. Order matters.</span>
<span class="sd"> Note that jinja/airflow includes the path of your DAG file by</span>
<span class="sd"> default</span>
<span class="sd"> :type template_searchpath: str or list[str]</span>
<span class="sd"> :param user_defined_macros: a dictionary of macros that will be exposed</span>
<span class="sd"> in your jinja templates. For example, passing ``dict(foo=&#39;bar&#39;)``</span>
<span class="sd"> to this argument allows you to ``{{ foo }}`` in all jinja</span>
<span class="sd"> templates related to this DAG. Note that you can pass any</span>
<span class="sd"> type of object here.</span>
<span class="sd"> :type user_defined_macros: dict</span>
<span class="sd"> :param user_defined_filters: a dictionary of filters that will be exposed</span>
<span class="sd"> in your jinja templates. For example, passing</span>
<span class="sd"> ``dict(hello=lambda name: &#39;Hello %s&#39; % name)`` to this argument allows</span>
<span class="sd"> you to ``{{ &#39;world&#39; | hello }}`` in all jinja templates related to</span>
<span class="sd"> this DAG.</span>
<span class="sd"> :type user_defined_filters: dict</span>
<span class="sd"> :param default_args: A dictionary of default parameters to be used</span>
<span class="sd"> as constructor keyword parameters when initialising operators.</span>
<span class="sd"> Note that operators have the same hook, and precede those defined</span>
<span class="sd"> here, meaning that if your dict contains `&#39;depends_on_past&#39;: True`</span>
<span class="sd"> here and `&#39;depends_on_past&#39;: False` in the operator&#39;s call</span>
<span class="sd"> `default_args`, the actual value will be `False`.</span>
<span class="sd"> :type default_args: dict</span>
<span class="sd"> :param params: a dictionary of DAG level parameters that are made</span>
<span class="sd"> accessible in templates, namespaced under `params`. These</span>
<span class="sd"> params can be overridden at the task level.</span>
<span class="sd"> :type params: dict</span>
<span class="sd"> :param concurrency: the number of task instances allowed to run</span>
<span class="sd"> concurrently</span>
<span class="sd"> :type concurrency: int</span>
<span class="sd"> :param max_active_runs: maximum number of active DAG runs, beyond this</span>
<span class="sd"> number of DAG runs in a running state, the scheduler won&#39;t create</span>
<span class="sd"> new active DAG runs</span>
<span class="sd"> :type max_active_runs: int</span>
<span class="sd"> :param dagrun_timeout: specify how long a DagRun should be up before</span>
<span class="sd"> timing out / failing, so that new DagRuns can be created</span>
<span class="sd"> :type dagrun_timeout: datetime.timedelta</span>
<span class="sd"> :param sla_miss_callback: specify a function to call when reporting SLA</span>
<span class="sd"> timeouts.</span>
<span class="sd"> :type sla_miss_callback: types.FunctionType</span>
<span class="sd"> :param default_view: Specify DAG default view (tree, graph, duration,</span>
<span class="sd"> gantt, landing_times)</span>
<span class="sd"> :type default_view: str</span>
<span class="sd"> :param orientation: Specify DAG orientation in graph view (LR, TB, RL, BT)</span>
<span class="sd"> :type orientation: str</span>
<span class="sd"> :param catchup: Perform scheduler catchup (or only run latest)? Defaults to True</span>
<span class="sd"> :type catchup: bool</span>
<span class="sd"> :param on_failure_callback: A function to be called when a DagRun of this dag fails.</span>
<span class="sd"> A context dictionary is passed as a single parameter to this function.</span>
<span class="sd"> :type on_failure_callback: callable</span>
<span class="sd"> :param on_success_callback: Much like the ``on_failure_callback`` except</span>
<span class="sd"> that it is executed when the dag succeeds.</span>
<span class="sd"> :type on_success_callback: callable</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span>
<span class="n">description</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">,</span>
<span class="n">schedule_interval</span><span class="o">=</span><span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">),</span>
<span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">full_filepath</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">template_searchpath</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">user_defined_macros</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">user_defined_filters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">default_args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">concurrency</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;dag_concurrency&#39;</span><span class="p">),</span>
<span class="n">max_active_runs</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span>
<span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;max_active_runs_per_dag&#39;</span><span class="p">),</span>
<span class="n">dagrun_timeout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">sla_miss_callback</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">default_view</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">orientation</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;webserver&#39;</span><span class="p">,</span> <span class="s1">&#39;dag_orientation&#39;</span><span class="p">),</span>
<span class="n">catchup</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;scheduler&#39;</span><span class="p">,</span> <span class="s1">&#39;catchup_by_default&#39;</span><span class="p">),</span>
<span class="n">on_success_callback</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">on_failure_callback</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">doc_md</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">params</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">user_defined_macros</span> <span class="o">=</span> <span class="n">user_defined_macros</span>
<span class="bp">self</span><span class="o">.</span><span class="n">user_defined_filters</span> <span class="o">=</span> <span class="n">user_defined_filters</span>
<span class="bp">self</span><span class="o">.</span><span class="n">default_args</span> <span class="o">=</span> <span class="n">default_args</span> <span class="ow">or</span> <span class="p">{}</span>
<span class="bp">self</span><span class="o">.</span><span class="n">params</span> <span class="o">=</span> <span class="n">params</span> <span class="ow">or</span> <span class="p">{}</span>
<span class="c1"># merging potentially conflicting default_args[&#39;params&#39;] into params</span>
<span class="k">if</span> <span class="s1">&#39;params&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">])</span>
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">&#39;params&#39;</span><span class="p">]</span>
<span class="n">validate_key</span><span class="p">(</span><span class="n">dag_id</span><span class="p">)</span>
<span class="c1"># Properties from BaseDag</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_dag_id</span> <span class="o">=</span> <span class="n">dag_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_full_filepath</span> <span class="o">=</span> <span class="n">full_filepath</span> <span class="k">if</span> <span class="n">full_filepath</span> <span class="k">else</span> <span class="s1">&#39;&#39;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_concurrency</span> <span class="o">=</span> <span class="n">concurrency</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_pickle_id</span> <span class="o">=</span> <span class="kc">None</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_description</span> <span class="o">=</span> <span class="n">description</span>
<span class="c1"># set file location to caller source path</span>
<span class="bp">self</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">_getframe</span><span class="p">()</span><span class="o">.</span><span class="n">f_back</span><span class="o">.</span><span class="n">f_code</span><span class="o">.</span><span class="n">co_filename</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
<span class="c1"># set timezone</span>
<span class="k">if</span> <span class="n">start_date</span> <span class="ow">and</span> <span class="n">start_date</span><span class="o">.</span><span class="n">tzinfo</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">timezone</span> <span class="o">=</span> <span class="n">start_date</span><span class="o">.</span><span class="n">tzinfo</span>
<span class="k">elif</span> <span class="s1">&#39;start_date&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">&#39;start_date&#39;</span><span class="p">]:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">&#39;start_date&#39;</span><span class="p">],</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">&#39;start_date&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">timezone</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">&#39;start_date&#39;</span><span class="p">])</span>
<span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">timezone</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">&#39;start_date&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">tzinfo</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;timezone&#39;</span><span class="p">)</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">timezone</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">timezone</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">TIMEZONE</span>
<span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="n">start_date</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="n">end_date</span><span class="p">)</span>
<span class="c1"># also convert tasks</span>
<span class="k">if</span> <span class="s1">&#39;start_date&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">&#39;start_date&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">&#39;start_date&#39;</span><span class="p">])</span>
<span class="p">)</span>
<span class="k">if</span> <span class="s1">&#39;end_date&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">&#39;end_date&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">&#39;end_date&#39;</span><span class="p">])</span>
<span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">schedule_interval</span> <span class="o">=</span> <span class="n">schedule_interval</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">schedule_interval</span><span class="p">,</span> <span class="n">Hashable</span><span class="p">)</span> <span class="ow">and</span> <span class="n">schedule_interval</span> <span class="ow">in</span> <span class="n">cron_presets</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span> <span class="o">=</span> <span class="n">cron_presets</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">schedule_interval</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">schedule_interval</span> <span class="o">==</span> <span class="s1">&#39;@once&#39;</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span> <span class="o">=</span> <span class="n">schedule_interval</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">template_searchpath</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
<span class="n">template_searchpath</span> <span class="o">=</span> <span class="p">[</span><span class="n">template_searchpath</span><span class="p">]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">template_searchpath</span> <span class="o">=</span> <span class="n">template_searchpath</span>
<span class="bp">self</span><span class="o">.</span><span class="n">parent_dag</span> <span class="o">=</span> <span class="kc">None</span> <span class="c1"># Gets set when DAGs are loaded</span>
<span class="bp">self</span><span class="o">.</span><span class="n">last_loaded</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">safe_dag_id</span> <span class="o">=</span> <span class="n">dag_id</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="s1">&#39;__dot__&#39;</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">max_active_runs</span> <span class="o">=</span> <span class="n">max_active_runs</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dagrun_timeout</span> <span class="o">=</span> <span class="n">dagrun_timeout</span>
<span class="bp">self</span><span class="o">.</span><span class="n">sla_miss_callback</span> <span class="o">=</span> <span class="n">sla_miss_callback</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_default_view</span> <span class="o">=</span> <span class="n">default_view</span>
<span class="bp">self</span><span class="o">.</span><span class="n">orientation</span> <span class="o">=</span> <span class="n">orientation</span>
<span class="bp">self</span><span class="o">.</span><span class="n">catchup</span> <span class="o">=</span> <span class="n">catchup</span>
<span class="bp">self</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="kc">False</span> <span class="c1"># DagBag.bag_dag() will set this to True if appropriate</span>
<span class="bp">self</span><span class="o">.</span><span class="n">partial</span> <span class="o">=</span> <span class="kc">False</span>
<span class="bp">self</span><span class="o">.</span><span class="n">on_success_callback</span> <span class="o">=</span> <span class="n">on_success_callback</span>
<span class="bp">self</span><span class="o">.</span><span class="n">on_failure_callback</span> <span class="o">=</span> <span class="n">on_failure_callback</span>
<span class="bp">self</span><span class="o">.</span><span class="n">doc_md</span> <span class="o">=</span> <span class="n">doc_md</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_old_context_manager_dags</span> <span class="o">=</span> <span class="p">[]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_comps</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;dag_id&#39;</span><span class="p">,</span>
<span class="s1">&#39;task_ids&#39;</span><span class="p">,</span>
<span class="s1">&#39;parent_dag&#39;</span><span class="p">,</span>
<span class="s1">&#39;start_date&#39;</span><span class="p">,</span>
<span class="s1">&#39;schedule_interval&#39;</span><span class="p">,</span>
<span class="s1">&#39;full_filepath&#39;</span><span class="p">,</span>
<span class="s1">&#39;template_searchpath&#39;</span><span class="p">,</span>
<span class="s1">&#39;last_loaded&#39;</span><span class="p">,</span>
<span class="p">}</span>
<div class="viewcode-block" id="DAG.__repr__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.__repr__">[docs]</a> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;&lt;DAG: </span><span class="si">{self.dag_id}</span><span class="s2">&gt;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span></div>
<div class="viewcode-block" id="DAG.__eq__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.__eq__">[docs]</a> <span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">if</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span> <span class="ow">and</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">dag_id</span><span class="p">):</span>
<span class="c1"># Use getattr() instead of __dict__ as __dict__ doesn&#39;t return</span>
<span class="c1"># correct values for properties.</span>
<span class="k">return</span> <span class="nb">all</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">c</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="o">==</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">c</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_comps</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span></div>
<div class="viewcode-block" id="DAG.__ne__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.__ne__">[docs]</a> <span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span> <span class="o">==</span> <span class="n">other</span></div>
<div class="viewcode-block" id="DAG.__lt__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.__lt__">[docs]</a> <span class="k">def</span> <span class="nf">__lt__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">&lt;</span> <span class="n">other</span><span class="o">.</span><span class="n">dag_id</span></div>
<div class="viewcode-block" id="DAG.__hash__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.__hash__">[docs]</a> <span class="k">def</span> <span class="nf">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">hash_components</span> <span class="o">=</span> <span class="p">[</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)]</span>
<span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_comps</span><span class="p">:</span>
<span class="c1"># task_ids returns a list and lists can&#39;t be hashed</span>
<span class="k">if</span> <span class="n">c</span> <span class="o">==</span> <span class="s1">&#39;task_ids&#39;</span><span class="p">:</span>
<span class="n">val</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">c</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="nb">hash</span><span class="p">(</span><span class="n">val</span><span class="p">)</span>
<span class="n">hash_components</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">val</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">TypeError</span><span class="p">:</span>
<span class="n">hash_components</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">repr</span><span class="p">(</span><span class="n">val</span><span class="p">))</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">tuple</span><span class="p">(</span><span class="n">hash_components</span><span class="p">))</span></div>
<span class="c1"># Context Manager -----------------------------------------------</span>
<div class="viewcode-block" id="DAG.__enter__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.__enter__">[docs]</a> <span class="k">def</span> <span class="nf">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">global</span> <span class="n">_CONTEXT_MANAGER_DAG</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_old_context_manager_dags</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_CONTEXT_MANAGER_DAG</span><span class="p">)</span>
<span class="n">_CONTEXT_MANAGER_DAG</span> <span class="o">=</span> <span class="bp">self</span>
<span class="k">return</span> <span class="bp">self</span></div>
<div class="viewcode-block" id="DAG.__exit__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.__exit__">[docs]</a> <span class="k">def</span> <span class="nf">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">_type</span><span class="p">,</span> <span class="n">_value</span><span class="p">,</span> <span class="n">_tb</span><span class="p">):</span>
<span class="k">global</span> <span class="n">_CONTEXT_MANAGER_DAG</span>
<span class="n">_CONTEXT_MANAGER_DAG</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_old_context_manager_dags</span><span class="o">.</span><span class="n">pop</span><span class="p">()</span></div>
<span class="c1"># /Context Manager ----------------------------------------------</span>
<div class="viewcode-block" id="DAG.get_default_view"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.get_default_view">[docs]</a> <span class="k">def</span> <span class="nf">get_default_view</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;This is only there for backward compatible jinja2 templates&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_default_view</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;webserver&#39;</span><span class="p">,</span> <span class="s1">&#39;dag_default_view&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_default_view</span></div>
<div class="viewcode-block" id="DAG.date_range"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.date_range">[docs]</a> <span class="k">def</span> <span class="nf">date_range</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">start_date</span><span class="p">,</span> <span class="n">num</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()):</span>
<span class="k">if</span> <span class="n">num</span><span class="p">:</span>
<span class="n">end_date</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">return</span> <span class="n">utils_date_range</span><span class="p">(</span>
<span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span>
<span class="n">num</span><span class="o">=</span><span class="n">num</span><span class="p">,</span> <span class="n">delta</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span><span class="p">)</span></div>
<div class="viewcode-block" id="DAG.is_fixed_time_schedule"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.is_fixed_time_schedule">[docs]</a> <span class="k">def</span> <span class="nf">is_fixed_time_schedule</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Figures out if the DAG schedule has a fixed time (e.g. 3 AM).</span>
<span class="sd"> :return: True if the schedule has a fixed time, False if not.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">now</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
<span class="n">cron</span> <span class="o">=</span> <span class="n">croniter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span><span class="p">,</span> <span class="n">now</span><span class="p">)</span>
<span class="n">start</span> <span class="o">=</span> <span class="n">cron</span><span class="o">.</span><span class="n">get_next</span><span class="p">(</span><span class="n">datetime</span><span class="p">)</span>
<span class="n">cron_next</span> <span class="o">=</span> <span class="n">cron</span><span class="o">.</span><span class="n">get_next</span><span class="p">(</span><span class="n">datetime</span><span class="p">)</span>
<span class="k">if</span> <span class="n">cron_next</span><span class="o">.</span><span class="n">minute</span> <span class="o">==</span> <span class="n">start</span><span class="o">.</span><span class="n">minute</span> <span class="ow">and</span> <span class="n">cron_next</span><span class="o">.</span><span class="n">hour</span> <span class="o">==</span> <span class="n">start</span><span class="o">.</span><span class="n">hour</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="k">return</span> <span class="kc">False</span></div>
<div class="viewcode-block" id="DAG.following_schedule"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.following_schedule">[docs]</a> <span class="k">def</span> <span class="nf">following_schedule</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dttm</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Calculates the following schedule for this dag in UTC.</span>
<span class="sd"> :param dttm: utc datetime</span>
<span class="sd"> :return: utc datetime</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
<span class="c1"># we don&#39;t want to rely on the transitions created by</span>
<span class="c1"># croniter as they are not always correct</span>
<span class="n">dttm</span> <span class="o">=</span> <span class="n">pendulum</span><span class="o">.</span><span class="n">instance</span><span class="p">(</span><span class="n">dttm</span><span class="p">)</span>
<span class="n">naive</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">make_naive</span><span class="p">(</span><span class="n">dttm</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">timezone</span><span class="p">)</span>
<span class="n">cron</span> <span class="o">=</span> <span class="n">croniter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span><span class="p">,</span> <span class="n">naive</span><span class="p">)</span>
<span class="c1"># We assume that DST transitions happen on the minute/hour</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_fixed_time_schedule</span><span class="p">():</span>
<span class="c1"># relative offset (eg. every 5 minutes)</span>
<span class="n">delta</span> <span class="o">=</span> <span class="n">cron</span><span class="o">.</span><span class="n">get_next</span><span class="p">(</span><span class="n">datetime</span><span class="p">)</span> <span class="o">-</span> <span class="n">naive</span>
<span class="n">following</span> <span class="o">=</span> <span class="n">dttm</span><span class="o">.</span><span class="n">in_timezone</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">timezone</span><span class="p">)</span><span class="o">.</span><span class="n">add_timedelta</span><span class="p">(</span><span class="n">delta</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># absolute (e.g. 3 AM)</span>
<span class="n">naive</span> <span class="o">=</span> <span class="n">cron</span><span class="o">.</span><span class="n">get_next</span><span class="p">(</span><span class="n">datetime</span><span class="p">)</span>
<span class="n">tz</span> <span class="o">=</span> <span class="n">pendulum</span><span class="o">.</span><span class="n">timezone</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">timezone</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="n">following</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">make_aware</span><span class="p">(</span><span class="n">naive</span><span class="p">,</span> <span class="n">tz</span><span class="p">)</span>
<span class="k">return</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="n">following</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">dttm</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span></div>
<div class="viewcode-block" id="DAG.previous_schedule"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.previous_schedule">[docs]</a> <span class="k">def</span> <span class="nf">previous_schedule</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dttm</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Calculates the previous schedule for this dag in UTC</span>
<span class="sd"> :param dttm: utc datetime</span>
<span class="sd"> :return: utc datetime</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
<span class="c1"># we don&#39;t want to rely on the transitions created by</span>
<span class="c1"># croniter as they are not always correct</span>
<span class="n">dttm</span> <span class="o">=</span> <span class="n">pendulum</span><span class="o">.</span><span class="n">instance</span><span class="p">(</span><span class="n">dttm</span><span class="p">)</span>
<span class="n">naive</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">make_naive</span><span class="p">(</span><span class="n">dttm</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">timezone</span><span class="p">)</span>
<span class="n">cron</span> <span class="o">=</span> <span class="n">croniter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span><span class="p">,</span> <span class="n">naive</span><span class="p">)</span>
<span class="c1"># We assume that DST transitions happen on the minute/hour</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_fixed_time_schedule</span><span class="p">():</span>
<span class="c1"># relative offset (eg. every 5 minutes)</span>
<span class="n">delta</span> <span class="o">=</span> <span class="n">naive</span> <span class="o">-</span> <span class="n">cron</span><span class="o">.</span><span class="n">get_prev</span><span class="p">(</span><span class="n">datetime</span><span class="p">)</span>
<span class="n">previous</span> <span class="o">=</span> <span class="n">dttm</span><span class="o">.</span><span class="n">in_timezone</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">timezone</span><span class="p">)</span><span class="o">.</span><span class="n">subtract_timedelta</span><span class="p">(</span><span class="n">delta</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># absolute (e.g. 3 AM)</span>
<span class="n">naive</span> <span class="o">=</span> <span class="n">cron</span><span class="o">.</span><span class="n">get_prev</span><span class="p">(</span><span class="n">datetime</span><span class="p">)</span>
<span class="n">tz</span> <span class="o">=</span> <span class="n">pendulum</span><span class="o">.</span><span class="n">timezone</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">timezone</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="n">previous</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">make_aware</span><span class="p">(</span><span class="n">naive</span><span class="p">,</span> <span class="n">tz</span><span class="p">)</span>
<span class="k">return</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="n">previous</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">dttm</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span></div>
<div class="viewcode-block" id="DAG.get_run_dates"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.get_run_dates">[docs]</a> <span class="k">def</span> <span class="nf">get_run_dates</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">start_date</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a list of dates between the interval received as parameter using this</span>
<span class="sd"> dag&#39;s schedule interval. Returned dates can be used for execution dates.</span>
<span class="sd"> :param start_date: the start date of the interval</span>
<span class="sd"> :type start_date: datetime</span>
<span class="sd"> :param end_date: the end date of the interval, defaults to timezone.utcnow()</span>
<span class="sd"> :type end_date: datetime</span>
<span class="sd"> :return: a list of dates within the interval following the dag&#39;s schedule</span>
<span class="sd"> :rtype: list</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">run_dates</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">using_start_date</span> <span class="o">=</span> <span class="n">start_date</span>
<span class="n">using_end_date</span> <span class="o">=</span> <span class="n">end_date</span>
<span class="c1"># dates for dag runs</span>
<span class="n">using_start_date</span> <span class="o">=</span> <span class="n">using_start_date</span> <span class="ow">or</span> <span class="nb">min</span><span class="p">([</span><span class="n">t</span><span class="o">.</span><span class="n">start_date</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">])</span>
<span class="n">using_end_date</span> <span class="o">=</span> <span class="n">using_end_date</span> <span class="ow">or</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="c1"># next run date for a subdag isn&#39;t relevant (schedule_interval for subdags</span>
<span class="c1"># is ignored) so we use the dag run&#39;s start date in the case of a subdag</span>
<span class="n">next_run_date</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">normalize_schedule</span><span class="p">(</span><span class="n">using_start_date</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_subdag</span> <span class="k">else</span> <span class="n">using_start_date</span><span class="p">)</span>
<span class="k">while</span> <span class="n">next_run_date</span> <span class="ow">and</span> <span class="n">next_run_date</span> <span class="o">&lt;=</span> <span class="n">using_end_date</span><span class="p">:</span>
<span class="n">run_dates</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">next_run_date</span><span class="p">)</span>
<span class="n">next_run_date</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">following_schedule</span><span class="p">(</span><span class="n">next_run_date</span><span class="p">)</span>
<span class="k">return</span> <span class="n">run_dates</span></div>
<div class="viewcode-block" id="DAG.normalize_schedule"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.normalize_schedule">[docs]</a> <span class="k">def</span> <span class="nf">normalize_schedule</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dttm</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns dttm + interval unless dttm is first interval then it returns dttm</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">following</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">following_schedule</span><span class="p">(</span><span class="n">dttm</span><span class="p">)</span>
<span class="c1"># in case of @once</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">following</span><span class="p">:</span>
<span class="k">return</span> <span class="n">dttm</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">previous_schedule</span><span class="p">(</span><span class="n">following</span><span class="p">)</span> <span class="o">!=</span> <span class="n">dttm</span><span class="p">:</span>
<span class="k">return</span> <span class="n">following</span>
<span class="k">return</span> <span class="n">dttm</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.get_last_dagrun"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.get_last_dagrun">[docs]</a> <span class="k">def</span> <span class="nf">get_last_dagrun</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">include_externally_triggered</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="k">return</span> <span class="n">get_last_dagrun</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span>
<span class="n">include_externally_triggered</span><span class="o">=</span><span class="n">include_externally_triggered</span><span class="p">)</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.dag_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.dag_id">[docs]</a> <span class="k">def</span> <span class="nf">dag_id</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dag_id</span></div>
<span class="nd">@dag_id</span><span class="o">.</span><span class="n">setter</span>
<span class="k">def</span> <span class="nf">dag_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_dag_id</span> <span class="o">=</span> <span class="n">value</span>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.full_filepath"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.full_filepath">[docs]</a> <span class="k">def</span> <span class="nf">full_filepath</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_full_filepath</span></div>
<span class="nd">@full_filepath</span><span class="o">.</span><span class="n">setter</span>
<span class="k">def</span> <span class="nf">full_filepath</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_full_filepath</span> <span class="o">=</span> <span class="n">value</span>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.concurrency"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.concurrency">[docs]</a> <span class="k">def</span> <span class="nf">concurrency</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_concurrency</span></div>
<span class="nd">@concurrency</span><span class="o">.</span><span class="n">setter</span>
<span class="k">def</span> <span class="nf">concurrency</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_concurrency</span> <span class="o">=</span> <span class="n">value</span>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.description"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.description">[docs]</a> <span class="k">def</span> <span class="nf">description</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_description</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.pickle_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.pickle_id">[docs]</a> <span class="k">def</span> <span class="nf">pickle_id</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pickle_id</span></div>
<span class="nd">@pickle_id</span><span class="o">.</span><span class="n">setter</span>
<span class="k">def</span> <span class="nf">pickle_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_pickle_id</span> <span class="o">=</span> <span class="n">value</span>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.tasks"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.tasks">[docs]</a> <span class="k">def</span> <span class="nf">tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">values</span><span class="p">())</span></div>
<span class="nd">@tasks</span><span class="o">.</span><span class="n">setter</span>
<span class="k">def</span> <span class="nf">tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">val</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span>
<span class="s1">&#39;DAG.tasks can not be modified. Use dag.add_task() instead.&#39;</span><span class="p">)</span>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.task_ids"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.task_ids">[docs]</a> <span class="k">def</span> <span class="nf">task_ids</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.filepath"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.filepath">[docs]</a> <span class="k">def</span> <span class="nf">filepath</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> File location of where the dag object is instantiated</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">fn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">full_filepath</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">settings</span><span class="o">.</span><span class="n">DAGS_FOLDER</span> <span class="o">+</span> <span class="s1">&#39;/&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="n">fn</span> <span class="o">=</span> <span class="n">fn</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span> <span class="o">+</span> <span class="s1">&#39;/&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">fn</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.folder"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.folder">[docs]</a> <span class="k">def</span> <span class="nf">folder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Folder location of where the dag object is instantiated</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">full_filepath</span><span class="p">)</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.owner"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.owner">[docs]</a> <span class="k">def</span> <span class="nf">owner</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">([</span><span class="n">t</span><span class="o">.</span><span class="n">owner</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">])))</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG._get_concurrency_reached"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG._get_concurrency_reached">[docs]</a> <span class="k">def</span> <span class="nf">_get_concurrency_reached</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">func</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">task_id</span><span class="p">))</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">qry</span><span class="o">.</span><span class="n">scalar</span><span class="p">()</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">concurrency</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.concurrency_reached"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.concurrency_reached">[docs]</a> <span class="k">def</span> <span class="nf">concurrency_reached</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a boolean indicating whether the concurrency limit for this DAG</span>
<span class="sd"> has been reached</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_concurrency_reached</span><span class="p">()</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG._get_is_paused"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG._get_is_paused">[docs]</a> <span class="k">def</span> <span class="nf">_get_is_paused</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span>
<span class="k">return</span> <span class="n">qry</span><span class="o">.</span><span class="n">value</span><span class="p">(</span><span class="s1">&#39;is_paused&#39;</span><span class="p">)</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.is_paused"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.is_paused">[docs]</a> <span class="k">def</span> <span class="nf">is_paused</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a boolean indicating whether this DAG is paused</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_is_paused</span><span class="p">()</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.handle_callback"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.handle_callback">[docs]</a> <span class="k">def</span> <span class="nf">handle_callback</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dagrun</span><span class="p">,</span> <span class="n">success</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">reason</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Triggers the appropriate callback depending on the value of success, namely the</span>
<span class="sd"> on_failure_callback or on_success_callback. This method gets the context of a</span>
<span class="sd"> single TaskInstance part of this DagRun and passes that to the callable along</span>
<span class="sd"> with a &#39;reason&#39;, primarily to differentiate DagRun failures.</span>
<span class="sd"> .. note: The logs end up in</span>
<span class="sd"> ``$AIRFLOW_HOME/logs/scheduler/latest/PROJECT/DAG_FILE.py.log``</span>
<span class="sd"> :param dagrun: DagRun object</span>
<span class="sd"> :param success: Flag to specify if failure or success callback should be called</span>
<span class="sd"> :param reason: Completion reason</span>
<span class="sd"> :param session: Database session</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">callback</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">on_success_callback</span> <span class="k">if</span> <span class="n">success</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">on_failure_callback</span>
<span class="k">if</span> <span class="n">callback</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Executing dag callback function: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">callback</span><span class="p">))</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">dagrun</span><span class="o">.</span><span class="n">get_task_instances</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="n">ti</span> <span class="o">=</span> <span class="n">tis</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="c1"># get first TaskInstance of DagRun</span>
<span class="n">ti</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
<span class="n">context</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">get_template_context</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="n">context</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">&#39;reason&#39;</span><span class="p">:</span> <span class="n">reason</span><span class="p">})</span>
<span class="n">callback</span><span class="p">(</span><span class="n">context</span><span class="p">)</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.get_active_runs"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.get_active_runs">[docs]</a> <span class="k">def</span> <span class="nf">get_active_runs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a list of dag run execution dates currently running</span>
<span class="sd"> :param session:</span>
<span class="sd"> :return: List of execution dates</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">runs</span> <span class="o">=</span> <span class="n">DagRun</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">state</span><span class="o">=</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">)</span>
<span class="n">active_dates</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">run</span> <span class="ow">in</span> <span class="n">runs</span><span class="p">:</span>
<span class="n">active_dates</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">run</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span>
<span class="k">return</span> <span class="n">active_dates</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.get_num_active_runs"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.get_num_active_runs">[docs]</a> <span class="k">def</span> <span class="nf">get_num_active_runs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">external_trigger</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the number of active &quot;running&quot; dag runs</span>
<span class="sd"> :param external_trigger: True for externally triggered active dag runs</span>
<span class="sd"> :type external_trigger: bool</span>
<span class="sd"> :param session:</span>
<span class="sd"> :return: number greater than 0 for active dag runs</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">query</span> <span class="o">=</span> <span class="p">(</span><span class="n">session</span>
<span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span>
<span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span>
<span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">))</span>
<span class="k">if</span> <span class="n">external_trigger</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">external_trigger</span> <span class="o">==</span> <span class="n">external_trigger</span><span class="p">)</span>
<span class="k">return</span> <span class="n">query</span><span class="o">.</span><span class="n">count</span><span class="p">()</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.get_dagrun"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.get_dagrun">[docs]</a> <span class="k">def</span> <span class="nf">get_dagrun</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the dag run for a given execution date if it exists, otherwise</span>
<span class="sd"> none.</span>
<span class="sd"> :param execution_date: The execution date of the DagRun to find.</span>
<span class="sd"> :param session:</span>
<span class="sd"> :return: The DagRun if found, otherwise None.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">dagrun</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span>
<span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">execution_date</span><span class="p">)</span>
<span class="o">.</span><span class="n">first</span><span class="p">())</span>
<span class="k">return</span> <span class="n">dagrun</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG._get_latest_execution_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG._get_latest_execution_date">[docs]</a> <span class="k">def</span> <span class="nf">_get_latest_execution_date</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">func</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span><span class="p">))</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span></div>
<span class="p">)</span><span class="o">.</span><span class="n">scalar</span><span class="p">()</span>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.latest_execution_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.latest_execution_date">[docs]</a> <span class="k">def</span> <span class="nf">latest_execution_date</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the latest date for which at least one dag run exists</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_latest_execution_date</span><span class="p">()</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.subdags"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.subdags">[docs]</a> <span class="k">def</span> <span class="nf">subdags</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a list of the subdag objects associated to this DAG</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># Check SubDag for class but don&#39;t check class directly</span>
<span class="kn">from</span> <span class="nn">airflow.operators.subdag_operator</span> <span class="k">import</span> <span class="n">SubDagOperator</span>
<span class="n">subdag_lst</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">:</span>
<span class="k">if</span> <span class="p">(</span><span class="nb">isinstance</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="n">SubDagOperator</span><span class="p">)</span> <span class="ow">or</span>
<span class="c1"># TODO remove in Airflow 2.0</span>
<span class="nb">type</span><span class="p">(</span><span class="n">task</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s1">&#39;SubDagOperator&#39;</span><span class="p">):</span>
<span class="n">subdag_lst</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">subdag</span><span class="p">)</span>
<span class="n">subdag_lst</span> <span class="o">+=</span> <span class="n">task</span><span class="o">.</span><span class="n">subdag</span><span class="o">.</span><span class="n">subdags</span>
<span class="k">return</span> <span class="n">subdag_lst</span></div>
<div class="viewcode-block" id="DAG.resolve_template_files"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.resolve_template_files">[docs]</a> <span class="k">def</span> <span class="nf">resolve_template_files</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">:</span>
<span class="n">t</span><span class="o">.</span><span class="n">resolve_template_files</span><span class="p">()</span></div>
<div class="viewcode-block" id="DAG.get_template_env"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.get_template_env">[docs]</a> <span class="k">def</span> <span class="nf">get_template_env</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a jinja2 Environment while taking into account the DAGs</span>
<span class="sd"> template_searchpath, user_defined_macros and user_defined_filters</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">searchpath</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">folder</span><span class="p">]</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">template_searchpath</span><span class="p">:</span>
<span class="n">searchpath</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">template_searchpath</span>
<span class="n">env</span> <span class="o">=</span> <span class="n">jinja2</span><span class="o">.</span><span class="n">Environment</span><span class="p">(</span>
<span class="n">loader</span><span class="o">=</span><span class="n">jinja2</span><span class="o">.</span><span class="n">FileSystemLoader</span><span class="p">(</span><span class="n">searchpath</span><span class="p">),</span>
<span class="n">extensions</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;jinja2.ext.do&quot;</span><span class="p">],</span>
<span class="n">cache_size</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">user_defined_macros</span><span class="p">:</span>
<span class="n">env</span><span class="o">.</span><span class="n">globals</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">user_defined_macros</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">user_defined_filters</span><span class="p">:</span>
<span class="n">env</span><span class="o">.</span><span class="n">filters</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">user_defined_filters</span><span class="p">)</span>
<span class="k">return</span> <span class="n">env</span></div>
<div class="viewcode-block" id="DAG.set_dependency"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.set_dependency">[docs]</a> <span class="k">def</span> <span class="nf">set_dependency</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">upstream_task_id</span><span class="p">,</span> <span class="n">downstream_task_id</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Simple utility method to set dependency between two tasks that</span>
<span class="sd"> already have been added to the DAG using add_task()</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">upstream_task_id</span><span class="p">)</span><span class="o">.</span><span class="n">set_downstream</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">downstream_task_id</span><span class="p">))</span></div>
<div class="viewcode-block" id="DAG.get_task_instances"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.get_task_instances">[docs]</a> <span class="k">def</span> <span class="nf">get_task_instances</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">,</span> <span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">state</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">start_date</span><span class="p">:</span>
<span class="n">start_date</span> <span class="o">=</span> <span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="mi">30</span><span class="p">))</span><span class="o">.</span><span class="n">date</span><span class="p">()</span>
<span class="n">start_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">make_aware</span><span class="p">(</span>
<span class="n">datetime</span><span class="o">.</span><span class="n">combine</span><span class="p">(</span><span class="n">start_date</span><span class="p">,</span> <span class="n">datetime</span><span class="o">.</span><span class="n">min</span><span class="o">.</span><span class="n">time</span><span class="p">()))</span>
<span class="n">end_date</span> <span class="o">=</span> <span class="n">end_date</span> <span class="ow">or</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&gt;=</span> <span class="n">start_date</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&lt;=</span> <span class="n">end_date</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">in_</span><span class="p">([</span><span class="n">t</span><span class="o">.</span><span class="n">task_id</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">]),</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">state</span><span class="p">:</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">state</span><span class="p">)</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
<span class="k">return</span> <span class="n">tis</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DAG.roots"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.roots">[docs]</a> <span class="k">def</span> <span class="nf">roots</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">[</span><span class="n">t</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">t</span><span class="o">.</span><span class="n">downstream_list</span><span class="p">]</span></div>
<div class="viewcode-block" id="DAG.topological_sort"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.topological_sort">[docs]</a> <span class="k">def</span> <span class="nf">topological_sort</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Sorts tasks in topographical order, such that a task comes after any of its</span>
<span class="sd"> upstream dependencies.</span>
<span class="sd"> Heavily inspired by:</span>
<span class="sd"> http://blog.jupo.org/2012/04/06/topological-sorting-acyclic-directed-graphs/</span>
<span class="sd"> :return: list of tasks in topological order</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># convert into an OrderedDict to speedup lookup while keeping order the same</span>
<span class="n">graph_unsorted</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">((</span><span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">task</span><span class="p">)</span> <span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">)</span>
<span class="n">graph_sorted</span> <span class="o">=</span> <span class="p">[]</span>
<span class="c1"># special case</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">graph_sorted</span><span class="p">)</span>
<span class="c1"># Run until the unsorted graph is empty.</span>
<span class="k">while</span> <span class="n">graph_unsorted</span><span class="p">:</span>
<span class="c1"># Go through each of the node/edges pairs in the unsorted</span>
<span class="c1"># graph. If a set of edges doesn&#39;t contain any nodes that</span>
<span class="c1"># haven&#39;t been resolved, that is, that are still in the</span>
<span class="c1"># unsorted graph, remove the pair from the unsorted graph,</span>
<span class="c1"># and append it to the sorted graph. Note here that by using</span>
<span class="c1"># using the items() method for iterating, a copy of the</span>
<span class="c1"># unsorted graph is used, allowing us to modify the unsorted</span>
<span class="c1"># graph as we move through it. We also keep a flag for</span>
<span class="c1"># checking that that graph is acyclic, which is true if any</span>
<span class="c1"># nodes are resolved during each pass through the graph. If</span>
<span class="c1"># not, we need to bail out as the graph therefore can&#39;t be</span>
<span class="c1"># sorted.</span>
<span class="n">acyclic</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">for</span> <span class="n">node</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="n">graph_unsorted</span><span class="o">.</span><span class="n">values</span><span class="p">()):</span>
<span class="k">for</span> <span class="n">edge</span> <span class="ow">in</span> <span class="n">node</span><span class="o">.</span><span class="n">upstream_list</span><span class="p">:</span>
<span class="k">if</span> <span class="n">edge</span><span class="o">.</span><span class="n">task_id</span> <span class="ow">in</span> <span class="n">graph_unsorted</span><span class="p">:</span>
<span class="k">break</span>
<span class="c1"># no edges in upstream tasks</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">acyclic</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">del</span> <span class="n">graph_unsorted</span><span class="p">[</span><span class="n">node</span><span class="o">.</span><span class="n">task_id</span><span class="p">]</span>
<span class="n">graph_sorted</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">node</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">acyclic</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;A cyclic dependency occurred in dag: </span><span class="si">{}</span><span class="s2">&quot;</span>
<span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">))</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">graph_sorted</span><span class="p">)</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.set_dag_runs_state"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.set_dag_runs_state">[docs]</a> <span class="k">def</span> <span class="nf">set_dag_runs_state</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">state</span><span class="o">=</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="p">):</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">start_date</span><span class="p">:</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&gt;=</span> <span class="n">start_date</span><span class="p">)</span>
<span class="k">if</span> <span class="n">end_date</span><span class="p">:</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&lt;=</span> <span class="n">end_date</span><span class="p">)</span>
<span class="n">drs</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
<span class="n">dirty_ids</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">dr</span> <span class="ow">in</span> <span class="n">drs</span><span class="p">:</span>
<span class="n">dr</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">state</span>
<span class="n">dirty_ids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dr</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.clear"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.clear">[docs]</a> <span class="k">def</span> <span class="nf">clear</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span> <span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">only_failed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">only_running</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">confirm_prompt</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">include_subdags</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">include_parentdag</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">reset_dag_runs</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">dry_run</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">get_tis</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Clears a set of task instances associated with the current dag for</span>
<span class="sd"> a specified date range.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span>
<span class="k">if</span> <span class="n">include_subdags</span><span class="p">:</span>
<span class="c1"># Crafting the right filter for dag_id and task_ids combo</span>
<span class="n">conditions</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">subdags</span> <span class="o">+</span> <span class="p">[</span><span class="bp">self</span><span class="p">]:</span>
<span class="n">conditions</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">like</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span> <span class="o">&amp;</span>
<span class="n">TI</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">task_ids</span><span class="p">)</span>
<span class="p">)</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">or_</span><span class="p">(</span><span class="o">*</span><span class="n">conditions</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_ids</span><span class="p">))</span>
<span class="k">if</span> <span class="n">include_parentdag</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_subdag</span><span class="p">:</span>
<span class="n">p_dag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parent_dag</span><span class="o">.</span><span class="n">sub_dag</span><span class="p">(</span>
<span class="n">task_regex</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">)[</span><span class="mi">1</span><span class="p">],</span>
<span class="n">include_upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">include_downstream</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">union</span><span class="p">(</span><span class="n">p_dag</span><span class="o">.</span><span class="n">clear</span><span class="p">(</span>
<span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span>
<span class="n">only_failed</span><span class="o">=</span><span class="n">only_failed</span><span class="p">,</span>
<span class="n">only_running</span><span class="o">=</span><span class="n">only_running</span><span class="p">,</span>
<span class="n">confirm_prompt</span><span class="o">=</span><span class="n">confirm_prompt</span><span class="p">,</span>
<span class="n">include_subdags</span><span class="o">=</span><span class="n">include_subdags</span><span class="p">,</span>
<span class="n">include_parentdag</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">reset_dag_runs</span><span class="o">=</span><span class="n">reset_dag_runs</span><span class="p">,</span>
<span class="n">get_tis</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span>
<span class="p">))</span>
<span class="k">if</span> <span class="n">start_date</span><span class="p">:</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&gt;=</span> <span class="n">start_date</span><span class="p">)</span>
<span class="k">if</span> <span class="n">end_date</span><span class="p">:</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&lt;=</span> <span class="n">end_date</span><span class="p">)</span>
<span class="k">if</span> <span class="n">only_failed</span><span class="p">:</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">or_</span><span class="p">(</span>
<span class="n">TI</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">UPSTREAM_FAILED</span><span class="p">))</span>
<span class="k">if</span> <span class="n">only_running</span><span class="p">:</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">)</span>
<span class="k">if</span> <span class="n">get_tis</span><span class="p">:</span>
<span class="k">return</span> <span class="n">tis</span>
<span class="k">if</span> <span class="n">dry_run</span><span class="p">:</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
<span class="n">session</span><span class="o">.</span><span class="n">expunge_all</span><span class="p">()</span>
<span class="k">return</span> <span class="n">tis</span>
<span class="n">count</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">count</span><span class="p">()</span>
<span class="n">do_it</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">if</span> <span class="n">count</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">return</span> <span class="mi">0</span>
<span class="k">if</span> <span class="n">confirm_prompt</span><span class="p">:</span>
<span class="n">ti_list</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">tis</span><span class="p">])</span>
<span class="n">question</span> <span class="o">=</span> <span class="p">(</span>
<span class="s2">&quot;You are about to delete these </span><span class="si">{count}</span><span class="s2"> tasks:</span><span class="se">\n</span><span class="s2">&quot;</span>
<span class="s2">&quot;</span><span class="si">{ti_list}</span><span class="se">\n\n</span><span class="s2">&quot;</span>
<span class="s2">&quot;Are you sure? (yes/no): &quot;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
<span class="n">do_it</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">helpers</span><span class="o">.</span><span class="n">ask_yesno</span><span class="p">(</span><span class="n">question</span><span class="p">)</span>
<span class="k">if</span> <span class="n">do_it</span><span class="p">:</span>
<span class="n">clear_task_instances</span><span class="p">(</span><span class="n">tis</span><span class="o">.</span><span class="n">all</span><span class="p">(),</span>
<span class="n">session</span><span class="p">,</span>
<span class="n">dag</span><span class="o">=</span><span class="bp">self</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">reset_dag_runs</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">set_dag_runs_state</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span>
<span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span>
<span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">count</span> <span class="o">=</span> <span class="mi">0</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Bail. Nothing was cleared.&quot;</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="k">return</span> <span class="n">count</span></div>
<span class="nd">@classmethod</span>
<div class="viewcode-block" id="DAG.clear_dags"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.clear_dags">[docs]</a> <span class="k">def</span> <span class="nf">clear_dags</span><span class="p">(</span>
<span class="bp">cls</span><span class="p">,</span> <span class="n">dags</span><span class="p">,</span>
<span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">only_failed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">only_running</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">confirm_prompt</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">include_subdags</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">include_parentdag</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">reset_dag_runs</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">dry_run</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="p">):</span>
<span class="n">all_tis</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">dags</span><span class="p">:</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">clear</span><span class="p">(</span>
<span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span>
<span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span>
<span class="n">only_failed</span><span class="o">=</span><span class="n">only_failed</span><span class="p">,</span>
<span class="n">only_running</span><span class="o">=</span><span class="n">only_running</span><span class="p">,</span>
<span class="n">confirm_prompt</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">include_subdags</span><span class="o">=</span><span class="n">include_subdags</span><span class="p">,</span>
<span class="n">include_parentdag</span><span class="o">=</span><span class="n">include_parentdag</span><span class="p">,</span>
<span class="n">reset_dag_runs</span><span class="o">=</span><span class="n">reset_dag_runs</span><span class="p">,</span>
<span class="n">dry_run</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">all_tis</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">tis</span><span class="p">)</span>
<span class="k">if</span> <span class="n">dry_run</span><span class="p">:</span>
<span class="k">return</span> <span class="n">all_tis</span>
<span class="n">count</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">all_tis</span><span class="p">)</span>
<span class="n">do_it</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">if</span> <span class="n">count</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Nothing to clear.&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="mi">0</span>
<span class="k">if</span> <span class="n">confirm_prompt</span><span class="p">:</span>
<span class="n">ti_list</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">all_tis</span><span class="p">])</span>
<span class="n">question</span> <span class="o">=</span> <span class="p">(</span>
<span class="s2">&quot;You are about to delete these </span><span class="si">{}</span><span class="s2"> tasks:</span><span class="se">\n</span><span class="s2">&quot;</span>
<span class="s2">&quot;</span><span class="si">{}</span><span class="se">\n\n</span><span class="s2">&quot;</span>
<span class="s2">&quot;Are you sure? (yes/no): &quot;</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">count</span><span class="p">,</span> <span class="n">ti_list</span><span class="p">)</span>
<span class="n">do_it</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">helpers</span><span class="o">.</span><span class="n">ask_yesno</span><span class="p">(</span><span class="n">question</span><span class="p">)</span>
<span class="k">if</span> <span class="n">do_it</span><span class="p">:</span>
<span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">dags</span><span class="p">:</span>
<span class="n">dag</span><span class="o">.</span><span class="n">clear</span><span class="p">(</span><span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span>
<span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span>
<span class="n">only_failed</span><span class="o">=</span><span class="n">only_failed</span><span class="p">,</span>
<span class="n">only_running</span><span class="o">=</span><span class="n">only_running</span><span class="p">,</span>
<span class="n">confirm_prompt</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">include_subdags</span><span class="o">=</span><span class="n">include_subdags</span><span class="p">,</span>
<span class="n">reset_dag_runs</span><span class="o">=</span><span class="n">reset_dag_runs</span><span class="p">,</span>
<span class="n">dry_run</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">count</span> <span class="o">=</span> <span class="mi">0</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Bail. Nothing was cleared.&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">count</span></div>
<div class="viewcode-block" id="DAG.__deepcopy__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.__deepcopy__">[docs]</a> <span class="k">def</span> <span class="nf">__deepcopy__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">memo</span><span class="p">):</span>
<span class="c1"># Swiwtcharoo to go around deepcopying objects coming through the</span>
<span class="c1"># backdoor</span>
<span class="bp">cls</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span>
<span class="n">result</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span>
<span class="n">memo</span><span class="p">[</span><span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="p">)]</span> <span class="o">=</span> <span class="n">result</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">()):</span>
<span class="k">if</span> <span class="n">k</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;user_defined_macros&#39;</span><span class="p">,</span> <span class="s1">&#39;user_defined_filters&#39;</span><span class="p">,</span> <span class="s1">&#39;params&#39;</span><span class="p">):</span>
<span class="nb">setattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">memo</span><span class="p">))</span>
<span class="n">result</span><span class="o">.</span><span class="n">user_defined_macros</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">user_defined_macros</span>
<span class="n">result</span><span class="o">.</span><span class="n">user_defined_filters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">user_defined_filters</span>
<span class="n">result</span><span class="o">.</span><span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">params</span>
<span class="k">return</span> <span class="n">result</span></div>
<div class="viewcode-block" id="DAG.sub_dag"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.sub_dag">[docs]</a> <span class="k">def</span> <span class="nf">sub_dag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_regex</span><span class="p">,</span> <span class="n">include_downstream</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">include_upstream</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a subset of the current dag as a deep copy of the current dag</span>
<span class="sd"> based on a regex that should match one or many tasks, and includes</span>
<span class="sd"> upstream and downstream neighbours based on the flag passed.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">dag</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">regex_match</span> <span class="o">=</span> <span class="p">[</span>
<span class="n">t</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">tasks</span> <span class="k">if</span> <span class="n">re</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">task_regex</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">task_id</span><span class="p">)]</span>
<span class="n">also_include</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">regex_match</span><span class="p">:</span>
<span class="k">if</span> <span class="n">include_downstream</span><span class="p">:</span>
<span class="n">also_include</span> <span class="o">+=</span> <span class="n">t</span><span class="o">.</span><span class="n">get_flat_relatives</span><span class="p">(</span><span class="n">upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="k">if</span> <span class="n">include_upstream</span><span class="p">:</span>
<span class="n">also_include</span> <span class="o">+=</span> <span class="n">t</span><span class="o">.</span><span class="n">get_flat_relatives</span><span class="p">(</span><span class="n">upstream</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="c1"># Compiling the unique list of tasks that made the cut</span>
<span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">t</span><span class="o">.</span><span class="n">task_id</span><span class="p">:</span> <span class="n">t</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">regex_match</span> <span class="o">+</span> <span class="n">also_include</span><span class="p">}</span>
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">tasks</span><span class="p">:</span>
<span class="c1"># Removing upstream/downstream references to tasks that did not</span>
<span class="c1"># made the cut</span>
<span class="n">t</span><span class="o">.</span><span class="n">_upstream_task_ids</span> <span class="o">=</span> <span class="n">t</span><span class="o">.</span><span class="n">_upstream_task_ids</span><span class="o">.</span><span class="n">intersection</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
<span class="n">t</span><span class="o">.</span><span class="n">_downstream_task_ids</span> <span class="o">=</span> <span class="n">t</span><span class="o">.</span><span class="n">_downstream_task_ids</span><span class="o">.</span><span class="n">intersection</span><span class="p">(</span>
<span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">tasks</span><span class="p">)</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">):</span>
<span class="n">dag</span><span class="o">.</span><span class="n">partial</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">return</span> <span class="n">dag</span></div>
<div class="viewcode-block" id="DAG.has_task"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.has_task">[docs]</a> <span class="k">def</span> <span class="nf">has_task</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">):</span>
<span class="k">return</span> <span class="n">task_id</span> <span class="ow">in</span> <span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">task_id</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">)</span></div>
<div class="viewcode-block" id="DAG.get_task"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.get_task">[docs]</a> <span class="k">def</span> <span class="nf">get_task</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">):</span>
<span class="k">if</span> <span class="n">task_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">[</span><span class="n">task_id</span><span class="p">]</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;Task </span><span class="si">{task_id}</span><span class="s2"> not found&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">()))</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.pickle_info"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.pickle_info">[docs]</a> <span class="k">def</span> <span class="nf">pickle_info</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">d</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">d</span><span class="p">[</span><span class="s1">&#39;is_picklable&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">dttm</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="n">pickled</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">d</span><span class="p">[</span><span class="s1">&#39;pickle_len&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">pickled</span><span class="p">)</span>
<span class="n">d</span><span class="p">[</span><span class="s1">&#39;pickling_duration&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">dttm</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
<span class="n">d</span><span class="p">[</span><span class="s1">&#39;is_picklable&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">d</span><span class="p">[</span><span class="s1">&#39;stacktrace&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">traceback</span><span class="o">.</span><span class="n">format_exc</span><span class="p">()</span>
<span class="k">return</span> <span class="n">d</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.pickle"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.pickle">[docs]</a> <span class="k">def</span> <span class="nf">pickle</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">dag</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
<span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="n">dp</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">dag</span> <span class="ow">and</span> <span class="n">dag</span><span class="o">.</span><span class="n">pickle_id</span><span class="p">:</span>
<span class="n">dp</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagPickle</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">DagPickle</span><span class="o">.</span><span class="n">id</span> <span class="o">==</span> <span class="n">dag</span><span class="o">.</span><span class="n">pickle_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">dp</span> <span class="ow">or</span> <span class="n">dp</span><span class="o">.</span><span class="n">pickle</span> <span class="o">!=</span> <span class="bp">self</span><span class="p">:</span>
<span class="n">dp</span> <span class="o">=</span> <span class="n">DagPickle</span><span class="p">(</span><span class="n">dag</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">dp</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">last_pickled</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pickle_id</span> <span class="o">=</span> <span class="n">dp</span><span class="o">.</span><span class="n">id</span>
<span class="k">return</span> <span class="n">dp</span></div>
<div class="viewcode-block" id="DAG.tree_view"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.tree_view">[docs]</a> <span class="k">def</span> <span class="nf">tree_view</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Shows an ascii tree representation of the DAG</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span> <span class="nf">get_downstream</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
<span class="nb">print</span><span class="p">((</span><span class="s2">&quot; &quot;</span> <span class="o">*</span> <span class="n">level</span> <span class="o">*</span> <span class="mi">4</span><span class="p">)</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">task</span><span class="p">))</span>
<span class="n">level</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">task</span><span class="o">.</span><span class="n">upstream_list</span><span class="p">:</span>
<span class="n">get_downstream</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">level</span><span class="p">)</span>
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">roots</span><span class="p">:</span>
<span class="n">get_downstream</span><span class="p">(</span><span class="n">t</span><span class="p">)</span></div>
<div class="viewcode-block" id="DAG.add_task"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.add_task">[docs]</a> <span class="k">def</span> <span class="nf">add_task</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Add a task to the DAG</span>
<span class="sd"> :param task: the task you want to add</span>
<span class="sd"> :type task: task</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">task</span><span class="o">.</span><span class="n">start_date</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;Task is missing the start_date parameter&quot;</span><span class="p">)</span>
<span class="c1"># if the task has no start date, assign it the same as the DAG</span>
<span class="k">elif</span> <span class="ow">not</span> <span class="n">task</span><span class="o">.</span><span class="n">start_date</span><span class="p">:</span>
<span class="n">task</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span>
<span class="c1"># otherwise, the task will start on the later of its own start date and</span>
<span class="c1"># the DAG&#39;s start date</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span><span class="p">:</span>
<span class="n">task</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">start_date</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span><span class="p">)</span>
<span class="c1"># if the task has no end date, assign it the same as the dag</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">task</span><span class="o">.</span><span class="n">end_date</span><span class="p">:</span>
<span class="n">task</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span>
<span class="c1"># otherwise, the task will end on the earlier of its own end date and</span>
<span class="c1"># the DAG&#39;s end date</span>
<span class="k">elif</span> <span class="n">task</span><span class="o">.</span><span class="n">end_date</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span><span class="p">:</span>
<span class="n">task</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">end_date</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span><span class="p">)</span>
<span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">:</span>
<span class="c1"># TODO: raise an error in Airflow 2.0</span>
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
<span class="s1">&#39;The requested task could not be added to the DAG because a &#39;</span>
<span class="s1">&#39;task with task_id </span><span class="si">{}</span><span class="s1"> is already in the DAG. Starting in &#39;</span>
<span class="s1">&#39;Airflow 2.0, trying to overwrite a task will raise an &#39;</span>
<span class="s1">&#39;exception.&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">),</span>
<span class="n">category</span><span class="o">=</span><span class="ne">PendingDeprecationWarning</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">[</span><span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">task</span>
<span class="n">task</span><span class="o">.</span><span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_count</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">)</span></div>
<div class="viewcode-block" id="DAG.add_tasks"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.add_tasks">[docs]</a> <span class="k">def</span> <span class="nf">add_tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tasks</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Add a list of tasks to the DAG</span>
<span class="sd"> :param tasks: a lit of tasks you want to add</span>
<span class="sd"> :type tasks: list of tasks</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">tasks</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">add_task</span><span class="p">(</span><span class="n">task</span><span class="p">)</span></div>
<div class="viewcode-block" id="DAG.run"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.run">[docs]</a> <span class="k">def</span> <span class="nf">run</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">mark_success</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">local</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">executor</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">donot_pickle</span><span class="o">=</span><span class="n">configuration</span><span class="o">.</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;donot_pickle&#39;</span><span class="p">),</span>
<span class="n">ignore_task_deps</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">ignore_first_depends_on_past</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">delay_on_limit_secs</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">conf</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">rerun_failed_tasks</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">run_backwards</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Runs the DAG.</span>
<span class="sd"> :param start_date: the start date of the range to run</span>
<span class="sd"> :type start_date: datetime.datetime</span>
<span class="sd"> :param end_date: the end date of the range to run</span>
<span class="sd"> :type end_date: datetime.datetime</span>
<span class="sd"> :param mark_success: True to mark jobs as succeeded without running them</span>
<span class="sd"> :type mark_success: bool</span>
<span class="sd"> :param local: True to run the tasks using the LocalExecutor</span>
<span class="sd"> :type local: bool</span>
<span class="sd"> :param executor: The executor instance to run the tasks</span>
<span class="sd"> :type executor: airflow.executor.BaseExecutor</span>
<span class="sd"> :param donot_pickle: True to avoid pickling DAG object and send to workers</span>
<span class="sd"> :type donot_pickle: bool</span>
<span class="sd"> :param ignore_task_deps: True to skip upstream tasks</span>
<span class="sd"> :type ignore_task_deps: bool</span>
<span class="sd"> :param ignore_first_depends_on_past: True to ignore depends_on_past</span>
<span class="sd"> dependencies for the first set of tasks only</span>
<span class="sd"> :type ignore_first_depends_on_past: bool</span>
<span class="sd"> :param pool: Resource pool to use</span>
<span class="sd"> :type pool: str</span>
<span class="sd"> :param delay_on_limit_secs: Time in seconds to wait before next attempt to run</span>
<span class="sd"> dag run when max_active_runs limit has been reached</span>
<span class="sd"> :type delay_on_limit_secs: float</span>
<span class="sd"> :param verbose: Make logging output more verbose</span>
<span class="sd"> :type verbose: bool</span>
<span class="sd"> :param conf: user defined dictionary passed from CLI</span>
<span class="sd"> :type conf: dict</span>
<span class="sd"> :param rerun_failed_tasks:</span>
<span class="sd"> :type: bool</span>
<span class="sd"> :param run_backwards:</span>
<span class="sd"> :type: bool</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="kn">from</span> <span class="nn">airflow.jobs</span> <span class="k">import</span> <span class="n">BackfillJob</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">executor</span> <span class="ow">and</span> <span class="n">local</span><span class="p">:</span>
<span class="n">executor</span> <span class="o">=</span> <span class="n">LocalExecutor</span><span class="p">()</span>
<span class="k">elif</span> <span class="ow">not</span> <span class="n">executor</span><span class="p">:</span>
<span class="n">executor</span> <span class="o">=</span> <span class="n">GetDefaultExecutor</span><span class="p">()</span>
<span class="n">job</span> <span class="o">=</span> <span class="n">BackfillJob</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span>
<span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span>
<span class="n">mark_success</span><span class="o">=</span><span class="n">mark_success</span><span class="p">,</span>
<span class="n">executor</span><span class="o">=</span><span class="n">executor</span><span class="p">,</span>
<span class="n">donot_pickle</span><span class="o">=</span><span class="n">donot_pickle</span><span class="p">,</span>
<span class="n">ignore_task_deps</span><span class="o">=</span><span class="n">ignore_task_deps</span><span class="p">,</span>
<span class="n">ignore_first_depends_on_past</span><span class="o">=</span><span class="n">ignore_first_depends_on_past</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="n">pool</span><span class="p">,</span>
<span class="n">delay_on_limit_secs</span><span class="o">=</span><span class="n">delay_on_limit_secs</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
<span class="n">conf</span><span class="o">=</span><span class="n">conf</span><span class="p">,</span>
<span class="n">rerun_failed_tasks</span><span class="o">=</span><span class="n">rerun_failed_tasks</span><span class="p">,</span>
<span class="n">run_backwards</span><span class="o">=</span><span class="n">run_backwards</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">job</span><span class="o">.</span><span class="n">run</span><span class="p">()</span></div>
<div class="viewcode-block" id="DAG.cli"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.cli">[docs]</a> <span class="k">def</span> <span class="nf">cli</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Exposes a CLI specific to this DAG</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="kn">from</span> <span class="nn">airflow.bin</span> <span class="k">import</span> <span class="n">cli</span>
<span class="n">parser</span> <span class="o">=</span> <span class="n">cli</span><span class="o">.</span><span class="n">CLIFactory</span><span class="o">.</span><span class="n">get_parser</span><span class="p">(</span><span class="n">dag_parser</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">args</span> <span class="o">=</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse_args</span><span class="p">()</span>
<span class="n">args</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.create_dagrun"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.create_dagrun">[docs]</a> <span class="k">def</span> <span class="nf">create_dagrun</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
<span class="n">run_id</span><span class="p">,</span>
<span class="n">state</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">external_trigger</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">conf</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Creates a dag run from this dag including the tasks associated with this dag.</span>
<span class="sd"> Returns the dag run.</span>
<span class="sd"> :param run_id: defines the the run id for this dag run</span>
<span class="sd"> :type run_id: str</span>
<span class="sd"> :param execution_date: the execution date of this dag run</span>
<span class="sd"> :type execution_date: datetime.datetime</span>
<span class="sd"> :param state: the state of the dag run</span>
<span class="sd"> :type state: airflow.utils.state.State</span>
<span class="sd"> :param start_date: the date this dag run should be evaluated</span>
<span class="sd"> :type start_date: datetime</span>
<span class="sd"> :param external_trigger: whether this dag run is externally triggered</span>
<span class="sd"> :type external_trigger: bool</span>
<span class="sd"> :param session: database session</span>
<span class="sd"> :type session: sqlalchemy.orm.session.Session</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">run</span> <span class="o">=</span> <span class="n">DagRun</span><span class="p">(</span>
<span class="n">dag_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">run_id</span><span class="o">=</span><span class="n">run_id</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="n">execution_date</span><span class="p">,</span>
<span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span>
<span class="n">external_trigger</span><span class="o">=</span><span class="n">external_trigger</span><span class="p">,</span>
<span class="n">conf</span><span class="o">=</span><span class="n">conf</span><span class="p">,</span>
<span class="n">state</span><span class="o">=</span><span class="n">state</span>
<span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">run</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="n">run</span><span class="o">.</span><span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span>
<span class="c1"># create the associated task instances</span>
<span class="c1"># state is None at the moment of creation</span>
<span class="n">run</span><span class="o">.</span><span class="n">verify_integrity</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="n">run</span><span class="o">.</span><span class="n">refresh_from_db</span><span class="p">()</span>
<span class="k">return</span> <span class="n">run</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.sync_to_db"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.sync_to_db">[docs]</a> <span class="k">def</span> <span class="nf">sync_to_db</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">owner</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sync_time</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Save attributes about this DAG to the DB. Note that this method</span>
<span class="sd"> can be called for both DAGs and SubDAGs. A SubDag is actually a</span>
<span class="sd"> SubDagOperator.</span>
<span class="sd"> :param dag: the DAG object to save to the DB</span>
<span class="sd"> :type dag: airflow.models.DAG</span>
<span class="sd"> :param sync_time: The time that the DAG should be marked as sync&#39;ed</span>
<span class="sd"> :type sync_time: datetime</span>
<span class="sd"> :return: None</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">owner</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">owner</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">owner</span>
<span class="k">if</span> <span class="n">sync_time</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">sync_time</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="n">orm_dag</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
<span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">orm_dag</span><span class="p">:</span>
<span class="n">orm_dag</span> <span class="o">=</span> <span class="n">DagModel</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Creating ORM DAG for </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span>
<span class="n">orm_dag</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fileloc</span>
<span class="n">orm_dag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_subdag</span>
<span class="n">orm_dag</span><span class="o">.</span><span class="n">owners</span> <span class="o">=</span> <span class="n">owner</span>
<span class="n">orm_dag</span><span class="o">.</span><span class="n">is_active</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">orm_dag</span><span class="o">.</span><span class="n">last_scheduler_run</span> <span class="o">=</span> <span class="n">sync_time</span>
<span class="n">orm_dag</span><span class="o">.</span><span class="n">default_view</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_default_view</span>
<span class="n">orm_dag</span><span class="o">.</span><span class="n">description</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">description</span>
<span class="n">orm_dag</span><span class="o">.</span><span class="n">schedule_interval</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">schedule_interval</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">orm_dag</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="k">for</span> <span class="n">subdag</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">subdags</span><span class="p">:</span>
<span class="n">subdag</span><span class="o">.</span><span class="n">sync_to_db</span><span class="p">(</span><span class="n">owner</span><span class="o">=</span><span class="n">owner</span><span class="p">,</span> <span class="n">sync_time</span><span class="o">=</span><span class="n">sync_time</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span></div>
<span class="nd">@staticmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.deactivate_unknown_dags"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.deactivate_unknown_dags">[docs]</a> <span class="k">def</span> <span class="nf">deactivate_unknown_dags</span><span class="p">(</span><span class="n">active_dag_ids</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Given a list of known DAGs, deactivate any other DAGs that are</span>
<span class="sd"> marked as active in the ORM</span>
<span class="sd"> :param active_dag_ids: list of DAG IDs that are active</span>
<span class="sd"> :type active_dag_ids: list[unicode]</span>
<span class="sd"> :return: None</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">active_dag_ids</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">return</span>
<span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
<span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="o">~</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">active_dag_ids</span><span class="p">))</span><span class="o">.</span><span class="n">all</span><span class="p">():</span>
<span class="n">dag</span><span class="o">.</span><span class="n">is_active</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">dag</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
<span class="nd">@staticmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.deactivate_stale_dags"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.deactivate_stale_dags">[docs]</a> <span class="k">def</span> <span class="nf">deactivate_stale_dags</span><span class="p">(</span><span class="n">expiration_date</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Deactivate any DAGs that were last touched by the scheduler before</span>
<span class="sd"> the expiration date. These DAGs were likely deleted.</span>
<span class="sd"> :param expiration_date: set inactive DAGs that were touched before this</span>
<span class="sd"> time</span>
<span class="sd"> :type expiration_date: datetime</span>
<span class="sd"> :return: None</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
<span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span>
<span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">last_scheduler_run</span> <span class="o">&lt;</span> <span class="n">expiration_date</span><span class="p">,</span>
<span class="n">DagModel</span><span class="o">.</span><span class="n">is_active</span><span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">():</span>
<span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
<span class="s2">&quot;Deactivating DAG ID </span><span class="si">%s</span><span class="s2"> since it was last touched by the scheduler at </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span>
<span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">dag</span><span class="o">.</span><span class="n">last_scheduler_run</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
<span class="p">)</span>
<span class="n">dag</span><span class="o">.</span><span class="n">is_active</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">dag</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
<span class="nd">@staticmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DAG.get_num_task_instances"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.get_num_task_instances">[docs]</a> <span class="k">def</span> <span class="nf">get_num_task_instances</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">task_ids</span><span class="p">,</span> <span class="n">states</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the number of task instances in the given DAG.</span>
<span class="sd"> :param session: ORM session</span>
<span class="sd"> :param dag_id: ID of the DAG to get the task concurrency of</span>
<span class="sd"> :type dag_id: unicode</span>
<span class="sd"> :param task_ids: A list of valid task IDs for the given DAG</span>
<span class="sd"> :type task_ids: list[unicode]</span>
<span class="sd"> :param states: A list of states to filter by if supplied</span>
<span class="sd"> :type states: list[state]</span>
<span class="sd"> :return: The number of running tasks</span>
<span class="sd"> :rtype: int</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">func</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">task_id</span><span class="p">))</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">TaskInstance</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">,</span>
<span class="n">TaskInstance</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">task_ids</span><span class="p">))</span>
<span class="k">if</span> <span class="n">states</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="kc">None</span> <span class="ow">in</span> <span class="n">states</span><span class="p">:</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">or_</span><span class="p">(</span>
<span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">states</span><span class="p">),</span>
<span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">is_</span><span class="p">(</span><span class="kc">None</span><span class="p">)))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">states</span><span class="p">))</span>
<span class="k">return</span> <span class="n">qry</span><span class="o">.</span><span class="n">scalar</span><span class="p">()</span></div>
<div class="viewcode-block" id="DAG.test_cycle"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG.test_cycle">[docs]</a> <span class="k">def</span> <span class="nf">test_cycle</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Check to see if there are any cycles in the DAG. Returns False if no cycle found,</span>
<span class="sd"> otherwise raises exception.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># default of int is 0 which corresponds to CYCLE_NEW</span>
<span class="n">visit_map</span> <span class="o">=</span> <span class="n">defaultdict</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span>
<span class="k">for</span> <span class="n">task_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
<span class="c1"># print(&#39;starting %s&#39; % task_id)</span>
<span class="k">if</span> <span class="n">visit_map</span><span class="p">[</span><span class="n">task_id</span><span class="p">]</span> <span class="o">==</span> <span class="n">DagBag</span><span class="o">.</span><span class="n">CYCLE_NEW</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_test_cycle_helper</span><span class="p">(</span><span class="n">visit_map</span><span class="p">,</span> <span class="n">task_id</span><span class="p">)</span>
<span class="k">return</span> <span class="kc">False</span></div>
<div class="viewcode-block" id="DAG._test_cycle_helper"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DAG._test_cycle_helper">[docs]</a> <span class="k">def</span> <span class="nf">_test_cycle_helper</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">visit_map</span><span class="p">,</span> <span class="n">task_id</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Checks if a cycle exists from the input task using DFS traversal</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># print(&#39;Inspecting %s&#39; % task_id)</span>
<span class="k">if</span> <span class="n">visit_map</span><span class="p">[</span><span class="n">task_id</span><span class="p">]</span> <span class="o">==</span> <span class="n">DagBag</span><span class="o">.</span><span class="n">CYCLE_DONE</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>
<span class="n">visit_map</span><span class="p">[</span><span class="n">task_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">DagBag</span><span class="o">.</span><span class="n">CYCLE_IN_PROGRESS</span>
<span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">[</span><span class="n">task_id</span><span class="p">]</span>
<span class="k">for</span> <span class="n">descendant_id</span> <span class="ow">in</span> <span class="n">task</span><span class="o">.</span><span class="n">get_direct_relative_ids</span><span class="p">():</span>
<span class="k">if</span> <span class="n">visit_map</span><span class="p">[</span><span class="n">descendant_id</span><span class="p">]</span> <span class="o">==</span> <span class="n">DagBag</span><span class="o">.</span><span class="n">CYCLE_IN_PROGRESS</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="s2">&quot;Cycle detected in DAG. Faulty task: </span><span class="si">{0}</span><span class="s2"> to </span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">task_id</span><span class="p">,</span> <span class="n">descendant_id</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">AirflowDagCycleException</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_test_cycle_helper</span><span class="p">(</span><span class="n">visit_map</span><span class="p">,</span> <span class="n">descendant_id</span><span class="p">)</span>
<span class="n">visit_map</span><span class="p">[</span><span class="n">task_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">DagBag</span><span class="o">.</span><span class="n">CYCLE_DONE</span></div></div>
<div class="viewcode-block" id="Chart"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart">[docs]</a><span class="k">class</span> <span class="nc">Chart</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span>
<div class="viewcode-block" id="Chart.__tablename__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.__tablename__">[docs]</a> <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">&quot;chart&quot;</span></div>
<div class="viewcode-block" id="Chart.id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.id">[docs]</a> <span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.label"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.label">[docs]</a> <span class="n">label</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">200</span><span class="p">))</span></div>
<div class="viewcode-block" id="Chart.conn_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.conn_id">[docs]</a> <span class="n">conn_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.user_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.user_id">[docs]</a> <span class="n">user_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">(),</span> <span class="n">ForeignKey</span><span class="p">(</span><span class="s1">&#39;users.id&#39;</span><span class="p">),</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.chart_type"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.chart_type">[docs]</a> <span class="n">chart_type</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span> <span class="n">default</span><span class="o">=</span><span class="s2">&quot;line&quot;</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.sql_layout"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.sql_layout">[docs]</a> <span class="n">sql_layout</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">50</span><span class="p">),</span> <span class="n">default</span><span class="o">=</span><span class="s2">&quot;series&quot;</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.sql"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.sql">[docs]</a> <span class="n">sql</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Text</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s2">&quot;SELECT series, x, y FROM table&quot;</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.y_log_scale"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.y_log_scale">[docs]</a> <span class="n">y_log_scale</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.show_datatable"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.show_datatable">[docs]</a> <span class="n">show_datatable</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.show_sql"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.show_sql">[docs]</a> <span class="n">show_sql</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.height"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.height">[docs]</a> <span class="n">height</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="mi">600</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.default_params"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.default_params">[docs]</a> <span class="n">default_params</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">5000</span><span class="p">),</span> <span class="n">default</span><span class="o">=</span><span class="s2">&quot;</span><span class="si">{}</span><span class="s2">&quot;</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.owner"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.owner">[docs]</a> <span class="n">owner</span> <span class="o">=</span> <span class="n">relationship</span><span class="p">(</span>
<span class="s2">&quot;User&quot;</span><span class="p">,</span> <span class="n">cascade</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">cascade_backrefs</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">backref</span><span class="o">=</span><span class="s1">&#39;charts&#39;</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.x_is_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.x_is_date">[docs]</a> <span class="n">x_is_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.iteration_no"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.iteration_no">[docs]</a> <span class="n">iteration_no</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.last_modified"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.last_modified">[docs]</a> <span class="n">last_modified</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">)</span></div>
<div class="viewcode-block" id="Chart.__repr__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Chart.__repr__">[docs]</a> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">label</span></div></div>
<div class="viewcode-block" id="KnownEventType"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEventType">[docs]</a><span class="k">class</span> <span class="nc">KnownEventType</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span>
<div class="viewcode-block" id="KnownEventType.__tablename__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEventType.__tablename__">[docs]</a> <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">&quot;known_event_type&quot;</span></div>
<div class="viewcode-block" id="KnownEventType.id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEventType.id">[docs]</a> <span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="KnownEventType.know_event_type"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEventType.know_event_type">[docs]</a> <span class="n">know_event_type</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">200</span><span class="p">))</span></div>
<div class="viewcode-block" id="KnownEventType.__repr__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEventType.__repr__">[docs]</a> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">know_event_type</span></div></div>
<div class="viewcode-block" id="KnownEvent"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEvent">[docs]</a><span class="k">class</span> <span class="nc">KnownEvent</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span>
<div class="viewcode-block" id="KnownEvent.__tablename__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEvent.__tablename__">[docs]</a> <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">&quot;known_event&quot;</span></div>
<div class="viewcode-block" id="KnownEvent.id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEvent.id">[docs]</a> <span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="KnownEvent.label"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEvent.label">[docs]</a> <span class="n">label</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">200</span><span class="p">))</span></div>
<div class="viewcode-block" id="KnownEvent.start_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEvent.start_date">[docs]</a> <span class="n">start_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div>
<div class="viewcode-block" id="KnownEvent.end_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEvent.end_date">[docs]</a> <span class="n">end_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div>
<div class="viewcode-block" id="KnownEvent.user_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEvent.user_id">[docs]</a> <span class="n">user_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">(),</span> <span class="n">ForeignKey</span><span class="p">(</span><span class="s1">&#39;users.id&#39;</span><span class="p">),)</span></div>
<div class="viewcode-block" id="KnownEvent.known_event_type_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEvent.known_event_type_id">[docs]</a> <span class="n">known_event_type_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">(),</span> <span class="n">ForeignKey</span><span class="p">(</span><span class="s1">&#39;known_event_type.id&#39;</span><span class="p">),)</span></div>
<div class="viewcode-block" id="KnownEvent.reported_by"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEvent.reported_by">[docs]</a> <span class="n">reported_by</span> <span class="o">=</span> <span class="n">relationship</span><span class="p">(</span>
<span class="s2">&quot;User&quot;</span><span class="p">,</span> <span class="n">cascade</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">cascade_backrefs</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">backref</span><span class="o">=</span><span class="s1">&#39;known_events&#39;</span><span class="p">)</span></div>
<div class="viewcode-block" id="KnownEvent.event_type"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEvent.event_type">[docs]</a> <span class="n">event_type</span> <span class="o">=</span> <span class="n">relationship</span><span class="p">(</span>
<span class="s2">&quot;KnownEventType&quot;</span><span class="p">,</span>
<span class="n">cascade</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">cascade_backrefs</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">backref</span><span class="o">=</span><span class="s1">&#39;known_events&#39;</span><span class="p">)</span></div>
<div class="viewcode-block" id="KnownEvent.description"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEvent.description">[docs]</a> <span class="n">description</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Text</span><span class="p">)</span></div>
<div class="viewcode-block" id="KnownEvent.__repr__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.KnownEvent.__repr__">[docs]</a> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">label</span></div></div>
<div class="viewcode-block" id="Variable"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable">[docs]</a><span class="k">class</span> <span class="nc">Variable</span><span class="p">(</span><span class="n">Base</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
<div class="viewcode-block" id="Variable.__tablename__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.__tablename__">[docs]</a> <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">&quot;variable&quot;</span></div>
<div class="viewcode-block" id="Variable.__NO_DEFAULT_SENTINEL"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.__NO_DEFAULT_SENTINEL">[docs]</a> <span class="n">__NO_DEFAULT_SENTINEL</span> <span class="o">=</span> <span class="nb">object</span><span class="p">()</span></div>
<div class="viewcode-block" id="Variable.id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.id">[docs]</a> <span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="Variable.key"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.key">[docs]</a> <span class="n">key</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">unique</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="Variable._val"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable._val">[docs]</a> <span class="n">_val</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="s1">&#39;val&#39;</span><span class="p">,</span> <span class="n">Text</span><span class="p">)</span></div>
<div class="viewcode-block" id="Variable.is_encrypted"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.is_encrypted">[docs]</a> <span class="n">is_encrypted</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">unique</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<div class="viewcode-block" id="Variable.__repr__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.__repr__">[docs]</a> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="c1"># Hiding the value</span>
<span class="k">return</span> <span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> : </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">key</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_val</span><span class="p">)</span></div>
<div class="viewcode-block" id="Variable.get_val"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.get_val">[docs]</a> <span class="k">def</span> <span class="nf">get_val</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_val</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_encrypted</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">fernet</span> <span class="o">=</span> <span class="n">get_fernet</span><span class="p">()</span>
<span class="k">return</span> <span class="n">fernet</span><span class="o">.</span><span class="n">decrypt</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_val</span><span class="p">,</span> <span class="s1">&#39;utf-8&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
<span class="k">except</span> <span class="n">InvalidFernetToken</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Can&#39;t decrypt _val for key=</span><span class="si">{}</span><span class="s2">, invalid token &quot;</span>
<span class="s2">&quot;or value&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">key</span><span class="p">))</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Can&#39;t decrypt _val for key=</span><span class="si">{}</span><span class="s2">, FERNET_KEY &quot;</span>
<span class="s2">&quot;configuration missing&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">key</span><span class="p">))</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_val</span></div>
<div class="viewcode-block" id="Variable.set_val"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.set_val">[docs]</a> <span class="k">def</span> <span class="nf">set_val</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="k">if</span> <span class="n">value</span><span class="p">:</span>
<span class="n">fernet</span> <span class="o">=</span> <span class="n">get_fernet</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_val</span> <span class="o">=</span> <span class="n">fernet</span><span class="o">.</span><span class="n">encrypt</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="s1">&#39;utf-8&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">is_encrypted</span> <span class="o">=</span> <span class="n">fernet</span><span class="o">.</span><span class="n">is_encrypted</span></div>
<span class="nd">@declared_attr</span>
<div class="viewcode-block" id="Variable.val"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.val">[docs]</a> <span class="k">def</span> <span class="nf">val</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="n">synonym</span><span class="p">(</span><span class="s1">&#39;_val&#39;</span><span class="p">,</span>
<span class="n">descriptor</span><span class="o">=</span><span class="nb">property</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">get_val</span><span class="p">,</span> <span class="bp">cls</span><span class="o">.</span><span class="n">set_val</span><span class="p">))</span></div>
<span class="nd">@classmethod</span>
<div class="viewcode-block" id="Variable.setdefault"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.setdefault">[docs]</a> <span class="k">def</span> <span class="nf">setdefault</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">default</span><span class="p">,</span> <span class="n">deserialize_json</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Like a Python builtin dict object, setdefault returns the current value</span>
<span class="sd"> for a key, and if it isn&#39;t there, stores the default value and returns it.</span>
<span class="sd"> :param key: Dict key for this Variable</span>
<span class="sd"> :type key: str</span>
<span class="sd"> :param default: Default value to set and return if the variable</span>
<span class="sd"> isn&#39;t already in the DB</span>
<span class="sd"> :type default: Mixed</span>
<span class="sd"> :param deserialize_json: Store this as a JSON encoded value in the DB</span>
<span class="sd"> and un-encode it when retrieving a value</span>
<span class="sd"> :return: Mixed</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">obj</span> <span class="o">=</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">default_var</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">deserialize_json</span><span class="o">=</span><span class="n">deserialize_json</span><span class="p">)</span>
<span class="k">if</span> <span class="n">obj</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">default</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">Variable</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">default</span><span class="p">,</span> <span class="n">serialize_json</span><span class="o">=</span><span class="n">deserialize_json</span><span class="p">)</span>
<span class="k">return</span> <span class="n">default</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Default Value must be set&#39;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">obj</span></div>
<span class="nd">@classmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="Variable.get"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.get">[docs]</a> <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">default_var</span><span class="o">=</span><span class="n">__NO_DEFAULT_SENTINEL</span><span class="p">,</span> <span class="n">deserialize_json</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">obj</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">key</span> <span class="o">==</span> <span class="n">key</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="k">if</span> <span class="n">obj</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">default_var</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">cls</span><span class="o">.</span><span class="n">__NO_DEFAULT_SENTINEL</span><span class="p">:</span>
<span class="k">return</span> <span class="n">default_var</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s1">&#39;Variable </span><span class="si">{}</span><span class="s1"> does not exist&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="n">deserialize_json</span><span class="p">:</span>
<span class="k">return</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="n">val</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">obj</span><span class="o">.</span><span class="n">val</span></div>
<span class="nd">@classmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="Variable.set"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.set">[docs]</a> <span class="k">def</span> <span class="nf">set</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">serialize_json</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">if</span> <span class="n">serialize_json</span><span class="p">:</span>
<span class="n">stored_value</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">stored_value</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">key</span> <span class="o">==</span> <span class="n">key</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
<span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">Variable</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">val</span><span class="o">=</span><span class="n">stored_value</span><span class="p">))</span>
<span class="n">session</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span></div>
<div class="viewcode-block" id="Variable.rotate_fernet_key"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Variable.rotate_fernet_key">[docs]</a> <span class="k">def</span> <span class="nf">rotate_fernet_key</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">fernet</span> <span class="o">=</span> <span class="n">get_fernet</span><span class="p">()</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_val</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_encrypted</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_val</span> <span class="o">=</span> <span class="n">fernet</span><span class="o">.</span><span class="n">rotate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_val</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span></div></div>
<div class="viewcode-block" id="XCom"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom">[docs]</a><span class="k">class</span> <span class="nc">XCom</span><span class="p">(</span><span class="n">Base</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Base class for XCom objects.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<div class="viewcode-block" id="XCom.__tablename__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.__tablename__">[docs]</a> <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">&quot;xcom&quot;</span></div>
<div class="viewcode-block" id="XCom.id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.id">[docs]</a> <span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="XCom.key"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.key">[docs]</a> <span class="n">key</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">512</span><span class="p">))</span></div>
<div class="viewcode-block" id="XCom.value"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.value">[docs]</a> <span class="n">value</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">LargeBinary</span><span class="p">)</span></div>
<div class="viewcode-block" id="XCom.timestamp"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.timestamp">[docs]</a> <span class="n">timestamp</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span>
<span class="n">UtcDateTime</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">,</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<div class="viewcode-block" id="XCom.execution_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.execution_date">[docs]</a> <span class="n">execution_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">,</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<span class="c1"># source information</span>
<div class="viewcode-block" id="XCom.task_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.task_id">[docs]</a> <span class="n">task_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<div class="viewcode-block" id="XCom.dag_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.dag_id">[docs]</a> <span class="n">dag_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div>
<div class="viewcode-block" id="XCom.__table_args__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.__table_args__">[docs]</a> <span class="n">__table_args__</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">Index</span><span class="p">(</span><span class="s1">&#39;idx_xcom_dag_task_date&#39;</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">,</span> <span class="n">unique</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span></div>
<span class="p">)</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> TODO: &quot;pickling&quot; has been deprecated and JSON is preferred.</span>
<span class="sd"> &quot;pickling&quot; will be removed in Airflow 2.0.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="nd">@reconstructor</span>
<div class="viewcode-block" id="XCom.init_on_load"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.init_on_load">[docs]</a> <span class="k">def</span> <span class="nf">init_on_load</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">enable_pickling</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;enable_xcom_pickling&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">enable_pickling</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">value</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">&#39;UTF-8&#39;</span><span class="p">))</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">UnicodeEncodeError</span><span class="p">,</span> <span class="ne">ValueError</span><span class="p">):</span>
<span class="c1"># For backward-compatibility.</span>
<span class="c1"># Preventing errors in webserver</span>
<span class="c1"># due to XComs mixed with pickled and unpickled.</span>
<span class="bp">self</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">value</span><span class="p">)</span></div>
<div class="viewcode-block" id="XCom.__repr__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.__repr__">[docs]</a> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s1">&#39;&lt;XCom &quot;</span><span class="si">{key}</span><span class="s1">&quot; (</span><span class="si">{task_id}</span><span class="s1"> @ </span><span class="si">{execution_date}</span><span class="s1">)&gt;&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">key</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span></div>
<span class="nd">@classmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="XCom.set"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.set">[docs]</a> <span class="k">def</span> <span class="nf">set</span><span class="p">(</span>
<span class="bp">cls</span><span class="p">,</span>
<span class="n">key</span><span class="p">,</span>
<span class="n">value</span><span class="p">,</span>
<span class="n">execution_date</span><span class="p">,</span>
<span class="n">task_id</span><span class="p">,</span>
<span class="n">dag_id</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Store an XCom value.</span>
<span class="sd"> TODO: &quot;pickling&quot; has been deprecated and JSON is preferred.</span>
<span class="sd"> &quot;pickling&quot; will be removed in Airflow 2.0.</span>
<span class="sd"> :return: None</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">session</span><span class="o">.</span><span class="n">expunge_all</span><span class="p">()</span>
<span class="n">enable_pickling</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;enable_xcom_pickling&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">enable_pickling</span><span class="p">:</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">value</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">value</span><span class="p">)</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;UTF-8&#39;</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Could not serialize the XCOM value into JSON. &quot;</span>
<span class="s2">&quot;If you are using pickles instead of JSON &quot;</span>
<span class="s2">&quot;for XCOM, then you need to enable pickle &quot;</span>
<span class="s2">&quot;support for XCOM in your airflow config.&quot;</span><span class="p">)</span>
<span class="k">raise</span>
<span class="c1"># remove any duplicate XComs</span>
<span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="bp">cls</span><span class="o">.</span><span class="n">key</span> <span class="o">==</span> <span class="n">key</span><span class="p">,</span>
<span class="bp">cls</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">execution_date</span><span class="p">,</span>
<span class="bp">cls</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="n">task_id</span><span class="p">,</span>
<span class="bp">cls</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="c1"># insert new XCom</span>
<span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">XCom</span><span class="p">(</span>
<span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span>
<span class="n">value</span><span class="o">=</span><span class="n">value</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="n">execution_date</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="n">task_id</span><span class="p">,</span>
<span class="n">dag_id</span><span class="o">=</span><span class="n">dag_id</span><span class="p">))</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
<span class="nd">@classmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="XCom.get_one"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.get_one">[docs]</a> <span class="k">def</span> <span class="nf">get_one</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span>
<span class="n">execution_date</span><span class="p">,</span>
<span class="n">key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">task_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">dag_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">include_prior_dates</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Retrieve an XCom value, optionally meeting certain criteria.</span>
<span class="sd"> TODO: &quot;pickling&quot; has been deprecated and JSON is preferred.</span>
<span class="sd"> &quot;pickling&quot; will be removed in Airflow 2.0.</span>
<span class="sd"> :return: XCom value</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">filters</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">key</span><span class="p">:</span>
<span class="n">filters</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">key</span> <span class="o">==</span> <span class="n">key</span><span class="p">)</span>
<span class="k">if</span> <span class="n">task_id</span><span class="p">:</span>
<span class="n">filters</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="n">task_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">dag_id</span><span class="p">:</span>
<span class="n">filters</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">include_prior_dates</span><span class="p">:</span>
<span class="n">filters</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&lt;=</span> <span class="n">execution_date</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">filters</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">execution_date</span><span class="p">)</span>
<span class="n">query</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">value</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">and_</span><span class="p">(</span><span class="o">*</span><span class="n">filters</span><span class="p">))</span>
<span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">desc</span><span class="p">(),</span> <span class="bp">cls</span><span class="o">.</span><span class="n">timestamp</span><span class="o">.</span><span class="n">desc</span><span class="p">()))</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="k">if</span> <span class="n">result</span><span class="p">:</span>
<span class="n">enable_pickling</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">&#39;core&#39;</span><span class="p">,</span> <span class="s1">&#39;enable_xcom_pickling&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">enable_pickling</span><span class="p">:</span>
<span class="k">return</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">return</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">value</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">&#39;UTF-8&#39;</span><span class="p">))</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="n">log</span> <span class="o">=</span> <span class="n">LoggingMixin</span><span class="p">()</span><span class="o">.</span><span class="n">log</span>
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Could not deserialize the XCOM value from JSON. &quot;</span>
<span class="s2">&quot;If you are using pickles instead of JSON &quot;</span>
<span class="s2">&quot;for XCOM, then you need to enable pickle &quot;</span>
<span class="s2">&quot;support for XCOM in your airflow config.&quot;</span><span class="p">)</span>
<span class="k">raise</span></div>
<span class="nd">@classmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="XCom.get_many"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.get_many">[docs]</a> <span class="k">def</span> <span class="nf">get_many</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span>
<span class="n">execution_date</span><span class="p">,</span>
<span class="n">key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">task_ids</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">dag_ids</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">include_prior_dates</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">limit</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Retrieve an XCom value, optionally meeting certain criteria</span>
<span class="sd"> TODO: &quot;pickling&quot; has been deprecated and JSON is preferred.</span>
<span class="sd"> &quot;pickling&quot; will be removed in Airflow 2.0.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">filters</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">key</span><span class="p">:</span>
<span class="n">filters</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">key</span> <span class="o">==</span> <span class="n">key</span><span class="p">)</span>
<span class="k">if</span> <span class="n">task_ids</span><span class="p">:</span>
<span class="n">filters</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">as_tuple</span><span class="p">(</span><span class="n">task_ids</span><span class="p">)))</span>
<span class="k">if</span> <span class="n">dag_ids</span><span class="p">:</span>
<span class="n">filters</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">as_tuple</span><span class="p">(</span><span class="n">dag_ids</span><span class="p">)))</span>
<span class="k">if</span> <span class="n">include_prior_dates</span><span class="p">:</span>
<span class="n">filters</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&lt;=</span> <span class="n">execution_date</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">filters</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">execution_date</span><span class="p">)</span>
<span class="n">query</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">and_</span><span class="p">(</span><span class="o">*</span><span class="n">filters</span><span class="p">))</span>
<span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">desc</span><span class="p">(),</span> <span class="bp">cls</span><span class="o">.</span><span class="n">timestamp</span><span class="o">.</span><span class="n">desc</span><span class="p">())</span>
<span class="o">.</span><span class="n">limit</span><span class="p">(</span><span class="n">limit</span><span class="p">))</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
<span class="k">return</span> <span class="n">results</span></div>
<span class="nd">@classmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="XCom.delete"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.XCom.delete">[docs]</a> <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">xcoms</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">xcoms</span><span class="p">,</span> <span class="n">XCom</span><span class="p">):</span>
<span class="n">xcoms</span> <span class="o">=</span> <span class="p">[</span><span class="n">xcoms</span><span class="p">]</span>
<span class="k">for</span> <span class="n">xcom</span> <span class="ow">in</span> <span class="n">xcoms</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">xcom</span><span class="p">,</span> <span class="n">XCom</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span>
<span class="s1">&#39;Expected XCom; received </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">xcom</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span>
<span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">xcom</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div></div>
<div class="viewcode-block" id="DagRun"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun">[docs]</a><span class="k">class</span> <span class="nc">DagRun</span><span class="p">(</span><span class="n">Base</span><span class="p">,</span> <span class="n">LoggingMixin</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> DagRun describes an instance of a Dag. It can be created</span>
<span class="sd"> by the scheduler (for regular runs) or by an external trigger</span>
<span class="sd"> &quot;&quot;&quot;</span>
<div class="viewcode-block" id="DagRun.__tablename__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.__tablename__">[docs]</a> <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">&quot;dag_run&quot;</span></div>
<div class="viewcode-block" id="DagRun.ID_PREFIX"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.ID_PREFIX">[docs]</a> <span class="n">ID_PREFIX</span> <span class="o">=</span> <span class="s1">&#39;scheduled__&#39;</span></div>
<div class="viewcode-block" id="DagRun.ID_FORMAT_PREFIX"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.ID_FORMAT_PREFIX">[docs]</a> <span class="n">ID_FORMAT_PREFIX</span> <span class="o">=</span> <span class="n">ID_PREFIX</span> <span class="o">+</span> <span class="s1">&#39;</span><span class="si">{0}</span><span class="s1">&#39;</span></div>
<div class="viewcode-block" id="DagRun.id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.id">[docs]</a> <span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagRun.dag_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.dag_id">[docs]</a> <span class="n">dag_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">))</span></div>
<div class="viewcode-block" id="DagRun.execution_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.execution_date">[docs]</a> <span class="n">execution_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagRun.start_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.start_date">[docs]</a> <span class="n">start_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagRun.end_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.end_date">[docs]</a> <span class="n">end_date</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagRun._state"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun._state">[docs]</a> <span class="n">_state</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="s1">&#39;state&#39;</span><span class="p">,</span> <span class="n">String</span><span class="p">(</span><span class="mi">50</span><span class="p">),</span> <span class="n">default</span><span class="o">=</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagRun.run_id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.run_id">[docs]</a> <span class="n">run_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">))</span></div>
<div class="viewcode-block" id="DagRun.external_trigger"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.external_trigger">[docs]</a> <span class="n">external_trigger</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagRun.conf"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.conf">[docs]</a> <span class="n">conf</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">PickleType</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagRun.dag"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.dag">[docs]</a> <span class="n">dag</span> <span class="o">=</span> <span class="kc">None</span></div>
<div class="viewcode-block" id="DagRun.__table_args__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.__table_args__">[docs]</a> <span class="n">__table_args__</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">Index</span><span class="p">(</span><span class="s1">&#39;dag_id_state&#39;</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">_state</span><span class="p">),</span>
<span class="n">UniqueConstraint</span><span class="p">(</span><span class="s1">&#39;dag_id&#39;</span><span class="p">,</span> <span class="s1">&#39;execution_date&#39;</span><span class="p">),</span>
<span class="n">UniqueConstraint</span><span class="p">(</span><span class="s1">&#39;dag_id&#39;</span><span class="p">,</span> <span class="s1">&#39;run_id&#39;</span><span class="p">),</span></div>
<span class="p">)</span>
<div class="viewcode-block" id="DagRun.__repr__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.__repr__">[docs]</a> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">(</span>
<span class="s1">&#39;&lt;DagRun </span><span class="si">{dag_id}</span><span class="s1"> @ </span><span class="si">{execution_date}</span><span class="s1">: </span><span class="si">{run_id}</span><span class="s1">, &#39;</span>
<span class="s1">&#39;externally triggered: </span><span class="si">{external_trigger}</span><span class="s1">&gt;&#39;</span>
<span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">dag_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">execution_date</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span>
<span class="n">run_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="p">,</span>
<span class="n">external_trigger</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">external_trigger</span><span class="p">)</span></div>
<div class="viewcode-block" id="DagRun.get_state"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.get_state">[docs]</a> <span class="k">def</span> <span class="nf">get_state</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_state</span></div>
<div class="viewcode-block" id="DagRun.set_state"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.set_state">[docs]</a> <span class="k">def</span> <span class="nf">set_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_state</span> <span class="o">!=</span> <span class="n">state</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_state</span> <span class="o">=</span> <span class="n">state</span>
<span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_state</span> <span class="ow">in</span> <span class="n">State</span><span class="o">.</span><span class="n">finished</span><span class="p">()</span> <span class="k">else</span> <span class="kc">None</span></div>
<span class="nd">@declared_attr</span>
<div class="viewcode-block" id="DagRun.state"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.state">[docs]</a> <span class="k">def</span> <span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">synonym</span><span class="p">(</span><span class="s1">&#39;_state&#39;</span><span class="p">,</span>
<span class="n">descriptor</span><span class="o">=</span><span class="nb">property</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_state</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">set_state</span><span class="p">))</span></div>
<span class="nd">@classmethod</span>
<div class="viewcode-block" id="DagRun.id_for_date"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.id_for_date">[docs]</a> <span class="k">def</span> <span class="nf">id_for_date</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">date</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="n">ID_FORMAT_PREFIX</span><span class="p">):</span>
<span class="k">return</span> <span class="n">prefix</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()[:</span><span class="mi">19</span><span class="p">])</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagRun.refresh_from_db"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.refresh_from_db">[docs]</a> <span class="k">def</span> <span class="nf">refresh_from_db</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Reloads the current dagrun from the database</span>
<span class="sd"> :param session: database session</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">DR</span> <span class="o">=</span> <span class="n">DagRun</span>
<span class="n">exec_date</span> <span class="o">=</span> <span class="n">func</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span> <span class="n">DateTime</span><span class="p">)</span>
<span class="n">dr</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DR</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">func</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span> <span class="n">DateTime</span><span class="p">)</span> <span class="o">==</span> <span class="n">exec_date</span><span class="p">,</span>
<span class="n">DR</span><span class="o">.</span><span class="n">run_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_id</span>
<span class="p">)</span><span class="o">.</span><span class="n">one</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">id</span> <span class="o">=</span> <span class="n">dr</span><span class="o">.</span><span class="n">id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">dr</span><span class="o">.</span><span class="n">state</span></div>
<span class="nd">@staticmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagRun.find"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.find">[docs]</a> <span class="k">def</span> <span class="nf">find</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">run_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">execution_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">state</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">external_trigger</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">no_backfills</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns a set of dag runs for the given search criteria.</span>
<span class="sd"> :param dag_id: the dag_id to find dag runs for</span>
<span class="sd"> :type dag_id: int, list</span>
<span class="sd"> :param run_id: defines the the run id for this dag run</span>
<span class="sd"> :type run_id: str</span>
<span class="sd"> :param execution_date: the execution date</span>
<span class="sd"> :type execution_date: datetime.datetime</span>
<span class="sd"> :param state: the state of the dag run</span>
<span class="sd"> :type state: airflow.utils.state.State</span>
<span class="sd"> :param external_trigger: whether this dag run is externally triggered</span>
<span class="sd"> :type external_trigger: bool</span>
<span class="sd"> :param no_backfills: return no backfills (True), return all (False).</span>
<span class="sd"> Defaults to False</span>
<span class="sd"> :type no_backfills: bool</span>
<span class="sd"> :param session: database session</span>
<span class="sd"> :type session: sqlalchemy.orm.session.Session</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">DR</span> <span class="o">=</span> <span class="n">DagRun</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DR</span><span class="p">)</span>
<span class="k">if</span> <span class="n">dag_id</span><span class="p">:</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">run_id</span><span class="p">:</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">run_id</span> <span class="o">==</span> <span class="n">run_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">execution_date</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">execution_date</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">execution_date</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">execution_date</span><span class="p">)</span>
<span class="k">if</span> <span class="n">state</span><span class="p">:</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">state</span><span class="p">)</span>
<span class="k">if</span> <span class="n">external_trigger</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">external_trigger</span> <span class="o">==</span> <span class="n">external_trigger</span><span class="p">)</span>
<span class="k">if</span> <span class="n">no_backfills</span><span class="p">:</span>
<span class="c1"># in order to prevent a circular dependency</span>
<span class="kn">from</span> <span class="nn">airflow.jobs</span> <span class="k">import</span> <span class="n">BackfillJob</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">run_id</span><span class="o">.</span><span class="n">notlike</span><span class="p">(</span><span class="n">BackfillJob</span><span class="o">.</span><span class="n">ID_PREFIX</span> <span class="o">+</span> <span class="s1">&#39;%&#39;</span><span class="p">))</span>
<span class="n">dr</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
<span class="k">return</span> <span class="n">dr</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagRun.get_task_instances"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.get_task_instances">[docs]</a> <span class="k">def</span> <span class="nf">get_task_instances</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the task instances for this dag run</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">if</span> <span class="n">state</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">state</span><span class="p">,</span> <span class="n">six</span><span class="o">.</span><span class="n">string_types</span><span class="p">):</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">state</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># this is required to deal with NULL values</span>
<span class="k">if</span> <span class="kc">None</span> <span class="ow">in</span> <span class="n">state</span><span class="p">:</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">or_</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">state</span><span class="p">),</span>
<span class="n">TI</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">is_</span><span class="p">(</span><span class="kc">None</span><span class="p">))</span>
<span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">state</span><span class="p">))</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">partial</span><span class="p">:</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">task_ids</span><span class="p">))</span>
<span class="k">return</span> <span class="n">tis</span><span class="o">.</span><span class="n">all</span><span class="p">()</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagRun.get_task_instance"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.get_task_instance">[docs]</a> <span class="k">def</span> <span class="nf">get_task_instance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the task instance specified by task_id for this dag run</span>
<span class="sd"> :param task_id: the task id</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span>
<span class="n">ti</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span>
<span class="n">TI</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="n">task_id</span>
<span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="k">return</span> <span class="n">ti</span></div>
<div class="viewcode-block" id="DagRun.get_dag"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.get_dag">[docs]</a> <span class="k">def</span> <span class="nf">get_dag</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the Dag associated with this DagRun.</span>
<span class="sd"> :return: DAG</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">&quot;The DAG (.dag) for </span><span class="si">{}</span><span class="s2"> needs to be set&quot;</span>
<span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagRun.get_previous_dagrun"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.get_previous_dagrun">[docs]</a> <span class="k">def</span> <span class="nf">get_previous_dagrun</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;The previous DagRun, if there is one&quot;&quot;&quot;</span>
<span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span>
<span class="p">)</span><span class="o">.</span><span class="n">order_by</span><span class="p">(</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">desc</span><span class="p">()</span></div>
<span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagRun.get_previous_scheduled_dagrun"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.get_previous_scheduled_dagrun">[docs]</a> <span class="k">def</span> <span class="nf">get_previous_scheduled_dagrun</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;The previous, SCHEDULED DagRun, if there is one&quot;&quot;&quot;</span>
<span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_dag</span><span class="p">()</span>
<span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">dag</span><span class="o">.</span><span class="n">previous_schedule</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span></div>
<span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagRun.update_state"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.update_state">[docs]</a> <span class="k">def</span> <span class="nf">update_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Determines the overall state of the DagRun based on the state</span>
<span class="sd"> of its TaskInstances.</span>
<span class="sd"> :return: State</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_dag</span><span class="p">()</span>
<span class="n">tis</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_task_instances</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Updating state for </span><span class="si">%s</span><span class="s2"> considering </span><span class="si">%s</span><span class="s2"> task(s)&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">tis</span><span class="p">))</span>
<span class="k">for</span> <span class="n">ti</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="n">tis</span><span class="p">):</span>
<span class="c1"># skip in db?</span>
<span class="k">if</span> <span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">REMOVED</span><span class="p">:</span>
<span class="n">tis</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">ti</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
<span class="c1"># pre-calculate</span>
<span class="c1"># db is faster</span>
<span class="n">start_dttm</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span>
<span class="n">unfinished_tasks</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_task_instances</span><span class="p">(</span>
<span class="n">state</span><span class="o">=</span><span class="n">State</span><span class="o">.</span><span class="n">unfinished</span><span class="p">(),</span>
<span class="n">session</span><span class="o">=</span><span class="n">session</span>
<span class="p">)</span>
<span class="n">none_depends_on_past</span> <span class="o">=</span> <span class="nb">all</span><span class="p">(</span><span class="ow">not</span> <span class="n">t</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">depends_on_past</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">unfinished_tasks</span><span class="p">)</span>
<span class="n">none_task_concurrency</span> <span class="o">=</span> <span class="nb">all</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">task_concurrency</span> <span class="ow">is</span> <span class="kc">None</span>
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">unfinished_tasks</span><span class="p">)</span>
<span class="c1"># small speed up</span>
<span class="k">if</span> <span class="n">unfinished_tasks</span> <span class="ow">and</span> <span class="n">none_depends_on_past</span> <span class="ow">and</span> <span class="n">none_task_concurrency</span><span class="p">:</span>
<span class="c1"># todo: this can actually get pretty slow: one task costs between 0.01-015s</span>
<span class="n">no_dependencies_met</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">for</span> <span class="n">ut</span> <span class="ow">in</span> <span class="n">unfinished_tasks</span><span class="p">:</span>
<span class="c1"># We need to flag upstream and check for changes because upstream</span>
<span class="c1"># failures/re-schedules can result in deadlock false positives</span>
<span class="n">old_state</span> <span class="o">=</span> <span class="n">ut</span><span class="o">.</span><span class="n">state</span>
<span class="n">deps_met</span> <span class="o">=</span> <span class="n">ut</span><span class="o">.</span><span class="n">are_dependencies_met</span><span class="p">(</span>
<span class="n">dep_context</span><span class="o">=</span><span class="n">DepContext</span><span class="p">(</span>
<span class="n">flag_upstream_failed</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">ignore_in_retry_period</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="n">ignore_in_reschedule_period</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
<span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="k">if</span> <span class="n">deps_met</span> <span class="ow">or</span> <span class="n">old_state</span> <span class="o">!=</span> <span class="n">ut</span><span class="o">.</span><span class="n">current_state</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">):</span>
<span class="n">no_dependencies_met</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">break</span>
<span class="n">duration</span> <span class="o">=</span> <span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">start_dttm</span><span class="p">)</span><span class="o">.</span><span class="n">total_seconds</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">timing</span><span class="p">(</span><span class="s2">&quot;dagrun.dependency-check.</span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">),</span> <span class="n">duration</span><span class="p">)</span>
<span class="n">root_ids</span> <span class="o">=</span> <span class="p">[</span><span class="n">t</span><span class="o">.</span><span class="n">task_id</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">roots</span><span class="p">]</span>
<span class="n">roots</span> <span class="o">=</span> <span class="p">[</span><span class="n">t</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">tis</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">task_id</span> <span class="ow">in</span> <span class="n">root_ids</span><span class="p">]</span>
<span class="c1"># if all roots finished and at least one failed, the run failed</span>
<span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="n">unfinished_tasks</span> <span class="ow">and</span>
<span class="nb">any</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">state</span> <span class="ow">in</span> <span class="p">(</span><span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">UPSTREAM_FAILED</span><span class="p">)</span> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">roots</span><span class="p">)):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Marking run </span><span class="si">%s</span><span class="s1"> failed&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">set_state</span><span class="p">(</span><span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">)</span>
<span class="n">dag</span><span class="o">.</span><span class="n">handle_callback</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">success</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">reason</span><span class="o">=</span><span class="s1">&#39;task_failure&#39;</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="c1"># if all roots succeeded and no unfinished tasks, the run succeeded</span>
<span class="k">elif</span> <span class="ow">not</span> <span class="n">unfinished_tasks</span> <span class="ow">and</span> <span class="nb">all</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">state</span> <span class="ow">in</span> <span class="p">(</span><span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">SKIPPED</span><span class="p">)</span>
<span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">roots</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Marking run </span><span class="si">%s</span><span class="s1"> successful&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">set_state</span><span class="p">(</span><span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">)</span>
<span class="n">dag</span><span class="o">.</span><span class="n">handle_callback</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">success</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">reason</span><span class="o">=</span><span class="s1">&#39;success&#39;</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="c1"># if *all tasks* are deadlocked, the run failed</span>
<span class="k">elif</span> <span class="p">(</span><span class="n">unfinished_tasks</span> <span class="ow">and</span> <span class="n">none_depends_on_past</span> <span class="ow">and</span>
<span class="n">none_task_concurrency</span> <span class="ow">and</span> <span class="n">no_dependencies_met</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Deadlock; marking run </span><span class="si">%s</span><span class="s1"> failed&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">set_state</span><span class="p">(</span><span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">)</span>
<span class="n">dag</span><span class="o">.</span><span class="n">handle_callback</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">success</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">reason</span><span class="o">=</span><span class="s1">&#39;all_tasks_deadlocked&#39;</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="c1"># finally, if the roots aren&#39;t done, the dag is still running</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">set_state</span><span class="p">(</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">)</span>
<span class="c1"># todo: determine we want to use with_for_update to make sure to lock the run</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagRun.verify_integrity"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.verify_integrity">[docs]</a> <span class="k">def</span> <span class="nf">verify_integrity</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Verifies the DagRun by checking for removed tasks or tasks that are not in the</span>
<span class="sd"> database yet. It will set state to removed or add the task if required.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_dag</span><span class="p">()</span>
<span class="n">tis</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_task_instances</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="c1"># check for removed or restored tasks</span>
<span class="n">task_ids</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">ti</span> <span class="ow">in</span> <span class="n">tis</span><span class="p">:</span>
<span class="n">task_ids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
<span class="n">task</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">task</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
<span class="k">except</span> <span class="n">AirflowException</span><span class="p">:</span>
<span class="k">if</span> <span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">REMOVED</span><span class="p">:</span>
<span class="k">pass</span> <span class="c1"># ti has already been removed, just ignore it</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">dag</span><span class="o">.</span><span class="n">partial</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;Failed to get task &#39;</span><span class="si">{}</span><span class="s2">&#39; for dag &#39;</span><span class="si">{}</span><span class="s2">&#39;. &quot;</span>
<span class="s2">&quot;Marking it as removed.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ti</span><span class="p">,</span> <span class="n">dag</span><span class="p">))</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span>
<span class="s2">&quot;task_removed_from_dag.</span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">REMOVED</span>
<span class="n">is_task_in_dag</span> <span class="o">=</span> <span class="n">task</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
<span class="n">should_restore_task</span> <span class="o">=</span> <span class="n">is_task_in_dag</span> <span class="ow">and</span> <span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">REMOVED</span>
<span class="k">if</span> <span class="n">should_restore_task</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Restoring task &#39;</span><span class="si">{}</span><span class="s2">&#39; which was previously &quot;</span>
<span class="s2">&quot;removed from DAG &#39;</span><span class="si">{}</span><span class="s2">&#39;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ti</span><span class="p">,</span> <span class="n">dag</span><span class="p">))</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span><span class="s2">&quot;task_restored_to_dag.</span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">NONE</span>
<span class="c1"># check for missing tasks</span>
<span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">six</span><span class="o">.</span><span class="n">itervalues</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">):</span>
<span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">start_date</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_backfill</span><span class="p">:</span>
<span class="k">continue</span>
<span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">task_ids</span><span class="p">:</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span>
<span class="s2">&quot;task_instance_created-</span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">),</span>
<span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">ti</span> <span class="o">=</span> <span class="n">TaskInstance</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="DagRun.get_run"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.get_run">[docs]</a> <span class="k">def</span> <span class="nf">get_run</span><span class="p">(</span><span class="n">session</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> :param dag_id: DAG ID</span>
<span class="sd"> :type dag_id: unicode</span>
<span class="sd"> :param execution_date: execution date</span>
<span class="sd"> :type execution_date: datetime</span>
<span class="sd"> :return: DagRun corresponding to the given dag_id and execution date</span>
<span class="sd"> if one exists. None otherwise.</span>
<span class="sd"> :rtype: DagRun</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">,</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">external_trigger</span> <span class="o">==</span> <span class="kc">False</span><span class="p">,</span> <span class="c1"># noqa</span>
<span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">execution_date</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">qry</span><span class="o">.</span><span class="n">first</span><span class="p">()</span></div>
<span class="nd">@property</span>
<div class="viewcode-block" id="DagRun.is_backfill"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.is_backfill">[docs]</a> <span class="k">def</span> <span class="nf">is_backfill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="kn">from</span> <span class="nn">airflow.jobs</span> <span class="k">import</span> <span class="n">BackfillJob</span>
<span class="k">return</span> <span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">run_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span>
<span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">BackfillJob</span><span class="o">.</span><span class="n">ID_PREFIX</span><span class="p">)</span></div>
<span class="p">)</span>
<span class="nd">@classmethod</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="DagRun.get_latest_runs"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.DagRun.get_latest_runs">[docs]</a> <span class="k">def</span> <span class="nf">get_latest_runs</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">session</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Returns the latest DagRun for each DAG. &quot;&quot;&quot;</span>
<span class="n">subquery</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">session</span>
<span class="o">.</span><span class="n">query</span><span class="p">(</span>
<span class="bp">cls</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="n">func</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span><span class="o">.</span><span class="n">label</span><span class="p">(</span><span class="s1">&#39;execution_date&#39;</span><span class="p">))</span>
<span class="o">.</span><span class="n">group_by</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span>
<span class="o">.</span><span class="n">subquery</span><span class="p">()</span>
<span class="p">)</span>
<span class="n">dagruns</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">session</span>
<span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span>
<span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">subquery</span><span class="p">,</span>
<span class="n">and_</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">subquery</span><span class="o">.</span><span class="n">c</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="bp">cls</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">subquery</span><span class="o">.</span><span class="n">c</span><span class="o">.</span><span class="n">execution_date</span><span class="p">))</span>
<span class="o">.</span><span class="n">all</span><span class="p">()</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">dagruns</span></div></div>
<div class="viewcode-block" id="Pool"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Pool">[docs]</a><span class="k">class</span> <span class="nc">Pool</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span>
<div class="viewcode-block" id="Pool.__tablename__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Pool.__tablename__">[docs]</a> <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">&quot;slot_pool&quot;</span></div>
<div class="viewcode-block" id="Pool.id"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Pool.id">[docs]</a> <span class="nb">id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="Pool.pool"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Pool.pool">[docs]</a> <span class="n">pool</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">50</span><span class="p">),</span> <span class="n">unique</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="Pool.slots"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Pool.slots">[docs]</a> <span class="n">slots</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span></div>
<div class="viewcode-block" id="Pool.description"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Pool.description">[docs]</a> <span class="n">description</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Text</span><span class="p">)</span></div>
<div class="viewcode-block" id="Pool.__repr__"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Pool.__repr__">[docs]</a> <span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">pool</span></div>
<div class="viewcode-block" id="Pool.to_json"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Pool.to_json">[docs]</a> <span class="k">def</span> <span class="nf">to_json</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">{</span>
<span class="s1">&#39;id&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
<span class="s1">&#39;pool&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">pool</span><span class="p">,</span>
<span class="s1">&#39;slots&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">slots</span><span class="p">,</span>
<span class="s1">&#39;description&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">description</span><span class="p">,</span></div>
<span class="p">}</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="Pool.used_slots"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Pool.used_slots">[docs]</a> <span class="k">def</span> <span class="nf">used_slots</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the number of slots used at the moment</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">running</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">session</span>
<span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TaskInstance</span><span class="p">)</span>
<span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">pool</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">pool</span><span class="p">)</span>
<span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">)</span>
<span class="o">.</span><span class="n">count</span><span class="p">()</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">running</span></div>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="Pool.queued_slots"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Pool.queued_slots">[docs]</a> <span class="k">def</span> <span class="nf">queued_slots</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the number of slots used at the moment</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">return</span> <span class="p">(</span>
<span class="n">session</span>
<span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TaskInstance</span><span class="p">)</span>
<span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">pool</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">pool</span><span class="p">)</span>
<span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">QUEUED</span><span class="p">)</span></div>
<span class="o">.</span><span class="n">count</span><span class="p">()</span>
<span class="p">)</span>
<span class="nd">@provide_session</span>
<div class="viewcode-block" id="Pool.open_slots"><a class="viewcode-back" href="../../_api/airflow/models/index.html#airflow.models.Pool.open_slots">[docs]</a> <span class="k">def</span> <span class="nf">open_slots</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the number of slots open at the moment</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">used_slots</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">used_slots</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="n">queued_slots</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">queued_slots</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">slots</span> <span class="o">-</span> <span class="n">used_slots</span> <span class="o">-</span> <span class="n">queued_slots</span></div></div>
<span class="c1"># To avoid circular import on Python2.7 we need to define this at the _bottom_</span>
<span class="kn">from</span> <span class="nn">airflow.models.connection</span> <span class="k">import</span> <span class="n">Connection</span> <span class="c1"># noqa: E402,F401</span>
<span class="kn">from</span> <span class="nn">airflow.models.skipmixin</span> <span class="k">import</span> <span class="n">SkipMixin</span> <span class="c1"># noqa: F401</span>
</pre></div>
</div>
</div>
<footer>
<hr/>
<div role="contentinfo">
<p>
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>