Google Git
Sign in
apache / airflow-site / f81e48aadee354653679693fa54b04ac42d24bfd / . / docs-archive / 1.10.5 / _api / airflow / models / dagbag / index.html
blob: a631e7ac33999007842c4c6490d6aa6bda7b5f04 [file] [log] [blame]
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>airflow.models.dagbag &mdash; Airflow Documentation</title>
<script type="text/javascript" src="../../../../_static/js/modernizr.min.js"></script>
<script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
<script type="text/javascript" src="../../../../_static/jquery.js"></script>
<script type="text/javascript" src="../../../../_static/underscore.js"></script>
<script type="text/javascript" src="../../../../_static/doctools.js"></script>
<script type="text/javascript" src="../../../../_static/language_data.js"></script>
<script type="text/javascript" src="../../../../_static/js/theme.js"></script>
<link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="../../../../genindex.html" />
<link rel="search" title="Search" href="../../../../search.html" />
<link rel="next" title="airflow.models.dagpickle" href="../dagpickle/index.html" />
<link rel="prev" title="airflow.models.dag" href="../dag/index.html" />
<script>
document.addEventListener('DOMContentLoaded', function() {
var el = document.getElementById('changelog');
if (el !== null ) {
// [AIRFLOW-...]
el.innerHTML = el.innerHTML.replace(
/\[(AIRFLOW-[\d]+)\]/g,
`<a href="https://issues.apache.org/jira/browse/$1">[$1]</a>`
);
// (#...)
el.innerHTML = el.innerHTML.replace(
/\(#([\d]+)\)/g,
`<a href="https://github.com/apache/airflow/pull/$1">(#$1)</a>`
);
};
})
</script>
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-140539454-1']);
_gaq.push(['_trackPageview']);
</script>
<style>
.example-header {
position: relative;
background: #9AAA7A;
padding: 8px 16px;
margin-bottom: 0;
}
.example-header--with-button {
padding-right: 166px;
}
.example-header:after{
content: '';
display: table;
clear: both;
}
.example-title {
display:block;
padding: 4px;
margin-right: 16px;
color: white;
overflow-x: auto;
}
.example-header-button {
top: 8px;
right: 16px;
position: absolute;
}
.example-header + .highlight-python {
margin-top: 0 !important;
}
.viewcode-button {
display: inline-block;
padding: 8px 16px;
border: 0;
margin: 0;
outline: 0;
border-radius: 2px;
-webkit-box-shadow: 0 3px 5px 0 rgba(0,0,0,.3);
box-shadow: 0 3px 6px 0 rgba(0,0,0,.3);
color: #404040;
background-color: #e7e7e7;
cursor: pointer;
font-size: 16px;
font-weight: 500;
line-height: 1;
text-decoration: none;
text-overflow: ellipsis;
overflow: hidden;
text-transform: uppercase;
-webkit-transition: background-color .2s;
transition: background-color .2s;
vertical-align: middle;
white-space: nowrap;
}
.viewcode-button:visited {
color: #404040;
}
.viewcode-button:hover, .viewcode-button:focus {
color: #404040;
background-color: #d6d6d6;
}
</style>
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="../../../../index.html" class="icon icon-home"> Airflow
</a>
<div class="version">
1.10.5
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../concepts.html">Concepts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../profiling.html">Data Profiling</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../cli.html">Command Line Interface Reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling &amp; Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">REST API Reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../metrics.html">Metrics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../macros.html">Macros reference</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../../../index.html">Python API Reference</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../../../index.html#operators">Operators</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../index.html#hooks">Hooks</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../../index.html#executors">Executors</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="../../../index.html#models">Models</a><ul class="current">
<li class="toctree-l3 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.models</span></code></a><ul class="current">
<li class="toctree-l4 current"><a class="reference internal" href="../index.html#submodules">Submodules</a></li>
<li class="toctree-l4"><a class="reference internal" href="../index.html#package-contents">Package Contents</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../../index.html#core-and-community-package">Core and community package</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../../index.html">Airflow</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../../../index.html">Docs</a> &raquo;</li>
<li><a href="../../../index.html">Python API Reference</a> &raquo;</li>
<li><a href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.models</span></code></a> &raquo;</li>
<li><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.models.dagbag</span></code></li>
<li class="wy-breadcrumbs-aside">
<a href="../../../../_sources/_api/airflow/models/dagbag/index.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="module-airflow.models.dagbag">
<span id="airflow-models-dagbag"></span><h1><a class="reference internal" href="#module-airflow.models.dagbag" title="airflow.models.dagbag"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.models.dagbag</span></code></a><a class="headerlink" href="#module-airflow.models.dagbag" title="Permalink to this headline"></a></h1>
<div class="section" id="module-contents">
<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline"></a></h2>
<dl class="class">
<dt id="airflow.models.dagbag.DagBag">
<em class="property">class </em><code class="sig-prename descclassname">airflow.models.dagbag.</code><code class="sig-name descname">DagBag</code><span class="sig-paren">(</span><em class="sig-param">dag_folder=None</em>, <em class="sig-param">executor=None</em>, <em class="sig-param">include_examples=configuration.conf.getboolean('core'</em>, <em class="sig-param">'LOAD_EXAMPLES')</em>, <em class="sig-param">safe_mode=configuration.conf.getboolean('core'</em>, <em class="sig-param">'DAG_DISCOVERY_SAFE_MODE')</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/models/dagbag.html#DagBag"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.dagbag.DagBag" title="Permalink to this definition"></a></dt>
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.dag.base_dag.BaseDagBag</span></code>, <code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.utils.log.logging_mixin.LoggingMixin</span></code></p>
<p>A dagbag is a collection of dags, parsed out of a folder tree and has high
level configuration settings, like what database to use as a backend and
what executor to use to fire off tasks. This makes it easier to run
distinct environments for say production and development, tests, or for
different teams or security profiles. What would have been system level
settings are now dagbag level so that one system can run multiple,
independent settings sets.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>dag_folder</strong> (<em>unicode</em>) – the folder to scan to find DAGs</p></li>
<li><p><strong>executor</strong> – the executor to use when executing task instances
in this DagBag</p></li>
<li><p><strong>include_examples</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a>) – whether to include the examples that ship
with airflow or not</p></li>
<li><p><strong>has_logged</strong> – an instance boolean that gets flipped from False to True after a
file has been skipped. This is to prevent overloading the user with logging
messages about skipped files. Therefore only once per DagBag is a file logged
being skipped.</p></li>
</ul>
</dd>
</dl>
<dl class="attribute">
<dt id="airflow.models.dagbag.DagBag.CYCLE_NEW">
<code class="sig-name descname">CYCLE_NEW</code><em class="property"> = 0</em><a class="reference internal" href="../../../../_modules/airflow/models/dagbag.html#DagBag.CYCLE_NEW"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.dagbag.DagBag.CYCLE_NEW" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="airflow.models.dagbag.DagBag.CYCLE_IN_PROGRESS">
<code class="sig-name descname">CYCLE_IN_PROGRESS</code><em class="property"> = 1</em><a class="reference internal" href="../../../../_modules/airflow/models/dagbag.html#DagBag.CYCLE_IN_PROGRESS"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.dagbag.DagBag.CYCLE_IN_PROGRESS" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="airflow.models.dagbag.DagBag.CYCLE_DONE">
<code class="sig-name descname">CYCLE_DONE</code><em class="property"> = 2</em><a class="reference internal" href="../../../../_modules/airflow/models/dagbag.html#DagBag.CYCLE_DONE"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.dagbag.DagBag.CYCLE_DONE" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="attribute">
<dt id="airflow.models.dagbag.DagBag.dag_ids">
<code class="sig-name descname">dag_ids</code><a class="reference internal" href="../../../../_modules/airflow/models/dagbag.html#DagBag.dag_ids"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.dagbag.DagBag.dag_ids" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
<dl class="method">
<dt id="airflow.models.dagbag.DagBag.size">
<code class="sig-name descname">size</code><span class="sig-paren">(</span><em class="sig-param">self</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/models/dagbag.html#DagBag.size"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.dagbag.DagBag.size" title="Permalink to this definition"></a></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>the amount of dags contained in this dagbag</p>
</dd>
</dl>
</dd></dl>
<dl class="method">
<dt id="airflow.models.dagbag.DagBag.get_dag">
<code class="sig-name descname">get_dag</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">dag_id</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/models/dagbag.html#DagBag.get_dag"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.dagbag.DagBag.get_dag" title="Permalink to this definition"></a></dt>
<dd><p>Gets the DAG out of the dictionary, and refreshes it if expired</p>
</dd></dl>
<dl class="method">
<dt id="airflow.models.dagbag.DagBag.process_file">
<code class="sig-name descname">process_file</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">filepath</em>, <em class="sig-param">only_if_updated=True</em>, <em class="sig-param">safe_mode=True</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/models/dagbag.html#DagBag.process_file"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.dagbag.DagBag.process_file" title="Permalink to this definition"></a></dt>
<dd><p>Given a path to a python module or zip file, this method imports
the module and look for dag objects within it.</p>
</dd></dl>
<dl class="method">
<dt id="airflow.models.dagbag.DagBag.kill_zombies">
<code class="sig-name descname">kill_zombies</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">session=None</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/models/dagbag.html#DagBag.kill_zombies"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.dagbag.DagBag.kill_zombies" title="Permalink to this definition"></a></dt>
<dd><p>Fail zombie tasks, which are tasks that haven’t
had a heartbeat for too long, in the current DagBag.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>session</strong> (<a class="reference external" href="https://docs.sqlalchemy.org/en/13/orm/session_api.html#sqlalchemy.orm.session.Session" title="(in SQLAlchemy v1.3)"><em>sqlalchemy.orm.session.Session</em></a>) – DB session.</p>
</dd>
</dl>
</dd></dl>
<dl class="method">
<dt id="airflow.models.dagbag.DagBag.bag_dag">
<code class="sig-name descname">bag_dag</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">dag</em>, <em class="sig-param">parent_dag</em>, <em class="sig-param">root_dag</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/models/dagbag.html#DagBag.bag_dag"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.dagbag.DagBag.bag_dag" title="Permalink to this definition"></a></dt>
<dd><p>Adds the DAG into the bag, recurses into sub dags.
Throws AirflowDagCycleException if a cycle is detected in this dag or its subdags</p>
</dd></dl>
<dl class="method">
<dt id="airflow.models.dagbag.DagBag.collect_dags">
<code class="sig-name descname">collect_dags</code><span class="sig-paren">(</span><em class="sig-param">self</em>, <em class="sig-param">dag_folder=None</em>, <em class="sig-param">only_if_updated=True</em>, <em class="sig-param">include_examples=configuration.conf.getboolean('core'</em>, <em class="sig-param">'LOAD_EXAMPLES')</em>, <em class="sig-param">safe_mode=configuration.conf.getboolean('core'</em>, <em class="sig-param">'DAG_DISCOVERY_SAFE_MODE')</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/models/dagbag.html#DagBag.collect_dags"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.dagbag.DagBag.collect_dags" title="Permalink to this definition"></a></dt>
<dd><p>Given a file path or a folder, this method looks for python modules,
imports them and adds them to the dagbag collection.</p>
<p>Note that if a <code class="docutils literal notranslate"><span class="pre">.airflowignore</span></code> file is found while processing
the directory, it will behave much like a <code class="docutils literal notranslate"><span class="pre">.gitignore</span></code>,
ignoring files that match any of the regex patterns specified
in the file.</p>
<p><strong>Note</strong>: The patterns in .airflowignore are treated as
un-anchored regexes, not shell-like glob patterns.</p>
</dd></dl>
<dl class="method">
<dt id="airflow.models.dagbag.DagBag.dagbag_report">
<code class="sig-name descname">dagbag_report</code><span class="sig-paren">(</span><em class="sig-param">self</em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/models/dagbag.html#DagBag.dagbag_report"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.dagbag.DagBag.dagbag_report" title="Permalink to this definition"></a></dt>
<dd><p>Prints a report around DagBag loading stats</p>
</dd></dl>
</dd></dl>
</div>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="../dagpickle/index.html" class="btn btn-neutral float-right" title="airflow.models.dagpickle" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="../dag/index.html" class="btn btn-neutral float-left" title="airflow.models.dag" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
<div class="footer">This page uses <a href="https://analytics.google.com/">
Google Analytics</a> to collect statistics. You can disable it by blocking
the JavaScript coming from www.google-analytics.com. Check our
<a href="../../../../privacy_notice.html">Privacy Policy</a>
for more details.
<script type="text/javascript">
(function() {
var ga = document.createElement('script');
ga.src = ('https:' == document.location.protocol ?
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
ga.setAttribute('async', 'true');
var nodes = document.documentElement.childNodes;
var i = -1;
var node;
do {
i++;
node = nodes[i]
} while(node.nodeType !== Node.ELEMENT_NODE);
node.appendChild(ga);
})();
</script>
</div>
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>