| |
| |
| |
| |
| <!DOCTYPE html> |
| <!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> |
| <!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <title>airflow.models.dag — Airflow Documentation</title> |
| <link rel="stylesheet" href="../../../_static/_gen/css/main.min.css" type="text/css" /> |
| <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" /> |
| <link rel="stylesheet" type="text/css" href="../../../_static/_gen/css/main.min.css" /> |
| <link rel="stylesheet" type="text/css" href="../../../_static/copybutton.css" /> |
| <link rel="stylesheet" type="text/css" href="../../../_static/_gen/css/main-custom.min.css" /> |
| <link rel="stylesheet" type="text/css" href="../../../_static/graphviz.css" /> |
| <link rel="shortcut icon" href="../../../_static/pin_32.png"/> |
| <link rel="index" title="Index" href="../../../genindex.html" /> |
| <link rel="search" title="Search" href="../../../search.html" /> |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| /* We explicitly disable cookie tracking to avoid privacy issues */ |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '13']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| |
| |
| </head><body class="td-section"> |
| |
| |
| <header> |
| <nav class="js-navbar-scroll navbar"> |
| <div class="navbar__icon-container"> |
| <a href="/"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="155.314" height="60" viewBox="0 0 155.314 60"> |
| <defs> |
| <clipPath id="clip-path"> |
| <path id="Rectangle_1" d="M0 0h155.314v60H0z" fill="none" data-name="Rectangle 1"></path> |
| </clipPath> |
| </defs> |
| <g id="logo" transform="translate(-1305 -780.355)"> |
| <g id="Group_2" clip-path="url(#clip-path)" data-name="Group 2" transform="translate(1305 780.355)"> |
| <g id="Group_1" data-name="Group 1" transform="translate(.486 .486)"> |
| <path id="Path_1" d="M1307.562 880.867l28.187-28.893a.521.521 0 0 0 .063-.666c-1.714-2.393-4.877-2.808-6.049-4.416-3.472-4.763-4.353-7.459-5.845-7.292a.456.456 0 0 0-.271.143l-10.182 10.438c-5.858 6-6.7 19.225-6.852 30.3a.552.552 0 0 0 .949.386z" fill="#017cee" data-name="Path 1" transform="translate(-1306.613 -822.232)"></path> |
| <path id="Path_2" d="M1405.512 908.489l-28.893-28.189a.521.521 0 0 0-.667-.063c-2.393 1.715-2.808 4.877-4.416 6.049-4.763 3.472-7.459 4.353-7.292 5.845a.456.456 0 0 0 .143.27l10.438 10.182c6 5.858 19.225 6.7 30.3 6.852a.552.552 0 0 0 .387-.946z" fill="#00ad46" data-name="Path 2" transform="translate(-1346.876 -850.567)"></path> |
| <path id="Path_3" d="M1373.909 902.252c-3.28-3.2-4.8-9.53 1.486-22.583-10.219 4.567-13.8 10.57-12.039 12.289z" fill="#04d659" data-name="Path 3" transform="translate(-1345.96 -850.233)"></path> |
| <path id="Path_4" d="M1433.132 782.359l-28.186 28.893a.52.52 0 0 0-.063.666c1.715 2.393 4.876 2.808 6.049 4.416 3.472 4.763 4.354 7.459 5.845 7.292a.454.454 0 0 0 .271-.143l10.182-10.438c5.858-6 6.7-19.225 6.852-30.3a.553.553 0 0 0-.95-.386z" fill="#00c7d4" data-name="Path 4" transform="translate(-1375.21 -782.123)"></path> |
| <path id="Path_5" d="M1426.9 881.155c-3.2 3.28-9.53 4.8-22.584-1.486 4.567 10.219 10.57 13.8 12.289 12.039z" fill="#11e1ee" data-name="Path 5" transform="translate(-1374.875 -850.233)"></path> |
| <path id="Path_6" d="M1307 782.919l28.893 28.186a.521.521 0 0 0 .666.063c2.393-1.715 2.808-4.877 4.416-6.049 4.763-3.472 7.459-4.353 7.292-5.845a.459.459 0 0 0-.143-.271l-10.438-10.182c-6-5.858-19.225-6.7-30.3-6.852a.552.552 0 0 0-.386.95z" fill="#e43921" data-name="Path 6" transform="translate(-1306.766 -781.97)"></path> |
| <path id="Path_7" d="M1405.8 804.711c3.28 3.2 4.8 9.53-1.486 22.584 10.219-4.567 13.8-10.571 12.039-12.289z" fill-rule="evenodd" fill="#ff7557" data-name="Path 7" transform="translate(-1374.875 -797.859)"></path> |
| <path id="Path_8" d="M1329.355 849.266c3.2-3.28 9.53-4.8 22.584 1.486-4.567-10.219-10.57-13.8-12.289-12.039z" fill="#0cb6ff" data-name="Path 8" transform="translate(-1322.503 -821.316)"></path> |
| <circle id="Ellipse_1" cx="1.26" cy="1.26" r="1.26" fill="#4a4848" data-name="Ellipse 1" transform="translate(28.18 28.171)"></circle> |
| <path id="Path_9" d="M1527.558 827.347a.229.229 0 0 1-.223-.223.458.458 0 0 1 .011-.123l2.766-7.214a.346.346 0 0 1 .357-.245h.758a.348.348 0 0 1 .357.245l2.754 7.214.022.123a.228.228 0 0 1-.223.223h-.568a.288.288 0 0 1-.19-.056.352.352 0 0 1-.089-.134l-.613-1.583h-3.657l-.613 1.583a.317.317 0 0 1-.1.134.269.269 0 0 1-.178.056zm4.795-2.732l-1.505-3.958-1.505 3.958zm3.322 4.85a.258.258 0 0 1-.189-.078.241.241 0 0 1-.067-.178v-7.4a.241.241 0 0 1 .067-.178.258.258 0 0 1 .189-.078h.513a.268.268 0 0 1 .256.256v.49a2.118 2.118 0 0 1 1.828-.858 2.092 2.092 0 0 1 1.751.736 3.135 3.135 0 0 1 .636 1.9q.011.122.011.379t-.011.379a3.168 3.168 0 0 1-.636 1.9 2.111 2.111 0 0 1-1.751.736 2.154 2.154 0 0 1-1.806-.836v2.587a.241.241 0 0 1-.067.178.223.223 0 0 1-.179.078zm2.364-2.91a1.324 1.324 0 0 0 1.149-.491 2.266 2.266 0 0 0 .4-1.293q.011-.111.011-.323 0-2.107-1.562-2.107a1.365 1.365 0 0 0-1.159.513 2.111 2.111 0 0 0-.412 1.2l-.012.424.012.435a1.862 1.862 0 0 0 .424 1.149 1.4 1.4 0 0 0 1.148.493zm5.628.9a2.329 2.329 0 0 1-1.015-.223 1.94 1.94 0 0 1-.747-.6 1.487 1.487 0 0 1-.268-.859 1.459 1.459 0 0 1 .6-1.2 3.4 3.4 0 0 1 1.65-.624l1.661-.234v-.323q0-1.137-1.3-1.137a1.4 1.4 0 0 0-.8.212 1.376 1.376 0 0 0-.468.48.305.305 0 0 1-.089.145.18.18 0 0 1-.134.045h-.48a.23.23 0 0 1-.245-.245 1.17 1.17 0 0 1 .245-.6 1.931 1.931 0 0 1 .747-.591 2.7 2.7 0 0 1 1.238-.256 2.351 2.351 0 0 1 1.8.591 2.032 2.032 0 0 1 .547 1.45v3.613a.257.257 0 0 1-.078.19.24.24 0 0 1-.178.067h-.513a.233.233 0 0 1-.257-.256v-.479a1.923 1.923 0 0 1-.714.6 2.557 2.557 0 0 1-1.203.237zm.234-.836a1.579 1.579 0 0 0 1.182-.469 1.881 1.881 0 0 0 .468-1.371v-.312l-1.293.19a2.918 2.918 0 0 0-1.193.379.761.761 0 0 0-.4.658.784.784 0 0 0 .368.691 1.585 1.585 0 0 0 .867.237zm6.643.836a2.556 2.556 0 0 1-1.873-.669 2.738 2.738 0 0 1-.714-1.9l-.011-.446.011-.446a2.7 2.7 0 0 1 .714-1.885 2.531 2.531 0 0 1 1.873-.68 2.917 2.917 0 0 1 1.36.29 2.077 2.077 0 0 1 .825.714 1.7 1.7 0 0 1 .3.848.2.2 0 0 1-.067.178.281.281 0 0 1-.19.067h-.535a.265.265 0 0 1-.168-.045.458.458 0 0 1-.111-.178 1.428 1.428 0 0 0-.535-.758 1.516 1.516 0 0 0-.87-.234 1.45 1.45 0 0 0-1.1.435 1.952 1.952 0 0 0-.435 1.3l-.011.4.011.379a1.969 1.969 0 0 0 .435 1.316 1.446 1.446 0 0 0 1.1.424 1.577 1.577 0 0 0 .87-.223 1.493 1.493 0 0 0 .535-.769.458.458 0 0 1 .111-.178.228.228 0 0 1 .168-.056h.535a.258.258 0 0 1 .19.078.2.2 0 0 1 .067.178 1.75 1.75 0 0 1-.3.847 2.078 2.078 0 0 1-.825.714 2.876 2.876 0 0 1-1.361.302zm4.078-.112a.233.233 0 0 1-.257-.256v-7.4a.241.241 0 0 1 .067-.178.259.259 0 0 1 .19-.078h.557a.267.267 0 0 1 .257.256v2.6a2.167 2.167 0 0 1 .758-.624 2.353 2.353 0 0 1 1.082-.223 2.067 2.067 0 0 1 1.661.691 2.642 2.642 0 0 1 .6 1.818v3.144a.257.257 0 0 1-.078.19.24.24 0 0 1-.178.067h-.557a.233.233 0 0 1-.256-.256V824a1.775 1.775 0 0 0-.39-1.227 1.387 1.387 0 0 0-1.1-.435 1.481 1.481 0 0 0-1.126.446 1.7 1.7 0 0 0-.412 1.215v3.088a.257.257 0 0 1-.078.19.24.24 0 0 1-.178.067zm8.846.112a2.466 2.466 0 0 1-1.84-.7 2.938 2.938 0 0 1-.747-1.94l-.011-.379.011-.368a2.953 2.953 0 0 1 .758-1.918 2.7 2.7 0 0 1 3.735.078 3.114 3.114 0 0 1 .68 2.119v.19a.257.257 0 0 1-.078.189.241.241 0 0 1-.178.067h-3.858v.1a2.11 2.11 0 0 0 .435 1.238 1.332 1.332 0 0 0 1.081.5 1.563 1.563 0 0 0 .836-.2 1.7 1.7 0 0 0 .491-.435.6.6 0 0 1 .145-.156.391.391 0 0 1 .19-.033h.547a.252.252 0 0 1 .167.056.192.192 0 0 1 .067.156.975.975 0 0 1-.312.591 2.51 2.51 0 0 1-.859.6 3.049 3.049 0 0 1-1.26.248zm1.527-3.434v-.033a1.817 1.817 0 0 0-.424-1.249 1.512 1.512 0 0 0-2.23 0 1.883 1.883 0 0 0-.4 1.249v.033z" fill="#51504f" data-name="Path 9" transform="translate(-1460.834 -808.144)"></path> |
| <path id="Path_10" d="M1527.2 827.081l-.061.061zm-.056-.279l-.08-.031zm2.766-7.214l.08.031zm1.472 0l-.081.029zm2.754 7.214l.084-.015a.064.064 0 0 0 0-.015zm.022.123h.086v-.015zm-.067.156l.06.061zm-.914.011l-.061.061.006.005zm-.089-.134l.081-.027zm-.613-1.583l.08-.031a.086.086 0 0 0-.08-.055zm-3.657 0v-.086a.086.086 0 0 0-.08.055zm-.613 1.583l-.08-.031zm-.1.134l.055.066zm4.047-2.676v.086a.086.086 0 0 0 .08-.116zm-1.505-3.958l.08-.03a.086.086 0 0 0-.16 0zm-1.505 3.958l-.08-.03a.086.086 0 0 0 .08.116zm-1.784 2.646a.128.128 0 0 1-.1-.042l-.122.121a.3.3 0 0 0 .217.092zm-.1-.042a.129.129 0 0 1-.042-.1h-.171a.3.3 0 0 0 .092.217zm-.042-.1a.38.38 0 0 1 .007-.1l-.163-.054a.514.514 0 0 0-.016.15zm.005-.092l2.765-7.214-.16-.061-2.765 7.214zm2.766-7.216a.283.283 0 0 1 .1-.143.3.3 0 0 1 .174-.046v-.172a.47.47 0 0 0-.271.076.453.453 0 0 0-.166.226zm.276-.189h.758v-.172h-.758zm.758 0a.3.3 0 0 1 .175.046.283.283 0 0 1 .1.143l.161-.059a.451.451 0 0 0-.166-.226.47.47 0 0 0-.272-.076zm.277.19l2.754 7.214.16-.061-2.754-7.214zm2.75 7.2l.022.123.169-.031-.022-.123zm.021.107a.13.13 0 0 1-.042.1l.121.121a.3.3 0 0 0 .092-.217zm-.042.1a.13.13 0 0 1-.1.042v.171a.3.3 0 0 0 .217-.092zm-.1.042h-.568v.171h.568zm-.568 0a.206.206 0 0 1-.135-.036l-.11.132a.373.373 0 0 0 .245.076zm-.129-.031a.262.262 0 0 1-.069-.1l-.162.054a.431.431 0 0 0 .11.167zm-.07-.1l-.613-1.584-.16.062.613 1.583zm-.693-1.638h-3.657v.171h3.657zm-3.737.055l-.614 1.584.16.062.613-1.583zm-.615 1.587a.235.235 0 0 1-.075.1l.111.13a.4.4 0 0 0 .126-.172zm-.074.1a.185.185 0 0 1-.124.036v.171a.354.354 0 0 0 .233-.076zm-.124.036h-.569v.171h.569zm4.306-2.677l-1.505-3.958-.16.061 1.505 3.958zm-1.666-3.958l-1.505 3.958.16.061 1.505-3.958zm-1.425 4.075h3.01v-.171h-3.01zm6.143 4.687l-.06.061zm0-7.761l.061.061zm.881 0l-.065.056.01.009zm.078.669h-.086a.086.086 0 0 0 .155.051zm3.579-.123l-.067.053zm.636 1.9h-.086zm0 .758l-.085-.007zm-.636 1.9l.067.054zm-3.557-.1l.068-.052a.086.086 0 0 0-.154.052zm-.067 2.765l-.061-.06zm2.787-3.323l-.069-.051zm.4-1.293l-.085-.008v.005zm-2.709-1.918l-.068-.052zm-.413 1.2h-.086zm-.011.423h-.085zm.011.435h-.086zm.424 1.149l.066-.054zm-1.216 3.315a.173.173 0 0 1-.129-.053l-.121.121a.342.342 0 0 0 .25.1zm-.129-.053a.157.157 0 0 1-.042-.118h-.172a.325.325 0 0 0 .092.239zm-.042-.118v-7.4h-.172v7.4zm0-7.4a.157.157 0 0 1 .042-.118l-.121-.121a.324.324 0 0 0-.092.239zm.042-.118a.173.173 0 0 1 .129-.053v-.172a.342.342 0 0 0-.25.1zm.129-.053h.513v-.172h-.513zm.513 0a.137.137 0 0 1 .113.048l.13-.111a.309.309 0 0 0-.244-.108zm.123.058a.137.137 0 0 1 .048.113h.171a.309.309 0 0 0-.108-.243zm.048.113v.49h.171v-.49zm.155.541a2.033 2.033 0 0 1 1.759-.823v-.171a2.2 2.2 0 0 0-1.9.894zm1.759-.823a2.007 2.007 0 0 1 1.683.7l.135-.106a2.177 2.177 0 0 0-1.818-.768zm1.683.7a3.045 3.045 0 0 1 .617 1.845l.171-.007a3.218 3.218 0 0 0-.654-1.946zm.617 1.85c.007.078.011.2.011.372h.171c0-.171 0-.3-.012-.387zm.011.372c0 .171 0 .294-.011.372l.17.015c.008-.086.012-.216.012-.387zm-.011.376a3.08 3.08 0 0 1-.617 1.846l.134.106a3.25 3.25 0 0 0 .654-1.945zm-.617 1.845a2.025 2.025 0 0 1-1.683.7v.171a2.2 2.2 0 0 0 1.817-.768zm-1.683.7a2.068 2.068 0 0 1-1.739-.8l-.136.1a2.239 2.239 0 0 0 1.874.87zm-1.892-.75v2.587h.172v-2.587zm0 2.587a.156.156 0 0 1-.042.118l.121.121a.325.325 0 0 0 .092-.239zm-.046.123a.138.138 0 0 1-.114.048v.172a.308.308 0 0 0 .244-.108zm-.114.048h-.546v.172h.546zm1.817-2.739a1.408 1.408 0 0 0 1.218-.526l-.138-.1a1.24 1.24 0 0 1-1.079.455zm1.217-.525a2.355 2.355 0 0 0 .419-1.341l-.171-.007a2.182 2.182 0 0 1-.385 1.246zm.418-1.336c.008-.079.012-.19.012-.332h-.172c0 .14 0 .245-.011.315zm.012-.332a2.726 2.726 0 0 0-.407-1.632 1.448 1.448 0 0 0-1.24-.562v.171a1.278 1.278 0 0 1 1.1.492 2.565 2.565 0 0 1 .374 1.53zm-1.647-2.193a1.452 1.452 0 0 0-1.228.547l.136.1a1.282 1.282 0 0 1 1.091-.479zm-1.228.547a2.2 2.2 0 0 0-.43 1.252l.172.008a2.028 2.028 0 0 1 .4-1.157zm-.43 1.254l-.011.424h.171l.011-.424zm-.011.428l.011.435h.172l-.011-.435zm.011.436a1.95 1.95 0 0 0 .443 1.2l.133-.109a1.776 1.776 0 0 1-.4-1.1zm.443 1.2a1.484 1.484 0 0 0 1.214.522v-.171a1.314 1.314 0 0 1-1.082-.459zm5.828 1.117l.037-.077zm-.747-.6l-.07.049zm.335-2.063l.052.068zm1.65-.624l.012.085zm1.661-.234l.012.085a.086.086 0 0 0 .074-.085zm-2.107-1.249l.046.072zm-.468.48l-.075-.042a.083.083 0 0 0-.006.015zm-.089.145l-.054-.067-.007.006zm-.792-.022l-.065.056.009.009zm-.067-.178h-.086zm.245-.6l-.07-.049zm.747-.591l.038.077zm3.033.334l-.063.058zm.468 5.252l.06.061zm-.881 0l-.065.056a.043.043 0 0 0 .009.009zm-.067-.669h.086a.086.086 0 0 0-.156-.048zm-.714.6l-.04-.076zm.223-1.059l-.062-.06zm.468-1.684h.086a.086.086 0 0 0-.1-.085zm-1.293.189l.012.085zm-1.193.379l.046.072zm-.033 1.349l-.047.071zm.635.985a2.241 2.241 0 0 1-.978-.215l-.074.155a2.412 2.412 0 0 0 1.051.231zm-.978-.215a1.859 1.859 0 0 1-.715-.576l-.138.1a2.024 2.024 0 0 0 .779.629zm-.713-.573a1.4 1.4 0 0 1-.253-.81h-.172a1.571 1.571 0 0 0 .283.907zm-.253-.81a1.374 1.374 0 0 1 .569-1.136l-.105-.135a1.544 1.544 0 0 0-.635 1.272zm.569-1.137a3.316 3.316 0 0 1 1.609-.607l-.024-.17a3.481 3.481 0 0 0-1.691.642zm1.609-.607l1.661-.234-.024-.17-1.662.234zm1.735-.319v-.323h-.171v.323zm0-.323a1.156 1.156 0 0 0-.355-.917 1.536 1.536 0 0 0-1.035-.306v.172a1.37 1.37 0 0 1 .922.263.986.986 0 0 1 .3.788zm-1.39-1.223a1.486 1.486 0 0 0-.851.227l.1.142a1.316 1.316 0 0 1 .755-.2zm-.849.226a1.452 1.452 0 0 0-.5.51l.15.084a1.286 1.286 0 0 1 .44-.449zm-.5.524a.226.226 0 0 1-.062.105l.107.134a.391.391 0 0 0 .117-.185zm-.068.112a.1.1 0 0 1-.073.019v.171a.266.266 0 0 0 .194-.07zm-.073.019h-.48v.171h.48zm-.48 0a.18.18 0 0 1-.122-.046l-.112.13a.352.352 0 0 0 .234.087zm-.113-.037a.18.18 0 0 1-.047-.123h-.171a.352.352 0 0 0 .087.234zm-.047-.119a1.1 1.1 0 0 1 .23-.557l-.14-.1a1.253 1.253 0 0 0-.261.648zm.23-.556a1.843 1.843 0 0 1 .715-.564l-.075-.154a2.018 2.018 0 0 0-.78.618zm.716-.564a2.611 2.611 0 0 1 1.2-.247v-.171a2.781 2.781 0 0 0-1.277.266zm1.2-.247a2.268 2.268 0 0 1 1.732.563l.126-.116a2.435 2.435 0 0 0-1.858-.618zm1.733.564a1.945 1.945 0 0 1 .523 1.391h.171a2.117 2.117 0 0 0-.57-1.508zm.523 1.391v3.613h.171v-3.613zm0 3.613a.172.172 0 0 1-.053.129l.121.121a.344.344 0 0 0 .1-.25zm-.053.129a.157.157 0 0 1-.118.042v.171a.326.326 0 0 0 .239-.092zm-.118.042h-.513v.171h.513zm-.513 0a.2.2 0 0 1-.134-.046l-.111.13a.367.367 0 0 0 .245.088zm-.124-.037a.194.194 0 0 1-.047-.134h-.171a.366.366 0 0 0 .087.245zm-.047-.134v-.479h-.171v.479zm-.156-.528a1.846 1.846 0 0 1-.683.575l.079.152a2.012 2.012 0 0 0 .745-.629zm-.683.575a2.476 2.476 0 0 1-1.153.236v.171a2.644 2.644 0 0 0 1.233-.255zm-.919-.429a1.666 1.666 0 0 0 1.244-.494l-.123-.12a1.493 1.493 0 0 1-1.121.442zm1.244-.494a1.969 1.969 0 0 0 .492-1.431h-.171a1.8 1.8 0 0 1-.444 1.312zm.492-1.431v-.312h-.171v.312zm-.1-.4l-1.293.189.025.17 1.293-.189zm-1.293.189a3 3 0 0 0-1.228.393l.095.143a2.837 2.837 0 0 1 1.158-.365zm-1.227.392a.845.845 0 0 0-.441.73h.172a.676.676 0 0 1 .362-.586zm-.441.73a.869.869 0 0 0 .406.762l.095-.142a.7.7 0 0 1-.33-.62zm.408.764a1.673 1.673 0 0 0 .916.247v-.171a1.5 1.5 0 0 1-.823-.221zm5.686.329l-.061.06zm-.714-1.9h-.085zm-.011-.446h-.085zm.011-.446h-.085zm.714-1.885l.061.061zm3.234-.39l-.04.076zm.825.713l-.073.046zm.3.848h-.086zm-.067.178l.056.065zm-.892.022l.054-.067zm-.112-.178l-.081.029zm-.535-.758l-.048.071zm-1.974.2l-.062-.059zm-.435 1.3h-.086zm-.011.4h-.086zm.011.379h-.086zm.435 1.316l-.062.059zm1.974.2l.046.072zm.535-.769l-.079-.033zm.112-.178l.054.067.007-.006zm.892.022l-.061.06zm.067.178l-.085-.009zm-.3.847l-.072-.046zm-.825.714l-.04-.076zm-1.36.2a2.471 2.471 0 0 1-1.814-.644l-.12.122a2.64 2.64 0 0 0 1.933.694zm-1.813-.643a2.653 2.653 0 0 1-.689-1.839l-.171.006a2.822 2.822 0 0 0 .738 1.952zm-.689-1.838l-.011-.446h-.171l.011.446zm-.011-.442l.011-.446h-.171l-.011.446zm.011-.445a2.611 2.611 0 0 1 .689-1.827l-.122-.121a2.78 2.78 0 0 0-.738 1.942zm.689-1.827a2.447 2.447 0 0 1 1.813-.655v-.171a2.617 2.617 0 0 0-1.934.705zm1.813-.655a2.836 2.836 0 0 1 1.32.28l.079-.152a3 3 0 0 0-1.4-.3zm1.32.28a1.99 1.99 0 0 1 .792.683l.145-.091a2.158 2.158 0 0 0-.858-.744zm.793.685a1.617 1.617 0 0 1 .287.8l.171-.009a1.789 1.789 0 0 0-.315-.89zm.287.809a.11.11 0 0 1-.037.1l.112.13a.281.281 0 0 0 .1-.252zm-.037.1a.2.2 0 0 1-.134.046v.171a.369.369 0 0 0 .246-.088zm-.134.046h-.535v.171h.535zm-.535 0a.184.184 0 0 1-.114-.026l-.107.134a.345.345 0 0 0 .221.064zm-.114-.026a.389.389 0 0 1-.086-.144l-.158.066a.533.533 0 0 0 .137.212zm-.084-.14a1.514 1.514 0 0 0-.57-.8l-.093.144a1.343 1.343 0 0 1 .5.715zm-.568-.8a1.6 1.6 0 0 0-.918-.249v.171a1.435 1.435 0 0 1 .822.219zm-.918-.249a1.535 1.535 0 0 0-1.166.462l.124.118a1.364 1.364 0 0 1 1.042-.408zm-1.166.462a2.036 2.036 0 0 0-.458 1.36l.171.006a1.872 1.872 0 0 1 .411-1.249zm-.458 1.361l-.011.4h.171l.011-.4zm-.011.406l.011.379.171-.005-.011-.379zm.011.38a2.052 2.052 0 0 0 .458 1.371l.124-.118a1.889 1.889 0 0 1-.411-1.26zm.458 1.371a1.533 1.533 0 0 0 1.166.451v-.172a1.363 1.363 0 0 1-1.042-.4zm1.166.451a1.661 1.661 0 0 0 .916-.237l-.093-.144a1.491 1.491 0 0 1-.823.209zm.918-.238a1.576 1.576 0 0 0 .568-.812l-.162-.057a1.409 1.409 0 0 1-.5.727zm.566-.807a.39.39 0 0 1 .086-.144l-.107-.134a.533.533 0 0 0-.137.213zm.093-.151a.144.144 0 0 1 .107-.031v-.171a.31.31 0 0 0-.228.081zm.107-.031h.535v-.171h-.535zm.535 0a.173.173 0 0 1 .129.053l.121-.121a.344.344 0 0 0-.25-.1zm.134.057a.11.11 0 0 1 .037.1l.17.017a.281.281 0 0 0-.1-.252zm.037.109a1.664 1.664 0 0 1-.288.806l.144.092a1.839 1.839 0 0 0 .315-.889zm-.288.806a1.989 1.989 0 0 1-.792.683l.079.152a2.162 2.162 0 0 0 .858-.744zm-.793.684a2.8 2.8 0 0 1-1.32.28v.171a2.96 2.96 0 0 0 1.4-.3zm2.568.187l-.065.056.01.009zm0-7.772l.061.06zm.926 0l-.065.056.009.009zm.078 2.776h-.085a.086.086 0 0 0 .153.053zm.758-.624l.038.077zm2.743.468l-.065.056zm.524 5.151l-.061-.06zm-.925 0l-.065.056.009.009zm-.457-4.5l-.065.056zm-2.23.011l-.062-.059zm-.49 4.493l-.061-.06zm-.736-.019a.2.2 0 0 1-.134-.046l-.112.13a.367.367 0 0 0 .245.088zm-.124-.037a.2.2 0 0 1-.046-.134h-.172a.367.367 0 0 0 .087.245zm-.046-.134v-7.4h-.172v7.4zm0-7.4a.156.156 0 0 1 .042-.118l-.121-.121a.326.326 0 0 0-.092.239zm.042-.118a.172.172 0 0 1 .129-.053v-.171a.343.343 0 0 0-.25.1zm.129-.053h.557v-.171h-.557zm.557 0a.137.137 0 0 1 .113.048l.13-.112a.308.308 0 0 0-.244-.108zm.122.057a.137.137 0 0 1 .048.113h.172a.309.309 0 0 0-.108-.243zm.048.113v2.6h.172v-2.6zm.153 2.651a2.076 2.076 0 0 1 .728-.6l-.075-.154a2.248 2.248 0 0 0-.788.649zm.73-.6a2.272 2.272 0 0 1 1.043-.214v-.171a2.438 2.438 0 0 0-1.121.232zm1.043-.214a1.982 1.982 0 0 1 1.6.661l.13-.112a2.152 2.152 0 0 0-1.727-.721zm1.6.663a2.557 2.557 0 0 1 .581 1.761h.171a2.727 2.727 0 0 0-.624-1.874zm.581 1.761v3.144h.171v-3.144zm0 3.144a.173.173 0 0 1-.053.129l.121.121a.345.345 0 0 0 .1-.25zm-.053.129a.156.156 0 0 1-.118.042v.171a.327.327 0 0 0 .239-.092zm-.118.042h-.557v.171h.557zm-.557 0a.2.2 0 0 1-.134-.046l-.112.13a.367.367 0 0 0 .245.088zm-.124-.037a.2.2 0 0 1-.046-.134h-.172a.368.368 0 0 0 .087.245zm-.046-.134V823.8h-.172v3.088zm0-3.088a1.859 1.859 0 0 0-.412-1.284l-.128.114a1.69 1.69 0 0 1 .368 1.169zm-.411-1.283a1.471 1.471 0 0 0-1.169-.464v.171a1.3 1.3 0 0 1 1.039.406zm-1.169-.464a1.566 1.566 0 0 0-1.188.473l.124.118a1.4 1.4 0 0 1 1.064-.419zm-1.188.473a1.779 1.779 0 0 0-.436 1.275h.172a1.609 1.609 0 0 1 .389-1.156zm-.436 1.275v3.088h.172V823.8zm0 3.088a.172.172 0 0 1-.053.129l.121.121a.344.344 0 0 0 .1-.25zm-.053.129a.156.156 0 0 1-.118.042v.171a.327.327 0 0 0 .239-.092zm-.118.042h-.557v.171h.557zm6.449-.505l-.062.059zm-.747-1.94h-.086zm-.012-.379h-.085v.005zm.012-.368l-.086-.006zm.758-1.918l-.061-.06zm3.735.078l-.065.056zm.6 2.5l.061.061zm-4.036.067v-.086a.086.086 0 0 0-.086.086zm0 .1h-.086zm.435 1.238l-.068.053zm1.918.3l.045.073zm.491-.435l-.069-.051zm.145-.156l.039.077h.006zm.9.022l-.055.066zm-.245.747l-.064-.057zm-.858.6l.035.078zm.267-3.189v.086a.086.086 0 0 0 .086-.086zm-.424-1.282l-.066.055zm-2.23 0l-.065-.055zm-.4 1.282h-.086a.086.086 0 0 0 .086.086zm1.528 3.349a2.38 2.38 0 0 1-1.779-.677l-.122.12a2.55 2.55 0 0 0 1.9.728zm-1.778-.676a2.86 2.86 0 0 1-.724-1.886l-.171.009a3.027 3.027 0 0 0 .771 1.995zm-.723-1.884l-.011-.379h-.171l.011.379zm-.011-.374l.011-.368-.172-.005-.011.368zm.011-.365a2.871 2.871 0 0 1 .735-1.864l-.124-.118a3.042 3.042 0 0 0-.782 1.971zm.734-1.864a2.331 2.331 0 0 1 1.756-.687v-.171a2.5 2.5 0 0 0-1.879.74zm1.756-.687a2.307 2.307 0 0 1 1.853.762l.13-.112a2.477 2.477 0 0 0-1.983-.821zm1.854.762a3.03 3.03 0 0 1 .659 2.062h.172a3.2 3.2 0 0 0-.7-2.175zm.659 2.062v.19h.172v-.19zm0 .19a.172.172 0 0 1-.053.129l.121.121a.345.345 0 0 0 .1-.25zm-.053.129a.156.156 0 0 1-.118.042v.171a.327.327 0 0 0 .239-.092zm-.118.042h-3.858v.171h3.858zm-3.944.086v.1h.172v-.1zm0 .1a2.2 2.2 0 0 0 .453 1.287l.135-.106a2.027 2.027 0 0 1-.417-1.189zm.454 1.288a1.418 1.418 0 0 0 1.148.533v-.171a1.247 1.247 0 0 1-1.015-.47zm1.148.533a1.647 1.647 0 0 0 .882-.214l-.09-.146a1.481 1.481 0 0 1-.791.188zm.882-.214a1.777 1.777 0 0 0 .515-.458l-.14-.1a1.613 1.613 0 0 1-.466.412zm.513-.456a1.251 1.251 0 0 1 .081-.1.28.28 0 0 1 .026-.025l.008-.006-.077-.153a.326.326 0 0 0-.083.068 1.55 1.55 0 0 0-.092.113zm.12-.134a.328.328 0 0 1 .146-.021v-.171a.468.468 0 0 0-.234.046zm.146-.021h.547v-.171h-.547zm.547 0a.166.166 0 0 1 .112.036l.11-.132a.337.337 0 0 0-.222-.076zm.112.036a.107.107 0 0 1 .036.09h.171a.277.277 0 0 0-.1-.222zm.036.09a.9.9 0 0 1-.291.534l.128.115a1.055 1.055 0 0 0 .334-.649zm-.291.535a2.42 2.42 0 0 1-.83.581l.072.156a2.6 2.6 0 0 0 .888-.624zm-.829.58a2.964 2.964 0 0 1-1.224.238v.171a3.133 3.133 0 0 0 1.295-.253zm.389-3.111v-.033h-.171v.033zm0-.033a1.9 1.9 0 0 0-.445-1.306l-.129.114a1.731 1.731 0 0 1 .4 1.192zm-.444-1.3a1.466 1.466 0 0 0-1.181-.521v.172a1.3 1.3 0 0 1 1.049.46zm-1.181-.521a1.466 1.466 0 0 0-1.18.521l.131.11a1.3 1.3 0 0 1 1.049-.46zm-1.181.521a1.965 1.965 0 0 0-.422 1.3h.172a1.794 1.794 0 0 1 .382-1.194zm-.422 1.3v.033h.172v-.033zm.086.119h3.055v-.171h-3.055z" fill="#51504f" data-name="Path 10" transform="translate(-1460.636 -807.945)"></path> |
| <path id="Path_11" d="M1519.066 884.011a.581.581 0 0 1-.567-.567 1.151 1.151 0 0 1 .028-.312l7.026-18.328a.881.881 0 0 1 .906-.623h1.926a.882.882 0 0 1 .907.623l7 18.328.057.312a.583.583 0 0 1-.567.567h-1.445a.735.735 0 0 1-.482-.142.9.9 0 0 1-.226-.34l-1.558-4.023h-9.292l-1.558 4.023a.8.8 0 0 1-.255.34.688.688 0 0 1-.453.142zm12.181-6.94l-3.824-10.056-3.823 10.055zm8.184-10.538a.592.592 0 0 1-.652-.651v-1.53a.714.714 0 0 1 .17-.482.656.656 0 0 1 .482-.2h1.785a.677.677 0 0 1 .68.68v1.53a.655.655 0 0 1-.2.481.713.713 0 0 1-.481.17zm.227 17.479a.593.593 0 0 1-.652-.652v-13.428a.611.611 0 0 1 .17-.453.656.656 0 0 1 .482-.2h1.359a.679.679 0 0 1 .652.651v13.427a.655.655 0 0 1-.2.482.613.613 0 0 1-.453.17zm6.861 0a.592.592 0 0 1-.651-.652v-13.4a.715.715 0 0 1 .17-.481.656.656 0 0 1 .482-.2h1.3a.677.677 0 0 1 .68.68v1.246a4.255 4.255 0 0 1 3.966-1.926h1.1a.679.679 0 0 1 .651.651v1.161a.566.566 0 0 1-.2.453.612.612 0 0 1-.453.17h-1.7a3.2 3.2 0 0 0-2.408.907 3.253 3.253 0 0 0-.879 2.408v8.328a.656.656 0 0 1-.2.482.716.716 0 0 1-.482.17zm12.234 0a.593.593 0 0 1-.651-.652v-11.814h-2.408a.592.592 0 0 1-.651-.651v-.963a.611.611 0 0 1 .17-.453.654.654 0 0 1 .481-.2h2.408v-1.417q0-4.816 4.872-4.815h1.586a.679.679 0 0 1 .652.651v.963a.656.656 0 0 1-.2.481.613.613 0 0 1-.453.17h-1.529a2.1 2.1 0 0 0-1.785.68 3.248 3.248 0 0 0-.51 2.011v1.275h6.062V863.7a.613.613 0 0 1 .17-.453.656.656 0 0 1 .482-.2h1.3a.679.679 0 0 1 .652.651v19.659a.655.655 0 0 1-.2.482.613.613 0 0 1-.454.17h-1.3a.592.592 0 0 1-.652-.652v-11.811h-6.062v11.813a.657.657 0 0 1-.2.482.614.614 0 0 1-.454.17zm20.9.283a6.487 6.487 0 0 1-4.844-1.757 6.837 6.837 0 0 1-1.813-4.674l-.029-1.218.029-1.218a6.732 6.732 0 0 1 1.841-4.646 7.389 7.389 0 0 1 9.631 0 6.736 6.736 0 0 1 1.841 4.646q.028.311.028 1.218t-.028 1.218a6.772 6.772 0 0 1-1.841 4.674 6.391 6.391 0 0 1-4.82 1.756zm0-2.181a3.582 3.582 0 0 0 2.8-1.133 4.931 4.931 0 0 0 1.133-3.258q.028-.283.028-1.076t-.028-1.076a4.931 4.931 0 0 0-1.133-3.258 3.582 3.582 0 0 0-2.8-1.133 3.671 3.671 0 0 0-2.833 1.133 4.83 4.83 0 0 0-1.1 3.258l-.028 1.076.028 1.076a4.83 4.83 0 0 0 1.1 3.258 3.671 3.671 0 0 0 2.828 1.132zm13.755 1.9a.846.846 0 0 1-.566-.17 1.321 1.321 0 0 1-.34-.538l-4.023-13.144-.056-.283a.575.575 0 0 1 .17-.425.641.641 0 0 1 .425-.17h1.246a.612.612 0 0 1 .453.17.646.646 0 0 1 .255.312l3.145 10.679 3.371-10.566a.761.761 0 0 1 .255-.4.726.726 0 0 1 .538-.2h.963a.728.728 0 0 1 .539.2.76.76 0 0 1 .255.4l3.371 10.566 3.144-10.679a.655.655 0 0 1 .2-.312.714.714 0 0 1 .482-.17h1.275a.542.542 0 0 1 .4.17.576.576 0 0 1 .17.425l-.057.283-3.994 13.144a1.323 1.323 0 0 1-.34.538.9.9 0 0 1-.6.17h-1.1a.86.86 0 0 1-.935-.708l-3.286-10.141-3.286 10.141a.928.928 0 0 1-.963.708z" fill="#51504f" data-name="Path 11" transform="translate(-1454.66 -838.62)"></path> |
| </g> |
| </g> |
| </g> |
| </svg> |
| |
| </a> |
| </div> |
| <div class="desktop-only navbar__menu-container"> |
| |
| <div class="navbar__menu-content" id="main_navbar"> |
| |
| <div class="navbar__links-container"> |
| |
| <a class="navbar__text-link" href="/community/"> |
| Community |
| </a> |
| |
| <a class="navbar__text-link" href="/meetups/"> |
| Meetups |
| </a> |
| |
| <a class="navbar__text-link" href="/docs/"> |
| Documentation |
| </a> |
| |
| <a class="navbar__text-link" href="/use-cases/"> |
| Use-cases |
| </a> |
| |
| <a class="navbar__text-link" href="/announcements/"> |
| Announcements |
| </a> |
| |
| <a class="navbar__text-link" href="/blog/"> |
| Blog |
| </a> |
| |
| <a class="navbar__text-link" href="/ecosystem/"> |
| Ecosystem |
| </a> |
| |
| </div> |
| |
| |
| |
| </div> |
| |
| </div> |
| <div class="mobile-only navbar__drawer-container"> |
| <button class="navbar__toggle-button" id="navbar-toggle-button"> |
| |
| <div id="hamburger-icon" class="navbar__toggle-button--icon visible"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="26" height="20" viewBox="0 0 26 20"> |
| <g id="Group_1294" data-name="Group 1294" transform="translate(-38.791 291)"> |
| <g id="Group_1291" data-name="Group 1291" transform="translate(39 -291)"> |
| <rect id="Rectangle_461" width="26" height="2" fill="#51504f" data-name="Rectangle 461" rx="1" transform="translate(-.209)"></rect> |
| </g> |
| <g id="Group_1292" data-name="Group 1292" transform="translate(39 -281.822)"> |
| <rect id="Rectangle_462" width="26" height="2" fill="#51504f" data-name="Rectangle 462" rx="1" transform="translate(-.209 -.178)"></rect> |
| </g> |
| <g id="Group_1293" data-name="Group 1293" transform="translate(39 -272.644)"> |
| <rect id="Rectangle_463" width="26" height="2" fill="#51504f" data-name="Rectangle 463" rx="1" transform="translate(-.209 -.356)"></rect> |
| </g> |
| </g> |
| </svg> |
| |
| </div> |
| |
| <div id="close-icon" class="navbar__toggle-button--icon"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="19.799" height="19.799" viewBox="0 0 19.799 19.799"> |
| <g id="Group_1574" data-name="Group 1574" transform="translate(-41.892 290.899)"> |
| <g id="Group_1291" data-name="Group 1291" transform="rotate(-45 -308.114 -187.077)"> |
| <rect id="Rectangle_461" width="26" height="2" fill="#51504f" data-name="Rectangle 461" rx="1" transform="translate(-.209)"></rect> |
| </g> |
| <g id="Group_1292" data-name="Group 1292" transform="rotate(45 372.48 -93.011)"> |
| <rect id="Rectangle_462" width="26" height="2" fill="#51504f" data-name="Rectangle 462" rx="1" transform="translate(-.209 -.178)"></rect> |
| </g> |
| </g> |
| </svg> |
| |
| </div> |
| |
| </button> |
| <div class="navbar__drawer" id="navbar-drawer"> |
| <div class="navbar__menu-content" id="main_navbar"> |
| |
| <div class="navbar__links-container"> |
| |
| <a class="navbar__text-link" href="/community/"> |
| Community |
| </a> |
| |
| <a class="navbar__text-link" href="/meetups/"> |
| Meetups |
| </a> |
| |
| <a class="navbar__text-link" href="/docs/"> |
| Documentation |
| </a> |
| |
| <a class="navbar__text-link" href="/use-cases/"> |
| Use-cases |
| </a> |
| |
| <a class="navbar__text-link" href="/announcements/"> |
| Announcements |
| </a> |
| |
| <a class="navbar__text-link" href="/blog/"> |
| Blog |
| </a> |
| |
| <a class="navbar__text-link" href="/ecosystem/"> |
| Ecosystem |
| </a> |
| |
| |
| </div> |
| |
| |
| </div> |
| </div> |
| </div> |
| </nav> |
| |
| </header> |
| |
| |
| <div class="roadmap container-fluid td-default base-layout"> |
| |
| |
| <div class="content-drawer-wrapper"> |
| <button class="content-drawer__toggle-button" id="content-open-button"> |
| |
| <div id="hamburger-icon" class="content-drawer__toggle-button--icon visible"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="20.005" height="13.879" viewBox="0 0 20.005 13.879"> |
| <g id="Group_1619" data-name="Group 1619" transform="translate(271.132 -418.872)"> |
| <g id="Group_1613" data-name="Group 1613" transform="translate(-266.229 431.045)"> |
| <path id="Path_1337" d="M-218.5 513.215h-14.583a.259.259 0 0 1-.259-.26.259.259 0 0 1 .259-.259h14.583a.26.26 0 0 1 .259.259.259.259 0 0 1-.259.26z" fill="#51504f" data-name="Path 1337" transform="translate(233.345 -512.696)"></path> |
| </g> |
| <g id="Group_1614" data-name="Group 1614" transform="translate(-266.229 425.684)"> |
| <path id="Path_1338" d="M-218.5 471.9h-14.583a.259.259 0 0 1-.259-.26.26.26 0 0 1 .259-.259h14.583a.26.26 0 0 1 .259.259.26.26 0 0 1-.259.26z" fill="#51504f" data-name="Path 1338" transform="translate(233.345 -471.376)"></path> |
| </g> |
| <g id="Group_1615" data-name="Group 1615" transform="translate(-266.229 420.323)"> |
| <path id="Path_1339" d="M-218.5 430.574h-14.583a.259.259 0 0 1-.259-.259.259.259 0 0 1 .259-.259h14.583a.26.26 0 0 1 .259.259.259.259 0 0 1-.259.259z" fill="#51504f" data-name="Path 1339" transform="translate(233.345 -430.055)"></path> |
| </g> |
| <g id="Group_1616" data-name="Group 1616" transform="translate(-271.132 429.814)"> |
| <path id="Path_1340" d="M-269.663 506.145a1.47 1.47 0 0 1-1.469-1.469 1.47 1.47 0 0 1 1.469-1.469 1.47 1.47 0 0 1 1.469 1.469 1.47 1.47 0 0 1-1.469 1.469zm0-2.418a.951.951 0 0 0-.95.95.951.951 0 0 0 .95.95.951.951 0 0 0 .95-.95.951.951 0 0 0-.95-.95z" fill="#51504f" data-name="Path 1340" transform="translate(271.132 -503.208)"></path> |
| </g> |
| <g id="Group_1617" data-name="Group 1617" transform="translate(-271.132 424.556)"> |
| <path id="Path_1341" d="M-269.663 465.62a1.47 1.47 0 0 1-1.469-1.469 1.47 1.47 0 0 1 1.469-1.469 1.47 1.47 0 0 1 1.469 1.469 1.47 1.47 0 0 1-1.469 1.469zm0-2.418a.951.951 0 0 0-.95.95.951.951 0 0 0 .95.95.951.951 0 0 0 .95-.95.951.951 0 0 0-.95-.952z" fill="#51504f" data-name="Path 1341" transform="translate(271.132 -462.683)"></path> |
| </g> |
| <g id="Group_1618" data-name="Group 1618" transform="translate(-271.132 418.872)"> |
| <path id="Path_1342" d="M-269.663 421.809a1.47 1.47 0 0 1-1.469-1.469 1.47 1.47 0 0 1 1.469-1.469 1.47 1.47 0 0 1 1.469 1.469 1.47 1.47 0 0 1-1.469 1.469zm0-2.418a.951.951 0 0 0-.95.95.951.951 0 0 0 .95.95.951.951 0 0 0 .95-.95.951.951 0 0 0-.95-.95z" fill="#51504f" data-name="Path 1342" transform="translate(271.132 -418.872)"></path> |
| </g> |
| </g> |
| </svg> |
| |
| <span class="bodytext__mobile--brownish-grey">Content</span> |
| </div> |
| |
| </button> |
| <nav id="content-navbar" class="navbar navbar--hidden"> |
| <div class="navbar__icon-container"> |
| <a href="/"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="155.314" height="60" viewBox="0 0 155.314 60"> |
| <defs> |
| <clipPath id="clip-path"> |
| <path id="Rectangle_1" d="M0 0h155.314v60H0z" fill="none" data-name="Rectangle 1"></path> |
| </clipPath> |
| </defs> |
| <g id="logo" transform="translate(-1305 -780.355)"> |
| <g id="Group_2" clip-path="url(#clip-path)" data-name="Group 2" transform="translate(1305 780.355)"> |
| <g id="Group_1" data-name="Group 1" transform="translate(.486 .486)"> |
| <path id="Path_1" d="M1307.562 880.867l28.187-28.893a.521.521 0 0 0 .063-.666c-1.714-2.393-4.877-2.808-6.049-4.416-3.472-4.763-4.353-7.459-5.845-7.292a.456.456 0 0 0-.271.143l-10.182 10.438c-5.858 6-6.7 19.225-6.852 30.3a.552.552 0 0 0 .949.386z" fill="#017cee" data-name="Path 1" transform="translate(-1306.613 -822.232)"></path> |
| <path id="Path_2" d="M1405.512 908.489l-28.893-28.189a.521.521 0 0 0-.667-.063c-2.393 1.715-2.808 4.877-4.416 6.049-4.763 3.472-7.459 4.353-7.292 5.845a.456.456 0 0 0 .143.27l10.438 10.182c6 5.858 19.225 6.7 30.3 6.852a.552.552 0 0 0 .387-.946z" fill="#00ad46" data-name="Path 2" transform="translate(-1346.876 -850.567)"></path> |
| <path id="Path_3" d="M1373.909 902.252c-3.28-3.2-4.8-9.53 1.486-22.583-10.219 4.567-13.8 10.57-12.039 12.289z" fill="#04d659" data-name="Path 3" transform="translate(-1345.96 -850.233)"></path> |
| <path id="Path_4" d="M1433.132 782.359l-28.186 28.893a.52.52 0 0 0-.063.666c1.715 2.393 4.876 2.808 6.049 4.416 3.472 4.763 4.354 7.459 5.845 7.292a.454.454 0 0 0 .271-.143l10.182-10.438c5.858-6 6.7-19.225 6.852-30.3a.553.553 0 0 0-.95-.386z" fill="#00c7d4" data-name="Path 4" transform="translate(-1375.21 -782.123)"></path> |
| <path id="Path_5" d="M1426.9 881.155c-3.2 3.28-9.53 4.8-22.584-1.486 4.567 10.219 10.57 13.8 12.289 12.039z" fill="#11e1ee" data-name="Path 5" transform="translate(-1374.875 -850.233)"></path> |
| <path id="Path_6" d="M1307 782.919l28.893 28.186a.521.521 0 0 0 .666.063c2.393-1.715 2.808-4.877 4.416-6.049 4.763-3.472 7.459-4.353 7.292-5.845a.459.459 0 0 0-.143-.271l-10.438-10.182c-6-5.858-19.225-6.7-30.3-6.852a.552.552 0 0 0-.386.95z" fill="#e43921" data-name="Path 6" transform="translate(-1306.766 -781.97)"></path> |
| <path id="Path_7" d="M1405.8 804.711c3.28 3.2 4.8 9.53-1.486 22.584 10.219-4.567 13.8-10.571 12.039-12.289z" fill-rule="evenodd" fill="#ff7557" data-name="Path 7" transform="translate(-1374.875 -797.859)"></path> |
| <path id="Path_8" d="M1329.355 849.266c3.2-3.28 9.53-4.8 22.584 1.486-4.567-10.219-10.57-13.8-12.289-12.039z" fill="#0cb6ff" data-name="Path 8" transform="translate(-1322.503 -821.316)"></path> |
| <circle id="Ellipse_1" cx="1.26" cy="1.26" r="1.26" fill="#4a4848" data-name="Ellipse 1" transform="translate(28.18 28.171)"></circle> |
| <path id="Path_9" d="M1527.558 827.347a.229.229 0 0 1-.223-.223.458.458 0 0 1 .011-.123l2.766-7.214a.346.346 0 0 1 .357-.245h.758a.348.348 0 0 1 .357.245l2.754 7.214.022.123a.228.228 0 0 1-.223.223h-.568a.288.288 0 0 1-.19-.056.352.352 0 0 1-.089-.134l-.613-1.583h-3.657l-.613 1.583a.317.317 0 0 1-.1.134.269.269 0 0 1-.178.056zm4.795-2.732l-1.505-3.958-1.505 3.958zm3.322 4.85a.258.258 0 0 1-.189-.078.241.241 0 0 1-.067-.178v-7.4a.241.241 0 0 1 .067-.178.258.258 0 0 1 .189-.078h.513a.268.268 0 0 1 .256.256v.49a2.118 2.118 0 0 1 1.828-.858 2.092 2.092 0 0 1 1.751.736 3.135 3.135 0 0 1 .636 1.9q.011.122.011.379t-.011.379a3.168 3.168 0 0 1-.636 1.9 2.111 2.111 0 0 1-1.751.736 2.154 2.154 0 0 1-1.806-.836v2.587a.241.241 0 0 1-.067.178.223.223 0 0 1-.179.078zm2.364-2.91a1.324 1.324 0 0 0 1.149-.491 2.266 2.266 0 0 0 .4-1.293q.011-.111.011-.323 0-2.107-1.562-2.107a1.365 1.365 0 0 0-1.159.513 2.111 2.111 0 0 0-.412 1.2l-.012.424.012.435a1.862 1.862 0 0 0 .424 1.149 1.4 1.4 0 0 0 1.148.493zm5.628.9a2.329 2.329 0 0 1-1.015-.223 1.94 1.94 0 0 1-.747-.6 1.487 1.487 0 0 1-.268-.859 1.459 1.459 0 0 1 .6-1.2 3.4 3.4 0 0 1 1.65-.624l1.661-.234v-.323q0-1.137-1.3-1.137a1.4 1.4 0 0 0-.8.212 1.376 1.376 0 0 0-.468.48.305.305 0 0 1-.089.145.18.18 0 0 1-.134.045h-.48a.23.23 0 0 1-.245-.245 1.17 1.17 0 0 1 .245-.6 1.931 1.931 0 0 1 .747-.591 2.7 2.7 0 0 1 1.238-.256 2.351 2.351 0 0 1 1.8.591 2.032 2.032 0 0 1 .547 1.45v3.613a.257.257 0 0 1-.078.19.24.24 0 0 1-.178.067h-.513a.233.233 0 0 1-.257-.256v-.479a1.923 1.923 0 0 1-.714.6 2.557 2.557 0 0 1-1.203.237zm.234-.836a1.579 1.579 0 0 0 1.182-.469 1.881 1.881 0 0 0 .468-1.371v-.312l-1.293.19a2.918 2.918 0 0 0-1.193.379.761.761 0 0 0-.4.658.784.784 0 0 0 .368.691 1.585 1.585 0 0 0 .867.237zm6.643.836a2.556 2.556 0 0 1-1.873-.669 2.738 2.738 0 0 1-.714-1.9l-.011-.446.011-.446a2.7 2.7 0 0 1 .714-1.885 2.531 2.531 0 0 1 1.873-.68 2.917 2.917 0 0 1 1.36.29 2.077 2.077 0 0 1 .825.714 1.7 1.7 0 0 1 .3.848.2.2 0 0 1-.067.178.281.281 0 0 1-.19.067h-.535a.265.265 0 0 1-.168-.045.458.458 0 0 1-.111-.178 1.428 1.428 0 0 0-.535-.758 1.516 1.516 0 0 0-.87-.234 1.45 1.45 0 0 0-1.1.435 1.952 1.952 0 0 0-.435 1.3l-.011.4.011.379a1.969 1.969 0 0 0 .435 1.316 1.446 1.446 0 0 0 1.1.424 1.577 1.577 0 0 0 .87-.223 1.493 1.493 0 0 0 .535-.769.458.458 0 0 1 .111-.178.228.228 0 0 1 .168-.056h.535a.258.258 0 0 1 .19.078.2.2 0 0 1 .067.178 1.75 1.75 0 0 1-.3.847 2.078 2.078 0 0 1-.825.714 2.876 2.876 0 0 1-1.361.302zm4.078-.112a.233.233 0 0 1-.257-.256v-7.4a.241.241 0 0 1 .067-.178.259.259 0 0 1 .19-.078h.557a.267.267 0 0 1 .257.256v2.6a2.167 2.167 0 0 1 .758-.624 2.353 2.353 0 0 1 1.082-.223 2.067 2.067 0 0 1 1.661.691 2.642 2.642 0 0 1 .6 1.818v3.144a.257.257 0 0 1-.078.19.24.24 0 0 1-.178.067h-.557a.233.233 0 0 1-.256-.256V824a1.775 1.775 0 0 0-.39-1.227 1.387 1.387 0 0 0-1.1-.435 1.481 1.481 0 0 0-1.126.446 1.7 1.7 0 0 0-.412 1.215v3.088a.257.257 0 0 1-.078.19.24.24 0 0 1-.178.067zm8.846.112a2.466 2.466 0 0 1-1.84-.7 2.938 2.938 0 0 1-.747-1.94l-.011-.379.011-.368a2.953 2.953 0 0 1 .758-1.918 2.7 2.7 0 0 1 3.735.078 3.114 3.114 0 0 1 .68 2.119v.19a.257.257 0 0 1-.078.189.241.241 0 0 1-.178.067h-3.858v.1a2.11 2.11 0 0 0 .435 1.238 1.332 1.332 0 0 0 1.081.5 1.563 1.563 0 0 0 .836-.2 1.7 1.7 0 0 0 .491-.435.6.6 0 0 1 .145-.156.391.391 0 0 1 .19-.033h.547a.252.252 0 0 1 .167.056.192.192 0 0 1 .067.156.975.975 0 0 1-.312.591 2.51 2.51 0 0 1-.859.6 3.049 3.049 0 0 1-1.26.248zm1.527-3.434v-.033a1.817 1.817 0 0 0-.424-1.249 1.512 1.512 0 0 0-2.23 0 1.883 1.883 0 0 0-.4 1.249v.033z" fill="#51504f" data-name="Path 9" transform="translate(-1460.834 -808.144)"></path> |
| <path id="Path_10" d="M1527.2 827.081l-.061.061zm-.056-.279l-.08-.031zm2.766-7.214l.08.031zm1.472 0l-.081.029zm2.754 7.214l.084-.015a.064.064 0 0 0 0-.015zm.022.123h.086v-.015zm-.067.156l.06.061zm-.914.011l-.061.061.006.005zm-.089-.134l.081-.027zm-.613-1.583l.08-.031a.086.086 0 0 0-.08-.055zm-3.657 0v-.086a.086.086 0 0 0-.08.055zm-.613 1.583l-.08-.031zm-.1.134l.055.066zm4.047-2.676v.086a.086.086 0 0 0 .08-.116zm-1.505-3.958l.08-.03a.086.086 0 0 0-.16 0zm-1.505 3.958l-.08-.03a.086.086 0 0 0 .08.116zm-1.784 2.646a.128.128 0 0 1-.1-.042l-.122.121a.3.3 0 0 0 .217.092zm-.1-.042a.129.129 0 0 1-.042-.1h-.171a.3.3 0 0 0 .092.217zm-.042-.1a.38.38 0 0 1 .007-.1l-.163-.054a.514.514 0 0 0-.016.15zm.005-.092l2.765-7.214-.16-.061-2.765 7.214zm2.766-7.216a.283.283 0 0 1 .1-.143.3.3 0 0 1 .174-.046v-.172a.47.47 0 0 0-.271.076.453.453 0 0 0-.166.226zm.276-.189h.758v-.172h-.758zm.758 0a.3.3 0 0 1 .175.046.283.283 0 0 1 .1.143l.161-.059a.451.451 0 0 0-.166-.226.47.47 0 0 0-.272-.076zm.277.19l2.754 7.214.16-.061-2.754-7.214zm2.75 7.2l.022.123.169-.031-.022-.123zm.021.107a.13.13 0 0 1-.042.1l.121.121a.3.3 0 0 0 .092-.217zm-.042.1a.13.13 0 0 1-.1.042v.171a.3.3 0 0 0 .217-.092zm-.1.042h-.568v.171h.568zm-.568 0a.206.206 0 0 1-.135-.036l-.11.132a.373.373 0 0 0 .245.076zm-.129-.031a.262.262 0 0 1-.069-.1l-.162.054a.431.431 0 0 0 .11.167zm-.07-.1l-.613-1.584-.16.062.613 1.583zm-.693-1.638h-3.657v.171h3.657zm-3.737.055l-.614 1.584.16.062.613-1.583zm-.615 1.587a.235.235 0 0 1-.075.1l.111.13a.4.4 0 0 0 .126-.172zm-.074.1a.185.185 0 0 1-.124.036v.171a.354.354 0 0 0 .233-.076zm-.124.036h-.569v.171h.569zm4.306-2.677l-1.505-3.958-.16.061 1.505 3.958zm-1.666-3.958l-1.505 3.958.16.061 1.505-3.958zm-1.425 4.075h3.01v-.171h-3.01zm6.143 4.687l-.06.061zm0-7.761l.061.061zm.881 0l-.065.056.01.009zm.078.669h-.086a.086.086 0 0 0 .155.051zm3.579-.123l-.067.053zm.636 1.9h-.086zm0 .758l-.085-.007zm-.636 1.9l.067.054zm-3.557-.1l.068-.052a.086.086 0 0 0-.154.052zm-.067 2.765l-.061-.06zm2.787-3.323l-.069-.051zm.4-1.293l-.085-.008v.005zm-2.709-1.918l-.068-.052zm-.413 1.2h-.086zm-.011.423h-.085zm.011.435h-.086zm.424 1.149l.066-.054zm-1.216 3.315a.173.173 0 0 1-.129-.053l-.121.121a.342.342 0 0 0 .25.1zm-.129-.053a.157.157 0 0 1-.042-.118h-.172a.325.325 0 0 0 .092.239zm-.042-.118v-7.4h-.172v7.4zm0-7.4a.157.157 0 0 1 .042-.118l-.121-.121a.324.324 0 0 0-.092.239zm.042-.118a.173.173 0 0 1 .129-.053v-.172a.342.342 0 0 0-.25.1zm.129-.053h.513v-.172h-.513zm.513 0a.137.137 0 0 1 .113.048l.13-.111a.309.309 0 0 0-.244-.108zm.123.058a.137.137 0 0 1 .048.113h.171a.309.309 0 0 0-.108-.243zm.048.113v.49h.171v-.49zm.155.541a2.033 2.033 0 0 1 1.759-.823v-.171a2.2 2.2 0 0 0-1.9.894zm1.759-.823a2.007 2.007 0 0 1 1.683.7l.135-.106a2.177 2.177 0 0 0-1.818-.768zm1.683.7a3.045 3.045 0 0 1 .617 1.845l.171-.007a3.218 3.218 0 0 0-.654-1.946zm.617 1.85c.007.078.011.2.011.372h.171c0-.171 0-.3-.012-.387zm.011.372c0 .171 0 .294-.011.372l.17.015c.008-.086.012-.216.012-.387zm-.011.376a3.08 3.08 0 0 1-.617 1.846l.134.106a3.25 3.25 0 0 0 .654-1.945zm-.617 1.845a2.025 2.025 0 0 1-1.683.7v.171a2.2 2.2 0 0 0 1.817-.768zm-1.683.7a2.068 2.068 0 0 1-1.739-.8l-.136.1a2.239 2.239 0 0 0 1.874.87zm-1.892-.75v2.587h.172v-2.587zm0 2.587a.156.156 0 0 1-.042.118l.121.121a.325.325 0 0 0 .092-.239zm-.046.123a.138.138 0 0 1-.114.048v.172a.308.308 0 0 0 .244-.108zm-.114.048h-.546v.172h.546zm1.817-2.739a1.408 1.408 0 0 0 1.218-.526l-.138-.1a1.24 1.24 0 0 1-1.079.455zm1.217-.525a2.355 2.355 0 0 0 .419-1.341l-.171-.007a2.182 2.182 0 0 1-.385 1.246zm.418-1.336c.008-.079.012-.19.012-.332h-.172c0 .14 0 .245-.011.315zm.012-.332a2.726 2.726 0 0 0-.407-1.632 1.448 1.448 0 0 0-1.24-.562v.171a1.278 1.278 0 0 1 1.1.492 2.565 2.565 0 0 1 .374 1.53zm-1.647-2.193a1.452 1.452 0 0 0-1.228.547l.136.1a1.282 1.282 0 0 1 1.091-.479zm-1.228.547a2.2 2.2 0 0 0-.43 1.252l.172.008a2.028 2.028 0 0 1 .4-1.157zm-.43 1.254l-.011.424h.171l.011-.424zm-.011.428l.011.435h.172l-.011-.435zm.011.436a1.95 1.95 0 0 0 .443 1.2l.133-.109a1.776 1.776 0 0 1-.4-1.1zm.443 1.2a1.484 1.484 0 0 0 1.214.522v-.171a1.314 1.314 0 0 1-1.082-.459zm5.828 1.117l.037-.077zm-.747-.6l-.07.049zm.335-2.063l.052.068zm1.65-.624l.012.085zm1.661-.234l.012.085a.086.086 0 0 0 .074-.085zm-2.107-1.249l.046.072zm-.468.48l-.075-.042a.083.083 0 0 0-.006.015zm-.089.145l-.054-.067-.007.006zm-.792-.022l-.065.056.009.009zm-.067-.178h-.086zm.245-.6l-.07-.049zm.747-.591l.038.077zm3.033.334l-.063.058zm.468 5.252l.06.061zm-.881 0l-.065.056a.043.043 0 0 0 .009.009zm-.067-.669h.086a.086.086 0 0 0-.156-.048zm-.714.6l-.04-.076zm.223-1.059l-.062-.06zm.468-1.684h.086a.086.086 0 0 0-.1-.085zm-1.293.189l.012.085zm-1.193.379l.046.072zm-.033 1.349l-.047.071zm.635.985a2.241 2.241 0 0 1-.978-.215l-.074.155a2.412 2.412 0 0 0 1.051.231zm-.978-.215a1.859 1.859 0 0 1-.715-.576l-.138.1a2.024 2.024 0 0 0 .779.629zm-.713-.573a1.4 1.4 0 0 1-.253-.81h-.172a1.571 1.571 0 0 0 .283.907zm-.253-.81a1.374 1.374 0 0 1 .569-1.136l-.105-.135a1.544 1.544 0 0 0-.635 1.272zm.569-1.137a3.316 3.316 0 0 1 1.609-.607l-.024-.17a3.481 3.481 0 0 0-1.691.642zm1.609-.607l1.661-.234-.024-.17-1.662.234zm1.735-.319v-.323h-.171v.323zm0-.323a1.156 1.156 0 0 0-.355-.917 1.536 1.536 0 0 0-1.035-.306v.172a1.37 1.37 0 0 1 .922.263.986.986 0 0 1 .3.788zm-1.39-1.223a1.486 1.486 0 0 0-.851.227l.1.142a1.316 1.316 0 0 1 .755-.2zm-.849.226a1.452 1.452 0 0 0-.5.51l.15.084a1.286 1.286 0 0 1 .44-.449zm-.5.524a.226.226 0 0 1-.062.105l.107.134a.391.391 0 0 0 .117-.185zm-.068.112a.1.1 0 0 1-.073.019v.171a.266.266 0 0 0 .194-.07zm-.073.019h-.48v.171h.48zm-.48 0a.18.18 0 0 1-.122-.046l-.112.13a.352.352 0 0 0 .234.087zm-.113-.037a.18.18 0 0 1-.047-.123h-.171a.352.352 0 0 0 .087.234zm-.047-.119a1.1 1.1 0 0 1 .23-.557l-.14-.1a1.253 1.253 0 0 0-.261.648zm.23-.556a1.843 1.843 0 0 1 .715-.564l-.075-.154a2.018 2.018 0 0 0-.78.618zm.716-.564a2.611 2.611 0 0 1 1.2-.247v-.171a2.781 2.781 0 0 0-1.277.266zm1.2-.247a2.268 2.268 0 0 1 1.732.563l.126-.116a2.435 2.435 0 0 0-1.858-.618zm1.733.564a1.945 1.945 0 0 1 .523 1.391h.171a2.117 2.117 0 0 0-.57-1.508zm.523 1.391v3.613h.171v-3.613zm0 3.613a.172.172 0 0 1-.053.129l.121.121a.344.344 0 0 0 .1-.25zm-.053.129a.157.157 0 0 1-.118.042v.171a.326.326 0 0 0 .239-.092zm-.118.042h-.513v.171h.513zm-.513 0a.2.2 0 0 1-.134-.046l-.111.13a.367.367 0 0 0 .245.088zm-.124-.037a.194.194 0 0 1-.047-.134h-.171a.366.366 0 0 0 .087.245zm-.047-.134v-.479h-.171v.479zm-.156-.528a1.846 1.846 0 0 1-.683.575l.079.152a2.012 2.012 0 0 0 .745-.629zm-.683.575a2.476 2.476 0 0 1-1.153.236v.171a2.644 2.644 0 0 0 1.233-.255zm-.919-.429a1.666 1.666 0 0 0 1.244-.494l-.123-.12a1.493 1.493 0 0 1-1.121.442zm1.244-.494a1.969 1.969 0 0 0 .492-1.431h-.171a1.8 1.8 0 0 1-.444 1.312zm.492-1.431v-.312h-.171v.312zm-.1-.4l-1.293.189.025.17 1.293-.189zm-1.293.189a3 3 0 0 0-1.228.393l.095.143a2.837 2.837 0 0 1 1.158-.365zm-1.227.392a.845.845 0 0 0-.441.73h.172a.676.676 0 0 1 .362-.586zm-.441.73a.869.869 0 0 0 .406.762l.095-.142a.7.7 0 0 1-.33-.62zm.408.764a1.673 1.673 0 0 0 .916.247v-.171a1.5 1.5 0 0 1-.823-.221zm5.686.329l-.061.06zm-.714-1.9h-.085zm-.011-.446h-.085zm.011-.446h-.085zm.714-1.885l.061.061zm3.234-.39l-.04.076zm.825.713l-.073.046zm.3.848h-.086zm-.067.178l.056.065zm-.892.022l.054-.067zm-.112-.178l-.081.029zm-.535-.758l-.048.071zm-1.974.2l-.062-.059zm-.435 1.3h-.086zm-.011.4h-.086zm.011.379h-.086zm.435 1.316l-.062.059zm1.974.2l.046.072zm.535-.769l-.079-.033zm.112-.178l.054.067.007-.006zm.892.022l-.061.06zm.067.178l-.085-.009zm-.3.847l-.072-.046zm-.825.714l-.04-.076zm-1.36.2a2.471 2.471 0 0 1-1.814-.644l-.12.122a2.64 2.64 0 0 0 1.933.694zm-1.813-.643a2.653 2.653 0 0 1-.689-1.839l-.171.006a2.822 2.822 0 0 0 .738 1.952zm-.689-1.838l-.011-.446h-.171l.011.446zm-.011-.442l.011-.446h-.171l-.011.446zm.011-.445a2.611 2.611 0 0 1 .689-1.827l-.122-.121a2.78 2.78 0 0 0-.738 1.942zm.689-1.827a2.447 2.447 0 0 1 1.813-.655v-.171a2.617 2.617 0 0 0-1.934.705zm1.813-.655a2.836 2.836 0 0 1 1.32.28l.079-.152a3 3 0 0 0-1.4-.3zm1.32.28a1.99 1.99 0 0 1 .792.683l.145-.091a2.158 2.158 0 0 0-.858-.744zm.793.685a1.617 1.617 0 0 1 .287.8l.171-.009a1.789 1.789 0 0 0-.315-.89zm.287.809a.11.11 0 0 1-.037.1l.112.13a.281.281 0 0 0 .1-.252zm-.037.1a.2.2 0 0 1-.134.046v.171a.369.369 0 0 0 .246-.088zm-.134.046h-.535v.171h.535zm-.535 0a.184.184 0 0 1-.114-.026l-.107.134a.345.345 0 0 0 .221.064zm-.114-.026a.389.389 0 0 1-.086-.144l-.158.066a.533.533 0 0 0 .137.212zm-.084-.14a1.514 1.514 0 0 0-.57-.8l-.093.144a1.343 1.343 0 0 1 .5.715zm-.568-.8a1.6 1.6 0 0 0-.918-.249v.171a1.435 1.435 0 0 1 .822.219zm-.918-.249a1.535 1.535 0 0 0-1.166.462l.124.118a1.364 1.364 0 0 1 1.042-.408zm-1.166.462a2.036 2.036 0 0 0-.458 1.36l.171.006a1.872 1.872 0 0 1 .411-1.249zm-.458 1.361l-.011.4h.171l.011-.4zm-.011.406l.011.379.171-.005-.011-.379zm.011.38a2.052 2.052 0 0 0 .458 1.371l.124-.118a1.889 1.889 0 0 1-.411-1.26zm.458 1.371a1.533 1.533 0 0 0 1.166.451v-.172a1.363 1.363 0 0 1-1.042-.4zm1.166.451a1.661 1.661 0 0 0 .916-.237l-.093-.144a1.491 1.491 0 0 1-.823.209zm.918-.238a1.576 1.576 0 0 0 .568-.812l-.162-.057a1.409 1.409 0 0 1-.5.727zm.566-.807a.39.39 0 0 1 .086-.144l-.107-.134a.533.533 0 0 0-.137.213zm.093-.151a.144.144 0 0 1 .107-.031v-.171a.31.31 0 0 0-.228.081zm.107-.031h.535v-.171h-.535zm.535 0a.173.173 0 0 1 .129.053l.121-.121a.344.344 0 0 0-.25-.1zm.134.057a.11.11 0 0 1 .037.1l.17.017a.281.281 0 0 0-.1-.252zm.037.109a1.664 1.664 0 0 1-.288.806l.144.092a1.839 1.839 0 0 0 .315-.889zm-.288.806a1.989 1.989 0 0 1-.792.683l.079.152a2.162 2.162 0 0 0 .858-.744zm-.793.684a2.8 2.8 0 0 1-1.32.28v.171a2.96 2.96 0 0 0 1.4-.3zm2.568.187l-.065.056.01.009zm0-7.772l.061.06zm.926 0l-.065.056.009.009zm.078 2.776h-.085a.086.086 0 0 0 .153.053zm.758-.624l.038.077zm2.743.468l-.065.056zm.524 5.151l-.061-.06zm-.925 0l-.065.056.009.009zm-.457-4.5l-.065.056zm-2.23.011l-.062-.059zm-.49 4.493l-.061-.06zm-.736-.019a.2.2 0 0 1-.134-.046l-.112.13a.367.367 0 0 0 .245.088zm-.124-.037a.2.2 0 0 1-.046-.134h-.172a.367.367 0 0 0 .087.245zm-.046-.134v-7.4h-.172v7.4zm0-7.4a.156.156 0 0 1 .042-.118l-.121-.121a.326.326 0 0 0-.092.239zm.042-.118a.172.172 0 0 1 .129-.053v-.171a.343.343 0 0 0-.25.1zm.129-.053h.557v-.171h-.557zm.557 0a.137.137 0 0 1 .113.048l.13-.112a.308.308 0 0 0-.244-.108zm.122.057a.137.137 0 0 1 .048.113h.172a.309.309 0 0 0-.108-.243zm.048.113v2.6h.172v-2.6zm.153 2.651a2.076 2.076 0 0 1 .728-.6l-.075-.154a2.248 2.248 0 0 0-.788.649zm.73-.6a2.272 2.272 0 0 1 1.043-.214v-.171a2.438 2.438 0 0 0-1.121.232zm1.043-.214a1.982 1.982 0 0 1 1.6.661l.13-.112a2.152 2.152 0 0 0-1.727-.721zm1.6.663a2.557 2.557 0 0 1 .581 1.761h.171a2.727 2.727 0 0 0-.624-1.874zm.581 1.761v3.144h.171v-3.144zm0 3.144a.173.173 0 0 1-.053.129l.121.121a.345.345 0 0 0 .1-.25zm-.053.129a.156.156 0 0 1-.118.042v.171a.327.327 0 0 0 .239-.092zm-.118.042h-.557v.171h.557zm-.557 0a.2.2 0 0 1-.134-.046l-.112.13a.367.367 0 0 0 .245.088zm-.124-.037a.2.2 0 0 1-.046-.134h-.172a.368.368 0 0 0 .087.245zm-.046-.134V823.8h-.172v3.088zm0-3.088a1.859 1.859 0 0 0-.412-1.284l-.128.114a1.69 1.69 0 0 1 .368 1.169zm-.411-1.283a1.471 1.471 0 0 0-1.169-.464v.171a1.3 1.3 0 0 1 1.039.406zm-1.169-.464a1.566 1.566 0 0 0-1.188.473l.124.118a1.4 1.4 0 0 1 1.064-.419zm-1.188.473a1.779 1.779 0 0 0-.436 1.275h.172a1.609 1.609 0 0 1 .389-1.156zm-.436 1.275v3.088h.172V823.8zm0 3.088a.172.172 0 0 1-.053.129l.121.121a.344.344 0 0 0 .1-.25zm-.053.129a.156.156 0 0 1-.118.042v.171a.327.327 0 0 0 .239-.092zm-.118.042h-.557v.171h.557zm6.449-.505l-.062.059zm-.747-1.94h-.086zm-.012-.379h-.085v.005zm.012-.368l-.086-.006zm.758-1.918l-.061-.06zm3.735.078l-.065.056zm.6 2.5l.061.061zm-4.036.067v-.086a.086.086 0 0 0-.086.086zm0 .1h-.086zm.435 1.238l-.068.053zm1.918.3l.045.073zm.491-.435l-.069-.051zm.145-.156l.039.077h.006zm.9.022l-.055.066zm-.245.747l-.064-.057zm-.858.6l.035.078zm.267-3.189v.086a.086.086 0 0 0 .086-.086zm-.424-1.282l-.066.055zm-2.23 0l-.065-.055zm-.4 1.282h-.086a.086.086 0 0 0 .086.086zm1.528 3.349a2.38 2.38 0 0 1-1.779-.677l-.122.12a2.55 2.55 0 0 0 1.9.728zm-1.778-.676a2.86 2.86 0 0 1-.724-1.886l-.171.009a3.027 3.027 0 0 0 .771 1.995zm-.723-1.884l-.011-.379h-.171l.011.379zm-.011-.374l.011-.368-.172-.005-.011.368zm.011-.365a2.871 2.871 0 0 1 .735-1.864l-.124-.118a3.042 3.042 0 0 0-.782 1.971zm.734-1.864a2.331 2.331 0 0 1 1.756-.687v-.171a2.5 2.5 0 0 0-1.879.74zm1.756-.687a2.307 2.307 0 0 1 1.853.762l.13-.112a2.477 2.477 0 0 0-1.983-.821zm1.854.762a3.03 3.03 0 0 1 .659 2.062h.172a3.2 3.2 0 0 0-.7-2.175zm.659 2.062v.19h.172v-.19zm0 .19a.172.172 0 0 1-.053.129l.121.121a.345.345 0 0 0 .1-.25zm-.053.129a.156.156 0 0 1-.118.042v.171a.327.327 0 0 0 .239-.092zm-.118.042h-3.858v.171h3.858zm-3.944.086v.1h.172v-.1zm0 .1a2.2 2.2 0 0 0 .453 1.287l.135-.106a2.027 2.027 0 0 1-.417-1.189zm.454 1.288a1.418 1.418 0 0 0 1.148.533v-.171a1.247 1.247 0 0 1-1.015-.47zm1.148.533a1.647 1.647 0 0 0 .882-.214l-.09-.146a1.481 1.481 0 0 1-.791.188zm.882-.214a1.777 1.777 0 0 0 .515-.458l-.14-.1a1.613 1.613 0 0 1-.466.412zm.513-.456a1.251 1.251 0 0 1 .081-.1.28.28 0 0 1 .026-.025l.008-.006-.077-.153a.326.326 0 0 0-.083.068 1.55 1.55 0 0 0-.092.113zm.12-.134a.328.328 0 0 1 .146-.021v-.171a.468.468 0 0 0-.234.046zm.146-.021h.547v-.171h-.547zm.547 0a.166.166 0 0 1 .112.036l.11-.132a.337.337 0 0 0-.222-.076zm.112.036a.107.107 0 0 1 .036.09h.171a.277.277 0 0 0-.1-.222zm.036.09a.9.9 0 0 1-.291.534l.128.115a1.055 1.055 0 0 0 .334-.649zm-.291.535a2.42 2.42 0 0 1-.83.581l.072.156a2.6 2.6 0 0 0 .888-.624zm-.829.58a2.964 2.964 0 0 1-1.224.238v.171a3.133 3.133 0 0 0 1.295-.253zm.389-3.111v-.033h-.171v.033zm0-.033a1.9 1.9 0 0 0-.445-1.306l-.129.114a1.731 1.731 0 0 1 .4 1.192zm-.444-1.3a1.466 1.466 0 0 0-1.181-.521v.172a1.3 1.3 0 0 1 1.049.46zm-1.181-.521a1.466 1.466 0 0 0-1.18.521l.131.11a1.3 1.3 0 0 1 1.049-.46zm-1.181.521a1.965 1.965 0 0 0-.422 1.3h.172a1.794 1.794 0 0 1 .382-1.194zm-.422 1.3v.033h.172v-.033zm.086.119h3.055v-.171h-3.055z" fill="#51504f" data-name="Path 10" transform="translate(-1460.636 -807.945)"></path> |
| <path id="Path_11" d="M1519.066 884.011a.581.581 0 0 1-.567-.567 1.151 1.151 0 0 1 .028-.312l7.026-18.328a.881.881 0 0 1 .906-.623h1.926a.882.882 0 0 1 .907.623l7 18.328.057.312a.583.583 0 0 1-.567.567h-1.445a.735.735 0 0 1-.482-.142.9.9 0 0 1-.226-.34l-1.558-4.023h-9.292l-1.558 4.023a.8.8 0 0 1-.255.34.688.688 0 0 1-.453.142zm12.181-6.94l-3.824-10.056-3.823 10.055zm8.184-10.538a.592.592 0 0 1-.652-.651v-1.53a.714.714 0 0 1 .17-.482.656.656 0 0 1 .482-.2h1.785a.677.677 0 0 1 .68.68v1.53a.655.655 0 0 1-.2.481.713.713 0 0 1-.481.17zm.227 17.479a.593.593 0 0 1-.652-.652v-13.428a.611.611 0 0 1 .17-.453.656.656 0 0 1 .482-.2h1.359a.679.679 0 0 1 .652.651v13.427a.655.655 0 0 1-.2.482.613.613 0 0 1-.453.17zm6.861 0a.592.592 0 0 1-.651-.652v-13.4a.715.715 0 0 1 .17-.481.656.656 0 0 1 .482-.2h1.3a.677.677 0 0 1 .68.68v1.246a4.255 4.255 0 0 1 3.966-1.926h1.1a.679.679 0 0 1 .651.651v1.161a.566.566 0 0 1-.2.453.612.612 0 0 1-.453.17h-1.7a3.2 3.2 0 0 0-2.408.907 3.253 3.253 0 0 0-.879 2.408v8.328a.656.656 0 0 1-.2.482.716.716 0 0 1-.482.17zm12.234 0a.593.593 0 0 1-.651-.652v-11.814h-2.408a.592.592 0 0 1-.651-.651v-.963a.611.611 0 0 1 .17-.453.654.654 0 0 1 .481-.2h2.408v-1.417q0-4.816 4.872-4.815h1.586a.679.679 0 0 1 .652.651v.963a.656.656 0 0 1-.2.481.613.613 0 0 1-.453.17h-1.529a2.1 2.1 0 0 0-1.785.68 3.248 3.248 0 0 0-.51 2.011v1.275h6.062V863.7a.613.613 0 0 1 .17-.453.656.656 0 0 1 .482-.2h1.3a.679.679 0 0 1 .652.651v19.659a.655.655 0 0 1-.2.482.613.613 0 0 1-.454.17h-1.3a.592.592 0 0 1-.652-.652v-11.811h-6.062v11.813a.657.657 0 0 1-.2.482.614.614 0 0 1-.454.17zm20.9.283a6.487 6.487 0 0 1-4.844-1.757 6.837 6.837 0 0 1-1.813-4.674l-.029-1.218.029-1.218a6.732 6.732 0 0 1 1.841-4.646 7.389 7.389 0 0 1 9.631 0 6.736 6.736 0 0 1 1.841 4.646q.028.311.028 1.218t-.028 1.218a6.772 6.772 0 0 1-1.841 4.674 6.391 6.391 0 0 1-4.82 1.756zm0-2.181a3.582 3.582 0 0 0 2.8-1.133 4.931 4.931 0 0 0 1.133-3.258q.028-.283.028-1.076t-.028-1.076a4.931 4.931 0 0 0-1.133-3.258 3.582 3.582 0 0 0-2.8-1.133 3.671 3.671 0 0 0-2.833 1.133 4.83 4.83 0 0 0-1.1 3.258l-.028 1.076.028 1.076a4.83 4.83 0 0 0 1.1 3.258 3.671 3.671 0 0 0 2.828 1.132zm13.755 1.9a.846.846 0 0 1-.566-.17 1.321 1.321 0 0 1-.34-.538l-4.023-13.144-.056-.283a.575.575 0 0 1 .17-.425.641.641 0 0 1 .425-.17h1.246a.612.612 0 0 1 .453.17.646.646 0 0 1 .255.312l3.145 10.679 3.371-10.566a.761.761 0 0 1 .255-.4.726.726 0 0 1 .538-.2h.963a.728.728 0 0 1 .539.2.76.76 0 0 1 .255.4l3.371 10.566 3.144-10.679a.655.655 0 0 1 .2-.312.714.714 0 0 1 .482-.17h1.275a.542.542 0 0 1 .4.17.576.576 0 0 1 .17.425l-.057.283-3.994 13.144a1.323 1.323 0 0 1-.34.538.9.9 0 0 1-.6.17h-1.1a.86.86 0 0 1-.935-.708l-3.286-10.141-3.286 10.141a.928.928 0 0 1-.963.708z" fill="#51504f" data-name="Path 11" transform="translate(-1454.66 -838.62)"></path> |
| </g> |
| </g> |
| </g> |
| </svg> |
| |
| </a> |
| </div> |
| |
| <div id="content-close-button"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="19.799" height="19.799" viewBox="0 0 19.799 19.799"> |
| <g id="Group_1574" data-name="Group 1574" transform="translate(-41.892 290.899)"> |
| <g id="Group_1291" data-name="Group 1291" transform="rotate(-45 -308.114 -187.077)"> |
| <rect id="Rectangle_461" width="26" height="2" fill="#51504f" data-name="Rectangle 461" rx="1" transform="translate(-.209)"></rect> |
| </g> |
| <g id="Group_1292" data-name="Group 1292" transform="rotate(45 372.48 -93.011)"> |
| <rect id="Rectangle_462" width="26" height="2" fill="#51504f" data-name="Rectangle 462" rx="1" transform="translate(-.209 -.178)"></rect> |
| </g> |
| </g> |
| </svg> |
| |
| </div> |
| |
| </nav> |
| <div class="content-drawer-container" id="content-drawer"> |
| <div class="content-drawer"> |
| <div class="td-sidebar"> |
| |
| <div id="docs-version-selector" class="docs-version-selector sidebar__version-selector"> |
| <a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" |
| aria-expanded="false"> |
| <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.3.2</span> |
| </a> |
| <div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink"> |
| |
| </div> |
| <script type="application/x-template" id="version-item-template"> |
| <a class="dropdown-item"></a> |
| </script> |
| </div> |
| |
| <div class="searchb-box"> |
| <form class="search-form" action="../../../search.html" method="get"> |
| <input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16"> |
| <input type="hidden" name="check_keywords" value="yes" /> |
| <input type="hidden" name="area" value="default" /> |
| |
| <button class="search-form__button" type="submit"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20"> |
| <g id="Group_1579" data-name="Group 1579" transform="translate(-41.001 -41)"> |
| <path id="Path_169" d="M71.415 64.687a7.215 7.215 0 1 0-6.729 6.728 7.222 7.222 0 0 0 6.729-6.728z" fill="none" data-name="Path 169" transform="translate(-14.277 -14.276)"></path> |
| <path id="Path_170" d="M60.863 59.8l-6.093-6.09a7.78 7.78 0 1 0-1.06 1.06l6.09 6.093a.468.468 0 0 0 .662 0l.4-.4a.468.468 0 0 0 .001-.663zM42.512 49.183a6.274 6.274 0 1 1 5.851 5.85 6.28 6.28 0 0 1-5.851-5.85z" fill="#51504f" data-name="Path 170"></path> |
| </g> |
| </svg> |
| </button> |
| </form> |
| </div> |
| |
| |
| <style> |
| .searchb-box { |
| margin-bottom: 26px; |
| } |
| .searchb-box .search-form { |
| width: 100%; |
| margin-top: 20px; |
| } |
| </style> |
| |
| |
| <div class="toctree" role="navigation" aria-label="main navigation"> |
| <p class="caption" role="heading"><span class="caption-text">Content</span></p> |
| <ul> |
| <li class="toctree-l1"><a class="reference internal" href="../../../index.html">Home</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../start/index.html">Quick Start</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../installation/index.html">Installation</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../upgrading-from-1-10/index.html">Upgrading from 1.10 to 2</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../tutorial_taskflow_api.html">Tutorial on the TaskFlow API</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../concepts/index.html">Concepts</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../executor/index.html">Executor</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../dag-run.html">DAG Runs</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../security/index.html">Security</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../logging-monitoring/index.html">Logging & Monitoring</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time Zones</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../usage-cli.html">Using the CLI</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../kubernetes.html">Kubernetes</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../listeners.html">Listeners</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../dag-serialization.html">DAG Serialization</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../modules_management.html">Modules Management</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../release-process.html">Release Policies</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../release_notes.html">Release Notes</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../best-practices.html">Best Practices</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../production-deployment.html">Production Deployment</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../privacy_notice.html">Privacy Notice</a></li> |
| </ul> |
| <p class="caption" role="heading"><span class="caption-text">References</span></p> |
| <ul> |
| <li class="toctree-l1"><a class="reference internal" href="../../../operators-and-hooks-ref.html">Operators and hooks</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../cli-and-env-variables-ref.html">CLI</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../templates-ref.html">Templates</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../python-api-ref.html">Python API</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../stable-rest-api-ref.html">Stable REST API</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../configurations-ref.html">Configurations</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../extra-packages-ref.html">Extra packages</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../migrations-ref.html">Database Migrations</a></li> |
| </ul> |
| |
| </div> |
| |
| <style type="text/css"> |
| .toctree { |
| } |
| .toctree li { |
| list-style: none; |
| } |
| .toctree .caption { |
| font-family: Roboto; |
| font-size: 18px; |
| font-weight: bold; |
| font-stretch: normal; |
| font-style: normal; |
| line-height: 1.33; |
| letter-spacing: normal; |
| color: #51504f; |
| padding-bottom: 13px; |
| text-transform: uppercase; |
| margin-bottom: 0; |
| } |
| .toctree .current > a:not([href="#"]) { |
| color: #017cee; |
| } |
| .toctree > ul { |
| padding-left: 0; |
| } |
| .toctree ul { |
| padding-left: 15px; |
| display: none; |
| } |
| .toctree > ul, |
| .toctree li.current > ul { |
| display: block; |
| } |
| |
| .toctree a .toctree-expand { |
| display: inline-block; |
| position: relative; |
| height: 1em; |
| } |
| |
| .toctree a .toctree-expand:before { |
| position: absolute; |
| top: 6px; |
| left: -12px; |
| content: '►'; |
| font-size: 7px; |
| } |
| .toctree .current > a > .toctree-expand:before { |
| content: '▼'; |
| } |
| |
| .toctree .current { |
| color: #017cee; |
| } |
| .toctree li { |
| font-family: Roboto; |
| font-size: 16px; |
| font-weight: normal; |
| font-stretch: normal; |
| font-style: normal; |
| line-height: 1.63; |
| letter-spacing: normal; |
| color: #707070; |
| } |
| |
| </style> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| <div class="d-flex"> |
| |
| |
| <div class="td-sidebar desktop-only d-print-none"> |
| |
| <div id="docs-version-selector" class="docs-version-selector sidebar__version-selector"> |
| <a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" |
| aria-expanded="false"> |
| <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.3.2</span> |
| </a> |
| <div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink"> |
| |
| </div> |
| <script type="application/x-template" id="version-item-template"> |
| <a class="dropdown-item"></a> |
| </script> |
| </div> |
| |
| <div class="searchb-box"> |
| <form class="search-form" action="../../../search.html" method="get"> |
| <input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16"> |
| <input type="hidden" name="check_keywords" value="yes" /> |
| <input type="hidden" name="area" value="default" /> |
| |
| <button class="search-form__button" type="submit"> |
| |
| <svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20"> |
| <g id="Group_1579" data-name="Group 1579" transform="translate(-41.001 -41)"> |
| <path id="Path_169" d="M71.415 64.687a7.215 7.215 0 1 0-6.729 6.728 7.222 7.222 0 0 0 6.729-6.728z" fill="none" data-name="Path 169" transform="translate(-14.277 -14.276)"></path> |
| <path id="Path_170" d="M60.863 59.8l-6.093-6.09a7.78 7.78 0 1 0-1.06 1.06l6.09 6.093a.468.468 0 0 0 .662 0l.4-.4a.468.468 0 0 0 .001-.663zM42.512 49.183a6.274 6.274 0 1 1 5.851 5.85 6.28 6.28 0 0 1-5.851-5.85z" fill="#51504f" data-name="Path 170"></path> |
| </g> |
| </svg> |
| </button> |
| </form> |
| </div> |
| |
| |
| <style> |
| .searchb-box { |
| margin-bottom: 26px; |
| } |
| .searchb-box .search-form { |
| width: 100%; |
| margin-top: 20px; |
| } |
| </style> |
| |
| |
| <div class="toctree" role="navigation" aria-label="main navigation"> |
| <p class="caption" role="heading"><span class="caption-text">Content</span></p> |
| <ul> |
| <li class="toctree-l1"><a class="reference internal" href="../../../index.html">Home</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../start/index.html">Quick Start</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../installation/index.html">Installation</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../upgrading-from-1-10/index.html">Upgrading from 1.10 to 2</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../tutorial_taskflow_api.html">Tutorial on the TaskFlow API</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../concepts/index.html">Concepts</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../executor/index.html">Executor</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../dag-run.html">DAG Runs</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../security/index.html">Security</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../logging-monitoring/index.html">Logging & Monitoring</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time Zones</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../usage-cli.html">Using the CLI</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../kubernetes.html">Kubernetes</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../listeners.html">Listeners</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../dag-serialization.html">DAG Serialization</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../modules_management.html">Modules Management</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../release-process.html">Release Policies</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../release_notes.html">Release Notes</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../best-practices.html">Best Practices</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../production-deployment.html">Production Deployment</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../privacy_notice.html">Privacy Notice</a></li> |
| </ul> |
| <p class="caption" role="heading"><span class="caption-text">References</span></p> |
| <ul> |
| <li class="toctree-l1"><a class="reference internal" href="../../../operators-and-hooks-ref.html">Operators and hooks</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../cli-and-env-variables-ref.html">CLI</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../templates-ref.html">Templates</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../python-api-ref.html">Python API</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../stable-rest-api-ref.html">Stable REST API</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../configurations-ref.html">Configurations</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../extra-packages-ref.html">Extra packages</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="../../../migrations-ref.html">Database Migrations</a></li> |
| </ul> |
| |
| </div> |
| |
| <style type="text/css"> |
| .toctree { |
| } |
| .toctree li { |
| list-style: none; |
| } |
| .toctree .caption { |
| font-family: Roboto; |
| font-size: 18px; |
| font-weight: bold; |
| font-stretch: normal; |
| font-style: normal; |
| line-height: 1.33; |
| letter-spacing: normal; |
| color: #51504f; |
| padding-bottom: 13px; |
| text-transform: uppercase; |
| margin-bottom: 0; |
| } |
| .toctree .current > a:not([href="#"]) { |
| color: #017cee; |
| } |
| .toctree > ul { |
| padding-left: 0; |
| } |
| .toctree ul { |
| padding-left: 15px; |
| display: none; |
| } |
| .toctree > ul, |
| .toctree li.current > ul { |
| display: block; |
| } |
| |
| .toctree a .toctree-expand { |
| display: inline-block; |
| position: relative; |
| height: 1em; |
| } |
| |
| .toctree a .toctree-expand:before { |
| position: absolute; |
| top: 6px; |
| left: -12px; |
| content: '►'; |
| font-size: 7px; |
| } |
| .toctree .current > a > .toctree-expand:before { |
| content: '▼'; |
| } |
| |
| .toctree .current { |
| color: #017cee; |
| } |
| .toctree li { |
| font-family: Roboto; |
| font-size: 16px; |
| font-weight: normal; |
| font-stretch: normal; |
| font-style: normal; |
| line-height: 1.63; |
| letter-spacing: normal; |
| color: #707070; |
| } |
| |
| </style> |
| </div> |
| |
| |
| |
| |
| <main class="col-12 col-md-9 col-xl-8" role="main"> |
| |
| |
| |
| |
| |
| |
| |
| |
| <div role="navigation" aria-label="breadcrumbs navigation" class="d-none d-md-block d-print-none"> |
| |
| <ul class="breadcrumb"> |
| |
| <li class="breadcrumb-item"><a href="../../../index.html" class="icon icon-home"> Home</a></li> |
| |
| <li class="breadcrumb-item"><a href="../../index.html">Module code</a></li> |
| |
| <li class="breadcrumb-item"><a href="dag.html"> airflow.models.dag</a></li> |
| |
| </ul> |
| </div> |
| |
| <div class="rst-content"> |
| <div class="document"> |
| <div class="documentwrapper"> |
| <div class="bodywrapper"> |
| <div class="body" role="main"> |
| |
| <h1>Source code for airflow.models.dag</h1><div class="highlight"><pre> |
| <span></span><span class="c1">#</span> |
| <span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span> |
| <span class="c1"># or more contributor license agreements. See the NOTICE file</span> |
| <span class="c1"># distributed with this work for additional information</span> |
| <span class="c1"># regarding copyright ownership. The ASF licenses this file</span> |
| <span class="c1"># to you under the Apache License, Version 2.0 (the</span> |
| <span class="c1"># "License"); you may not use this file except in compliance</span> |
| <span class="c1"># with the License. You may obtain a copy of the License at</span> |
| <span class="c1">#</span> |
| <span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> |
| <span class="c1">#</span> |
| <span class="c1"># Unless required by applicable law or agreed to in writing,</span> |
| <span class="c1"># software distributed under the License is distributed on an</span> |
| <span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span> |
| <span class="c1"># KIND, either express or implied. See the License for the</span> |
| <span class="c1"># specific language governing permissions and limitations</span> |
| <span class="c1"># under the License.</span> |
| |
| <span class="kn">import</span> <span class="nn">copy</span> |
| <span class="kn">import</span> <span class="nn">functools</span> |
| <span class="kn">import</span> <span class="nn">itertools</span> |
| <span class="kn">import</span> <span class="nn">logging</span> |
| <span class="kn">import</span> <span class="nn">os</span> |
| <span class="kn">import</span> <span class="nn">pathlib</span> |
| <span class="kn">import</span> <span class="nn">pickle</span> |
| <span class="kn">import</span> <span class="nn">re</span> |
| <span class="kn">import</span> <span class="nn">sys</span> |
| <span class="kn">import</span> <span class="nn">traceback</span> |
| <span class="kn">import</span> <span class="nn">warnings</span> |
| <span class="kn">import</span> <span class="nn">weakref</span> |
| <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span> |
| <span class="kn">from</span> <span class="nn">inspect</span> <span class="kn">import</span> <span class="n">signature</span> |
| <span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="p">(</span> |
| <span class="n">TYPE_CHECKING</span><span class="p">,</span> |
| <span class="n">Any</span><span class="p">,</span> |
| <span class="n">Callable</span><span class="p">,</span> |
| <span class="n">Collection</span><span class="p">,</span> |
| <span class="n">Dict</span><span class="p">,</span> |
| <span class="n">FrozenSet</span><span class="p">,</span> |
| <span class="n">Iterable</span><span class="p">,</span> |
| <span class="n">List</span><span class="p">,</span> |
| <span class="n">Optional</span><span class="p">,</span> |
| <span class="n">Sequence</span><span class="p">,</span> |
| <span class="n">Set</span><span class="p">,</span> |
| <span class="n">Tuple</span><span class="p">,</span> |
| <span class="n">Type</span><span class="p">,</span> |
| <span class="n">Union</span><span class="p">,</span> |
| <span class="n">cast</span><span class="p">,</span> |
| <span class="n">overload</span><span class="p">,</span> |
| <span class="p">)</span> |
| |
| <span class="kn">import</span> <span class="nn">jinja2</span> |
| <span class="kn">import</span> <span class="nn">pendulum</span> |
| <span class="kn">from</span> <span class="nn">dateutil.relativedelta</span> <span class="kn">import</span> <span class="n">relativedelta</span> |
| <span class="kn">from</span> <span class="nn">pendulum.tz.timezone</span> <span class="kn">import</span> <span class="n">Timezone</span> |
| <span class="kn">from</span> <span class="nn">sqlalchemy</span> <span class="kn">import</span> <span class="n">Boolean</span><span class="p">,</span> <span class="n">Column</span><span class="p">,</span> <span class="n">ForeignKey</span><span class="p">,</span> <span class="n">Index</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">String</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="n">not_</span><span class="p">,</span> <span class="n">or_</span> |
| <span class="kn">from</span> <span class="nn">sqlalchemy.orm</span> <span class="kn">import</span> <span class="n">backref</span><span class="p">,</span> <span class="n">joinedload</span><span class="p">,</span> <span class="n">relationship</span> |
| <span class="kn">from</span> <span class="nn">sqlalchemy.orm.query</span> <span class="kn">import</span> <span class="n">Query</span> |
| <span class="kn">from</span> <span class="nn">sqlalchemy.orm.session</span> <span class="kn">import</span> <span class="n">Session</span> |
| <span class="kn">from</span> <span class="nn">sqlalchemy.sql</span> <span class="kn">import</span> <span class="n">expression</span> |
| |
| <span class="kn">import</span> <span class="nn">airflow.templates</span> |
| <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">settings</span><span class="p">,</span> <span class="n">utils</span> |
| <span class="kn">from</span> <span class="nn">airflow.compat.functools</span> <span class="kn">import</span> <span class="n">cached_property</span> |
| <span class="kn">from</span> <span class="nn">airflow.configuration</span> <span class="kn">import</span> <span class="n">conf</span> |
| <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span><span class="p">,</span> <span class="n">DuplicateTaskIdFound</span><span class="p">,</span> <span class="n">TaskNotFound</span> |
| <span class="kn">from</span> <span class="nn">airflow.models.abstractoperator</span> <span class="kn">import</span> <span class="n">AbstractOperator</span> |
| <span class="kn">from</span> <span class="nn">airflow.models.base</span> <span class="kn">import</span> <span class="n">ID_LEN</span><span class="p">,</span> <span class="n">Base</span> |
| <span class="kn">from</span> <span class="nn">airflow.models.dagbag</span> <span class="kn">import</span> <span class="n">DagBag</span> |
| <span class="kn">from</span> <span class="nn">airflow.models.dagcode</span> <span class="kn">import</span> <span class="n">DagCode</span> |
| <span class="kn">from</span> <span class="nn">airflow.models.dagpickle</span> <span class="kn">import</span> <span class="n">DagPickle</span> |
| <span class="kn">from</span> <span class="nn">airflow.models.dagrun</span> <span class="kn">import</span> <span class="n">DagRun</span> |
| <span class="kn">from</span> <span class="nn">airflow.models.operator</span> <span class="kn">import</span> <span class="n">Operator</span> |
| <span class="kn">from</span> <span class="nn">airflow.models.param</span> <span class="kn">import</span> <span class="n">DagParam</span><span class="p">,</span> <span class="n">ParamsDict</span> |
| <span class="kn">from</span> <span class="nn">airflow.models.taskinstance</span> <span class="kn">import</span> <span class="n">Context</span><span class="p">,</span> <span class="n">TaskInstance</span><span class="p">,</span> <span class="n">TaskInstanceKey</span><span class="p">,</span> <span class="n">clear_task_instances</span> |
| <span class="kn">from</span> <span class="nn">airflow.security</span> <span class="kn">import</span> <span class="n">permissions</span> |
| <span class="kn">from</span> <span class="nn">airflow.stats</span> <span class="kn">import</span> <span class="n">Stats</span> |
| <span class="kn">from</span> <span class="nn">airflow.timetables.base</span> <span class="kn">import</span> <span class="n">DagRunInfo</span><span class="p">,</span> <span class="n">DataInterval</span><span class="p">,</span> <span class="n">TimeRestriction</span><span class="p">,</span> <span class="n">Timetable</span> |
| <span class="kn">from</span> <span class="nn">airflow.timetables.interval</span> <span class="kn">import</span> <span class="n">CronDataIntervalTimetable</span><span class="p">,</span> <span class="n">DeltaDataIntervalTimetable</span> |
| <span class="kn">from</span> <span class="nn">airflow.timetables.simple</span> <span class="kn">import</span> <span class="n">NullTimetable</span><span class="p">,</span> <span class="n">OnceTimetable</span> |
| <span class="kn">from</span> <span class="nn">airflow.typing_compat</span> <span class="kn">import</span> <span class="n">Literal</span><span class="p">,</span> <span class="n">RePatternType</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="kn">import</span> <span class="n">timezone</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils.dag_cycle_tester</span> <span class="kn">import</span> <span class="n">check_cycle</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils.dates</span> <span class="kn">import</span> <span class="n">cron_presets</span><span class="p">,</span> <span class="n">date_range</span> <span class="k">as</span> <span class="n">utils_date_range</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="kn">import</span> <span class="n">correct_maybe_zipped</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils.helpers</span> <span class="kn">import</span> <span class="n">exactly_one</span><span class="p">,</span> <span class="n">validate_key</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="kn">import</span> <span class="n">LoggingMixin</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils.session</span> <span class="kn">import</span> <span class="n">NEW_SESSION</span><span class="p">,</span> <span class="n">provide_session</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils.sqlalchemy</span> <span class="kn">import</span> <span class="n">Interval</span><span class="p">,</span> <span class="n">UtcDateTime</span><span class="p">,</span> <span class="n">skip_locked</span><span class="p">,</span> <span class="n">tuple_in_condition</span><span class="p">,</span> <span class="n">with_row_locks</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">DagRunState</span><span class="p">,</span> <span class="n">State</span><span class="p">,</span> <span class="n">TaskInstanceState</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils.types</span> <span class="kn">import</span> <span class="n">NOTSET</span><span class="p">,</span> <span class="n">ArgNotSet</span><span class="p">,</span> <span class="n">DagRunType</span><span class="p">,</span> <span class="n">EdgeInfoType</span> |
| |
| <span class="k">if</span> <span class="n">TYPE_CHECKING</span><span class="p">:</span> |
| <span class="kn">from</span> <span class="nn">airflow.decorators</span> <span class="kn">import</span> <span class="n">TaskDecoratorCollection</span> |
| <span class="kn">from</span> <span class="nn">airflow.models.slamiss</span> <span class="kn">import</span> <span class="n">SlaMiss</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils.task_group</span> <span class="kn">import</span> <span class="n">TaskGroup</span> |
| |
| |
| <div class="viewcode-block" id="log"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.log">[docs]</a><span class="n">log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="DEFAULT_VIEW_PRESETS"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DEFAULT_VIEW_PRESETS">[docs]</a><span class="n">DEFAULT_VIEW_PRESETS</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'grid'</span><span class="p">,</span> <span class="s1">'graph'</span><span class="p">,</span> <span class="s1">'duration'</span><span class="p">,</span> <span class="s1">'gantt'</span><span class="p">,</span> <span class="s1">'landing_times'</span><span class="p">]</span></div> |
| <div class="viewcode-block" id="ORIENTATION_PRESETS"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.ORIENTATION_PRESETS">[docs]</a><span class="n">ORIENTATION_PRESETS</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'LR'</span><span class="p">,</span> <span class="s1">'TB'</span><span class="p">,</span> <span class="s1">'RL'</span><span class="p">,</span> <span class="s1">'BT'</span><span class="p">]</span></div> |
| |
| |
| <div class="viewcode-block" id="DagStateChangeCallback"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagStateChangeCallback">[docs]</a><span class="n">DagStateChangeCallback</span> <span class="o">=</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">Context</span><span class="p">],</span> <span class="kc">None</span><span class="p">]</span></div> |
| <div class="viewcode-block" id="ScheduleInterval"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.ScheduleInterval">[docs]</a><span class="n">ScheduleInterval</span> <span class="o">=</span> <span class="n">Union</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timedelta</span><span class="p">,</span> <span class="n">relativedelta</span><span class="p">]</span></div> |
| |
| <span class="c1"># FIXME: Ideally this should be Union[Literal[NOTSET], ScheduleInterval],</span> |
| <span class="c1"># but Mypy cannot handle that right now. Track progress of PEP 661 for progress.</span> |
| <span class="c1"># See also: https://discuss.python.org/t/9126/7</span> |
| <div class="viewcode-block" id="ScheduleIntervalArg"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.ScheduleIntervalArg">[docs]</a><span class="n">ScheduleIntervalArg</span> <span class="o">=</span> <span class="n">Union</span><span class="p">[</span><span class="n">ArgNotSet</span><span class="p">,</span> <span class="n">ScheduleInterval</span><span class="p">]</span></div> |
| |
| |
| <span class="c1"># Backward compatibility: If neither schedule_interval nor timetable is</span> |
| <span class="c1"># *provided by the user*, default to a one-day interval.</span> |
| <div class="viewcode-block" id="DEFAULT_SCHEDULE_INTERVAL"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DEFAULT_SCHEDULE_INTERVAL">[docs]</a><span class="n">DEFAULT_SCHEDULE_INTERVAL</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span></div> |
| |
| |
| <div class="viewcode-block" id="InconsistentDataInterval"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.InconsistentDataInterval">[docs]</a><span class="k">class</span> <span class="nc">InconsistentDataInterval</span><span class="p">(</span><span class="n">AirflowException</span><span class="p">):</span> |
| <span class="sd">"""Exception raised when a model populates data interval fields incorrectly.</span> |
| |
| <span class="sd"> The data interval fields should either both be None (for runs scheduled</span> |
| <span class="sd"> prior to AIP-39), or both be datetime (for runs scheduled after AIP-39 is</span> |
| <span class="sd"> implemented). This is raised if exactly one of the fields is None.</span> |
| <span class="sd"> """</span> |
| |
| <span class="n">_template</span> <span class="o">=</span> <span class="p">(</span> |
| <span class="s2">"Inconsistent </span><span class="si">{cls}</span><span class="s2">: </span><span class="si">{start[0]}</span><span class="s2">=</span><span class="si">{start[1]!r}</span><span class="s2">, </span><span class="si">{end[0]}</span><span class="s2">=</span><span class="si">{end[1]!r}</span><span class="s2">, "</span> |
| <span class="s2">"they must be either both None or both datetime"</span> |
| <span class="p">)</span> |
| |
| <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">instance</span><span class="p">:</span> <span class="n">Any</span><span class="p">,</span> <span class="n">start_field_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">end_field_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_class_name</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">instance</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_start_field</span> <span class="o">=</span> <span class="p">(</span><span class="n">start_field_name</span><span class="p">,</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">instance</span><span class="p">,</span> <span class="n">start_field_name</span><span class="p">))</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_end_field</span> <span class="o">=</span> <span class="p">(</span><span class="n">end_field_name</span><span class="p">,</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">instance</span><span class="p">,</span> <span class="n">end_field_name</span><span class="p">))</span> |
| |
| <div class="viewcode-block" id="InconsistentDataInterval.__str__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.InconsistentDataInterval.__str__">[docs]</a> <span class="k">def</span> <span class="fm">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_template</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">cls</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_class_name</span><span class="p">,</span> <span class="n">start</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_start_field</span><span class="p">,</span> <span class="n">end</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_end_field</span><span class="p">)</span></div></div> |
| |
| |
| <span class="k">def</span> <span class="nf">_get_model_data_interval</span><span class="p">(</span> |
| <span class="n">instance</span><span class="p">:</span> <span class="n">Any</span><span class="p">,</span> |
| <span class="n">start_field_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
| <span class="n">end_field_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">DataInterval</span><span class="p">]:</span> |
| <span class="n">start</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="n">instance</span><span class="p">,</span> <span class="n">start_field_name</span><span class="p">))</span> |
| <span class="n">end</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="n">instance</span><span class="p">,</span> <span class="n">end_field_name</span><span class="p">))</span> |
| <span class="k">if</span> <span class="n">start</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">if</span> <span class="n">end</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="n">InconsistentDataInterval</span><span class="p">(</span><span class="n">instance</span><span class="p">,</span> <span class="n">start_field_name</span><span class="p">,</span> <span class="n">end_field_name</span><span class="p">)</span> |
| <span class="k">return</span> <span class="kc">None</span> |
| <span class="k">elif</span> <span class="n">end</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="n">InconsistentDataInterval</span><span class="p">(</span><span class="n">instance</span><span class="p">,</span> <span class="n">start_field_name</span><span class="p">,</span> <span class="n">end_field_name</span><span class="p">)</span> |
| <span class="k">return</span> <span class="n">DataInterval</span><span class="p">(</span><span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">)</span> |
| |
| |
| <div class="viewcode-block" id="create_timetable"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.create_timetable">[docs]</a><span class="k">def</span> <span class="nf">create_timetable</span><span class="p">(</span><span class="n">interval</span><span class="p">:</span> <span class="n">ScheduleIntervalArg</span><span class="p">,</span> <span class="n">timezone</span><span class="p">:</span> <span class="n">Timezone</span><span class="p">)</span> <span class="o">-></span> <span class="n">Timetable</span><span class="p">:</span> |
| <span class="sd">"""Create a Timetable instance from a ``schedule_interval`` argument."""</span> |
| <span class="k">if</span> <span class="n">interval</span> <span class="ow">is</span> <span class="n">NOTSET</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">DeltaDataIntervalTimetable</span><span class="p">(</span><span class="n">DEFAULT_SCHEDULE_INTERVAL</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">interval</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">NullTimetable</span><span class="p">()</span> |
| <span class="k">if</span> <span class="n">interval</span> <span class="o">==</span> <span class="s2">"@once"</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">OnceTimetable</span><span class="p">()</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">interval</span><span class="p">,</span> <span class="p">(</span><span class="n">timedelta</span><span class="p">,</span> <span class="n">relativedelta</span><span class="p">)):</span> |
| <span class="k">return</span> <span class="n">DeltaDataIntervalTimetable</span><span class="p">(</span><span class="n">interval</span><span class="p">)</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">interval</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">CronDataIntervalTimetable</span><span class="p">(</span><span class="n">interval</span><span class="p">,</span> <span class="n">timezone</span><span class="p">)</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">interval</span><span class="si">!r}</span><span class="s2"> is not a valid schedule_interval."</span><span class="p">)</span></div> |
| |
| |
| <div class="viewcode-block" id="get_last_dagrun"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.get_last_dagrun">[docs]</a><span class="k">def</span> <span class="nf">get_last_dagrun</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">session</span><span class="p">,</span> <span class="n">include_externally_triggered</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Returns the last dag run for a dag, None if there was none.</span> |
| <span class="sd"> Last dag run can be any type of run eg. scheduled or backfilled.</span> |
| <span class="sd"> Overridden DagRuns are ignored.</span> |
| <span class="sd"> """</span> |
| <span class="n">DR</span> <span class="o">=</span> <span class="n">DagRun</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DR</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">include_externally_triggered</span><span class="p">:</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">external_trigger</span> <span class="o">==</span> <span class="n">expression</span><span class="o">.</span><span class="n">false</span><span class="p">())</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">desc</span><span class="p">())</span> |
| <span class="k">return</span> <span class="n">query</span><span class="o">.</span><span class="n">first</span><span class="p">()</span></div> |
| |
| |
| <span class="nd">@functools</span><span class="o">.</span><span class="n">total_ordering</span> |
| <div class="viewcode-block" id="DAG"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG">[docs]</a><span class="k">class</span> <span class="nc">DAG</span><span class="p">(</span><span class="n">LoggingMixin</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> A dag (directed acyclic graph) is a collection of tasks with directional</span> |
| <span class="sd"> dependencies. A dag also has a schedule, a start date and an end date</span> |
| <span class="sd"> (optional). For each schedule, (say daily or hourly), the DAG needs to run</span> |
| <span class="sd"> each individual tasks as their dependencies are met. Certain tasks have</span> |
| <span class="sd"> the property of depending on their own past, meaning that they can't run</span> |
| <span class="sd"> until their previous schedule (and upstream tasks) are completed.</span> |
| |
| <span class="sd"> DAGs essentially act as namespaces for tasks. A task_id can only be</span> |
| <span class="sd"> added once to a DAG.</span> |
| |
| <span class="sd"> Note that if you plan to use time zones all the dates provided should be pendulum</span> |
| <span class="sd"> dates. See :ref:`timezone_aware_dags`.</span> |
| |
| <span class="sd"> :param dag_id: The id of the DAG; must consist exclusively of alphanumeric</span> |
| <span class="sd"> characters, dashes, dots and underscores (all ASCII)</span> |
| <span class="sd"> :param description: The description for the DAG to e.g. be shown on the webserver</span> |
| <span class="sd"> :param schedule_interval: Defines how often that DAG runs, this</span> |
| <span class="sd"> timedelta object gets added to your latest task instance's</span> |
| <span class="sd"> execution_date to figure out the next schedule</span> |
| <span class="sd"> :param timetable: Specify which timetable to use (in which case schedule_interval</span> |
| <span class="sd"> must not be set). See :doc:`/howto/timetable` for more information</span> |
| <span class="sd"> :param start_date: The timestamp from which the scheduler will</span> |
| <span class="sd"> attempt to backfill</span> |
| <span class="sd"> :param end_date: A date beyond which your DAG won't run, leave to None</span> |
| <span class="sd"> for open ended scheduling</span> |
| <span class="sd"> :param template_searchpath: This list of folders (non relative)</span> |
| <span class="sd"> defines where jinja will look for your templates. Order matters.</span> |
| <span class="sd"> Note that jinja/airflow includes the path of your DAG file by</span> |
| <span class="sd"> default</span> |
| <span class="sd"> :param template_undefined: Template undefined type.</span> |
| <span class="sd"> :param user_defined_macros: a dictionary of macros that will be exposed</span> |
| <span class="sd"> in your jinja templates. For example, passing ``dict(foo='bar')``</span> |
| <span class="sd"> to this argument allows you to ``{{ foo }}`` in all jinja</span> |
| <span class="sd"> templates related to this DAG. Note that you can pass any</span> |
| <span class="sd"> type of object here.</span> |
| <span class="sd"> :param user_defined_filters: a dictionary of filters that will be exposed</span> |
| <span class="sd"> in your jinja templates. For example, passing</span> |
| <span class="sd"> ``dict(hello=lambda name: 'Hello %s' % name)`` to this argument allows</span> |
| <span class="sd"> you to ``{{ 'world' | hello }}`` in all jinja templates related to</span> |
| <span class="sd"> this DAG.</span> |
| <span class="sd"> :param default_args: A dictionary of default parameters to be used</span> |
| <span class="sd"> as constructor keyword parameters when initialising operators.</span> |
| <span class="sd"> Note that operators have the same hook, and precede those defined</span> |
| <span class="sd"> here, meaning that if your dict contains `'depends_on_past': True`</span> |
| <span class="sd"> here and `'depends_on_past': False` in the operator's call</span> |
| <span class="sd"> `default_args`, the actual value will be `False`.</span> |
| <span class="sd"> :param params: a dictionary of DAG level parameters that are made</span> |
| <span class="sd"> accessible in templates, namespaced under `params`. These</span> |
| <span class="sd"> params can be overridden at the task level.</span> |
| <span class="sd"> :param max_active_tasks: the number of task instances allowed to run</span> |
| <span class="sd"> concurrently</span> |
| <span class="sd"> :param max_active_runs: maximum number of active DAG runs, beyond this</span> |
| <span class="sd"> number of DAG runs in a running state, the scheduler won't create</span> |
| <span class="sd"> new active DAG runs</span> |
| <span class="sd"> :param dagrun_timeout: specify how long a DagRun should be up before</span> |
| <span class="sd"> timing out / failing, so that new DagRuns can be created. The timeout</span> |
| <span class="sd"> is only enforced for scheduled DagRuns.</span> |
| <span class="sd"> :param sla_miss_callback: specify a function to call when reporting SLA</span> |
| <span class="sd"> timeouts. See :ref:`sla_miss_callback<concepts:sla_miss_callback>` for</span> |
| <span class="sd"> more information about the function signature and parameters that are</span> |
| <span class="sd"> passed to the callback.</span> |
| <span class="sd"> :param default_view: Specify DAG default view (grid, graph, duration,</span> |
| <span class="sd"> gantt, landing_times), default grid</span> |
| <span class="sd"> :param orientation: Specify DAG orientation in graph view (LR, TB, RL, BT), default LR</span> |
| <span class="sd"> :param catchup: Perform scheduler catchup (or only run latest)? Defaults to True</span> |
| <span class="sd"> :param on_failure_callback: A function to be called when a DagRun of this dag fails.</span> |
| <span class="sd"> A context dictionary is passed as a single parameter to this function.</span> |
| <span class="sd"> :param on_success_callback: Much like the ``on_failure_callback`` except</span> |
| <span class="sd"> that it is executed when the dag succeeds.</span> |
| <span class="sd"> :param access_control: Specify optional DAG-level actions, e.g.,</span> |
| <span class="sd"> "{'role1': {'can_read'}, 'role2': {'can_read', 'can_edit', 'can_delete'}}"</span> |
| <span class="sd"> :param is_paused_upon_creation: Specifies if the dag is paused when created for the first time.</span> |
| <span class="sd"> If the dag exists already, this flag will be ignored. If this optional parameter</span> |
| <span class="sd"> is not specified, the global config setting will be used.</span> |
| <span class="sd"> :param jinja_environment_kwargs: additional configuration options to be passed to Jinja</span> |
| <span class="sd"> ``Environment`` for template rendering</span> |
| |
| <span class="sd"> **Example**: to avoid Jinja from removing a trailing newline from template strings ::</span> |
| |
| <span class="sd"> DAG(dag_id='my-dag',</span> |
| <span class="sd"> jinja_environment_kwargs={</span> |
| <span class="sd"> 'keep_trailing_newline': True,</span> |
| <span class="sd"> # some other jinja2 Environment options here</span> |
| <span class="sd"> }</span> |
| <span class="sd"> )</span> |
| |
| <span class="sd"> **See**: `Jinja Environment documentation</span> |
| <span class="sd"> <https://jinja.palletsprojects.com/en/2.11.x/api/#jinja2.Environment>`_</span> |
| |
| <span class="sd"> :param render_template_as_native_obj: If True, uses a Jinja ``NativeEnvironment``</span> |
| <span class="sd"> to render templates as native Python types. If False, a Jinja</span> |
| <span class="sd"> ``Environment`` is used to render templates as string values.</span> |
| <span class="sd"> :param tags: List of tags to help filtering DAGs in the UI.</span> |
| <span class="sd"> """</span> |
| |
| <span class="n">_comps</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s1">'dag_id'</span><span class="p">,</span> |
| <span class="s1">'task_ids'</span><span class="p">,</span> |
| <span class="s1">'parent_dag'</span><span class="p">,</span> |
| <span class="s1">'start_date'</span><span class="p">,</span> |
| <span class="s1">'end_date'</span><span class="p">,</span> |
| <span class="s1">'schedule_interval'</span><span class="p">,</span> |
| <span class="s1">'fileloc'</span><span class="p">,</span> |
| <span class="s1">'template_searchpath'</span><span class="p">,</span> |
| <span class="s1">'last_loaded'</span><span class="p">,</span> |
| <span class="p">}</span> |
| |
| <span class="n">__serialized_fields</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">FrozenSet</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span> |
| |
| <div class="viewcode-block" id="DAG.fileloc"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.fileloc">[docs]</a> <span class="n">fileloc</span><span class="p">:</span> <span class="nb">str</span></div> |
| <span class="sd">"""</span> |
| <span class="sd"> File path that needs to be imported to load this DAG or subdag.</span> |
| |
| <span class="sd"> This may not be an actual file on disk in the case when this DAG is loaded</span> |
| <span class="sd"> from a ZIP file or other DAG distribution format.</span> |
| <span class="sd"> """</span> |
| |
| <div class="viewcode-block" id="DAG.parent_dag"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.parent_dag">[docs]</a> <span class="n">parent_dag</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="s2">"DAG"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span> <span class="c1"># Gets set when DAGs are loaded</span></div> |
| |
| <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">dag_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
| <span class="n">description</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">schedule_interval</span><span class="p">:</span> <span class="n">ScheduleIntervalArg</span> <span class="o">=</span> <span class="n">NOTSET</span><span class="p">,</span> |
| <span class="n">timetable</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Timetable</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">full_filepath</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">template_searchpath</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Iterable</span><span class="p">[</span><span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">template_undefined</span><span class="p">:</span> <span class="n">Type</span><span class="p">[</span><span class="n">jinja2</span><span class="o">.</span><span class="n">StrictUndefined</span><span class="p">]</span> <span class="o">=</span> <span class="n">jinja2</span><span class="o">.</span><span class="n">StrictUndefined</span><span class="p">,</span> |
| <span class="n">user_defined_macros</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">user_defined_filters</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">default_args</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">concurrency</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">max_active_tasks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'max_active_tasks_per_dag'</span><span class="p">),</span> |
| <span class="n">max_active_runs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'max_active_runs_per_dag'</span><span class="p">),</span> |
| <span class="n">dagrun_timeout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">timedelta</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">sla_miss_callback</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span> |
| <span class="n">Callable</span><span class="p">[[</span><span class="s2">"DAG"</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="s2">"SlaMiss"</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">TaskInstance</span><span class="p">]],</span> <span class="kc">None</span><span class="p">]</span> |
| <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">default_view</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get_mandatory_value</span><span class="p">(</span><span class="s1">'webserver'</span><span class="p">,</span> <span class="s1">'dag_default_view'</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">(),</span> |
| <span class="n">orientation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get_mandatory_value</span><span class="p">(</span><span class="s1">'webserver'</span><span class="p">,</span> <span class="s1">'dag_orientation'</span><span class="p">),</span> |
| <span class="n">catchup</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'scheduler'</span><span class="p">,</span> <span class="s1">'catchup_by_default'</span><span class="p">),</span> |
| <span class="n">on_success_callback</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DagStateChangeCallback</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">on_failure_callback</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DagStateChangeCallback</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">doc_md</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">access_control</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">is_paused_upon_creation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">jinja_environment_kwargs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">render_template_as_native_obj</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> |
| <span class="n">tags</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="p">):</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils.task_group</span> <span class="kn">import</span> <span class="n">TaskGroup</span> |
| |
| <span class="bp">self</span><span class="o">.</span><span class="n">user_defined_macros</span> <span class="o">=</span> <span class="n">user_defined_macros</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">user_defined_filters</span> <span class="o">=</span> <span class="n">user_defined_filters</span> |
| <span class="k">if</span> <span class="n">default_args</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">default_args</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"default_args must be a dict"</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">default_args</span> <span class="ow">or</span> <span class="p">{})</span> |
| <span class="n">params</span> <span class="o">=</span> <span class="n">params</span> <span class="ow">or</span> <span class="p">{}</span> |
| |
| <span class="c1"># merging potentially conflicting default_args['params'] into params</span> |
| <span class="k">if</span> <span class="s1">'params'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">:</span> |
| <span class="n">params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'params'</span><span class="p">])</span> |
| <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'params'</span><span class="p">]</span> |
| |
| <span class="c1"># check self.params and convert them into ParamsDict</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">params</span> <span class="o">=</span> <span class="n">ParamsDict</span><span class="p">(</span><span class="n">params</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">full_filepath</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"Passing full_filepath to DAG() is deprecated and has no effect"</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| |
| <span class="n">validate_key</span><span class="p">(</span><span class="n">dag_id</span><span class="p">)</span> |
| |
| <span class="bp">self</span><span class="o">.</span><span class="n">_dag_id</span> <span class="o">=</span> <span class="n">dag_id</span> |
| <span class="k">if</span> <span class="n">concurrency</span><span class="p">:</span> |
| <span class="c1"># TODO: Remove in Airflow 3.0</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"The 'concurrency' parameter is deprecated. Please use 'max_active_tasks'."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">max_active_tasks</span> <span class="o">=</span> <span class="n">concurrency</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_max_active_tasks</span> <span class="o">=</span> <span class="n">max_active_tasks</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_pickle_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span> |
| |
| <span class="bp">self</span><span class="o">.</span><span class="n">_description</span> <span class="o">=</span> <span class="n">description</span> |
| <span class="c1"># set file location to caller source path</span> |
| <span class="n">back</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">_getframe</span><span class="p">()</span><span class="o">.</span><span class="n">f_back</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">=</span> <span class="n">back</span><span class="o">.</span><span class="n">f_code</span><span class="o">.</span><span class="n">co_filename</span> <span class="k">if</span> <span class="n">back</span> <span class="k">else</span> <span class="s2">""</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Operator</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span> |
| |
| <span class="c1"># set timezone from start_date</span> |
| <span class="n">tz</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">if</span> <span class="n">start_date</span> <span class="ow">and</span> <span class="n">start_date</span><span class="o">.</span><span class="n">tzinfo</span><span class="p">:</span> |
| <span class="n">tzinfo</span> <span class="o">=</span> <span class="kc">None</span> <span class="k">if</span> <span class="n">start_date</span><span class="o">.</span><span class="n">tzinfo</span> <span class="k">else</span> <span class="n">settings</span><span class="o">.</span><span class="n">TIMEZONE</span> |
| <span class="n">tz</span> <span class="o">=</span> <span class="n">pendulum</span><span class="o">.</span><span class="n">instance</span><span class="p">(</span><span class="n">start_date</span><span class="p">,</span> <span class="n">tz</span><span class="o">=</span><span class="n">tzinfo</span><span class="p">)</span><span class="o">.</span><span class="n">timezone</span> |
| <span class="k">elif</span> <span class="s1">'start_date'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'start_date'</span><span class="p">]:</span> |
| <span class="n">date</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'start_date'</span><span class="p">]</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">date</span><span class="p">,</span> <span class="n">datetime</span><span class="p">):</span> |
| <span class="n">date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">date</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'start_date'</span><span class="p">]</span> <span class="o">=</span> <span class="n">date</span> |
| <span class="n">start_date</span> <span class="o">=</span> <span class="n">date</span> |
| |
| <span class="n">tzinfo</span> <span class="o">=</span> <span class="kc">None</span> <span class="k">if</span> <span class="n">date</span><span class="o">.</span><span class="n">tzinfo</span> <span class="k">else</span> <span class="n">settings</span><span class="o">.</span><span class="n">TIMEZONE</span> |
| <span class="n">tz</span> <span class="o">=</span> <span class="n">pendulum</span><span class="o">.</span><span class="n">instance</span><span class="p">(</span><span class="n">date</span><span class="p">,</span> <span class="n">tz</span><span class="o">=</span><span class="n">tzinfo</span><span class="p">)</span><span class="o">.</span><span class="n">timezone</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">timezone</span> <span class="o">=</span> <span class="n">tz</span> <span class="ow">or</span> <span class="n">settings</span><span class="o">.</span><span class="n">TIMEZONE</span> |
| |
| <span class="c1"># Apply the timezone we settled on to end_date if it wasn't supplied</span> |
| <span class="k">if</span> <span class="s1">'end_date'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'end_date'</span><span class="p">]:</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'end_date'</span><span class="p">],</span> <span class="nb">str</span><span class="p">):</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'end_date'</span><span class="p">]</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'end_date'</span><span class="p">],</span> <span class="n">timezone</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timezone</span> |
| <span class="p">)</span> |
| |
| <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="n">start_date</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="n">end_date</span><span class="p">)</span> |
| |
| <span class="c1"># also convert tasks</span> |
| <span class="k">if</span> <span class="s1">'start_date'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'start_date'</span><span class="p">]</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'start_date'</span><span class="p">])</span> |
| <span class="k">if</span> <span class="s1">'end_date'</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'end_date'</span><span class="p">]</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">convert_to_utc</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">default_args</span><span class="p">[</span><span class="s1">'end_date'</span><span class="p">])</span> |
| |
| <span class="c1"># Calculate the DAG's timetable.</span> |
| <span class="k">if</span> <span class="n">timetable</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span> <span class="o">=</span> <span class="n">create_timetable</span><span class="p">(</span><span class="n">schedule_interval</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">timezone</span><span class="p">)</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">schedule_interval</span><span class="p">,</span> <span class="n">ArgNotSet</span><span class="p">):</span> |
| <span class="n">schedule_interval</span> <span class="o">=</span> <span class="n">DEFAULT_SCHEDULE_INTERVAL</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">schedule_interval</span><span class="p">:</span> <span class="n">ScheduleInterval</span> <span class="o">=</span> <span class="n">schedule_interval</span> |
| <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">schedule_interval</span><span class="p">,</span> <span class="n">ArgNotSet</span><span class="p">):</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span> <span class="o">=</span> <span class="n">timetable</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">schedule_interval</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="o">.</span><span class="n">summary</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"cannot specify both 'schedule_interval' and 'timetable'"</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">template_searchpath</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span> |
| <span class="n">template_searchpath</span> <span class="o">=</span> <span class="p">[</span><span class="n">template_searchpath</span><span class="p">]</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">template_searchpath</span> <span class="o">=</span> <span class="n">template_searchpath</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">template_undefined</span> <span class="o">=</span> <span class="n">template_undefined</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">last_loaded</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">safe_dag_id</span> <span class="o">=</span> <span class="n">dag_id</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="s1">'__dot__'</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">max_active_runs</span> <span class="o">=</span> <span class="n">max_active_runs</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">dagrun_timeout</span> <span class="o">=</span> <span class="n">dagrun_timeout</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">sla_miss_callback</span> <span class="o">=</span> <span class="n">sla_miss_callback</span> |
| <span class="k">if</span> <span class="n">default_view</span> <span class="ow">in</span> <span class="n">DEFAULT_VIEW_PRESETS</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_default_view</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">default_view</span> |
| <span class="k">elif</span> <span class="n">default_view</span> <span class="o">==</span> <span class="s1">'tree'</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"`default_view` of 'tree' has been renamed to 'grid' -- please update your DAG"</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_default_view</span> <span class="o">=</span> <span class="s1">'grid'</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span> |
| <span class="sa">f</span><span class="s1">'Invalid values of dag.default_view: only support '</span> |
| <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">DEFAULT_VIEW_PRESETS</span><span class="si">}</span><span class="s1">, but get </span><span class="si">{</span><span class="n">default_view</span><span class="si">}</span><span class="s1">'</span> |
| <span class="p">)</span> |
| <span class="k">if</span> <span class="n">orientation</span> <span class="ow">in</span> <span class="n">ORIENTATION_PRESETS</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">orientation</span> <span class="o">=</span> <span class="n">orientation</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span> |
| <span class="sa">f</span><span class="s1">'Invalid values of dag.orientation: only support '</span> |
| <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">ORIENTATION_PRESETS</span><span class="si">}</span><span class="s1">, but get </span><span class="si">{</span><span class="n">orientation</span><span class="si">}</span><span class="s1">'</span> |
| <span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">catchup</span> <span class="o">=</span> <span class="n">catchup</span> |
| |
| <span class="bp">self</span><span class="o">.</span><span class="n">partial</span> <span class="o">=</span> <span class="kc">False</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">on_success_callback</span> <span class="o">=</span> <span class="n">on_success_callback</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">on_failure_callback</span> <span class="o">=</span> <span class="n">on_failure_callback</span> |
| |
| <span class="c1"># Keeps track of any extra edge metadata (sparse; will not contain all</span> |
| <span class="c1"># edges, so do not iterate over it for that). Outer key is upstream</span> |
| <span class="c1"># task ID, inner key is downstream task ID.</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">edge_info</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">EdgeInfoType</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span> |
| |
| <span class="c1"># To keep it in parity with Serialized DAGs</span> |
| <span class="c1"># and identify if DAG has on_*_callback without actually storing them in Serialized JSON</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">has_on_success_callback</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">on_success_callback</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">has_on_failure_callback</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">on_failure_callback</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> |
| |
| <span class="bp">self</span><span class="o">.</span><span class="n">doc_md</span> <span class="o">=</span> <span class="n">doc_md</span> |
| |
| <span class="bp">self</span><span class="o">.</span><span class="n">_access_control</span> <span class="o">=</span> <span class="n">DAG</span><span class="o">.</span><span class="n">_upgrade_outdated_dag_access_control</span><span class="p">(</span><span class="n">access_control</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">is_paused_upon_creation</span> <span class="o">=</span> <span class="n">is_paused_upon_creation</span> |
| |
| <span class="bp">self</span><span class="o">.</span><span class="n">jinja_environment_kwargs</span> <span class="o">=</span> <span class="n">jinja_environment_kwargs</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">render_template_as_native_obj</span> <span class="o">=</span> <span class="n">render_template_as_native_obj</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">tags</span> <span class="o">=</span> <span class="n">tags</span> <span class="ow">or</span> <span class="p">[]</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_task_group</span> <span class="o">=</span> <span class="n">TaskGroup</span><span class="o">.</span><span class="n">create_root</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">validate_schedule_and_params</span><span class="p">()</span> |
| |
| <div class="viewcode-block" id="DAG.__repr__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.__repr__">[docs]</a> <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="k">return</span> <span class="sa">f</span><span class="s2">"<DAG: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="si">}</span><span class="s2">>"</span></div> |
| |
| <div class="viewcode-block" id="DAG.__eq__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.__eq__">[docs]</a> <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span> |
| <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">):</span> |
| <span class="c1"># Use getattr() instead of __dict__ as __dict__ doesn't return</span> |
| <span class="c1"># correct values for properties.</span> |
| <span class="k">return</span> <span class="nb">all</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">c</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="o">==</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">c</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_comps</span><span class="p">)</span> |
| <span class="k">return</span> <span class="kc">False</span></div> |
| |
| <div class="viewcode-block" id="DAG.__ne__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.__ne__">[docs]</a> <span class="k">def</span> <span class="fm">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span> |
| <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span> <span class="o">==</span> <span class="n">other</span></div> |
| |
| <div class="viewcode-block" id="DAG.__lt__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.__lt__">[docs]</a> <span class="k">def</span> <span class="fm">__lt__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span> <span class="o"><</span> <span class="n">other</span><span class="o">.</span><span class="n">dag_id</span></div> |
| |
| <div class="viewcode-block" id="DAG.__hash__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.__hash__">[docs]</a> <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="n">hash_components</span> <span class="o">=</span> <span class="p">[</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)]</span> |
| <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_comps</span><span class="p">:</span> |
| <span class="c1"># task_ids returns a list and lists can't be hashed</span> |
| <span class="k">if</span> <span class="n">c</span> <span class="o">==</span> <span class="s1">'task_ids'</span><span class="p">:</span> |
| <span class="n">val</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">c</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="nb">hash</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> |
| <span class="n">hash_components</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">TypeError</span><span class="p">:</span> |
| <span class="n">hash_components</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">repr</span><span class="p">(</span><span class="n">val</span><span class="p">))</span> |
| <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">tuple</span><span class="p">(</span><span class="n">hash_components</span><span class="p">))</span></div> |
| |
| <span class="c1"># Context Manager -----------------------------------------------</span> |
| <div class="viewcode-block" id="DAG.__enter__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.__enter__">[docs]</a> <span class="k">def</span> <span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="n">DagContext</span><span class="o">.</span><span class="n">push_context_managed_dag</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| <span class="k">return</span> <span class="bp">self</span></div> |
| |
| <div class="viewcode-block" id="DAG.__exit__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.__exit__">[docs]</a> <span class="k">def</span> <span class="fm">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">_type</span><span class="p">,</span> <span class="n">_value</span><span class="p">,</span> <span class="n">_tb</span><span class="p">):</span> |
| <span class="n">DagContext</span><span class="o">.</span><span class="n">pop_context_managed_dag</span><span class="p">()</span></div> |
| |
| <span class="c1"># /Context Manager ----------------------------------------------</span> |
| |
| <span class="nd">@staticmethod</span> |
| <span class="k">def</span> <span class="nf">_upgrade_outdated_dag_access_control</span><span class="p">(</span><span class="n">access_control</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Looks for outdated dag level actions (can_dag_read and can_dag_edit) in DAG</span> |
| <span class="sd"> access_controls (for example, {'role1': {'can_dag_read'}, 'role2': {'can_dag_read', 'can_dag_edit'}})</span> |
| <span class="sd"> and replaces them with updated actions (can_read and can_edit).</span> |
| <span class="sd"> """</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">access_control</span><span class="p">:</span> |
| <span class="k">return</span> <span class="kc">None</span> |
| <span class="n">new_perm_mapping</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="n">permissions</span><span class="o">.</span><span class="n">DEPRECATED_ACTION_CAN_DAG_READ</span><span class="p">:</span> <span class="n">permissions</span><span class="o">.</span><span class="n">ACTION_CAN_READ</span><span class="p">,</span> |
| <span class="n">permissions</span><span class="o">.</span><span class="n">DEPRECATED_ACTION_CAN_DAG_EDIT</span><span class="p">:</span> <span class="n">permissions</span><span class="o">.</span><span class="n">ACTION_CAN_EDIT</span><span class="p">,</span> |
| <span class="p">}</span> |
| <span class="n">updated_access_control</span> <span class="o">=</span> <span class="p">{}</span> |
| <span class="k">for</span> <span class="n">role</span><span class="p">,</span> <span class="n">perms</span> <span class="ow">in</span> <span class="n">access_control</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> |
| <span class="n">updated_access_control</span><span class="p">[</span><span class="n">role</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">new_perm_mapping</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">perm</span><span class="p">,</span> <span class="n">perm</span><span class="p">)</span> <span class="k">for</span> <span class="n">perm</span> <span class="ow">in</span> <span class="n">perms</span><span class="p">}</span> |
| |
| <span class="k">if</span> <span class="n">access_control</span> <span class="o">!=</span> <span class="n">updated_access_control</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"The 'can_dag_read' and 'can_dag_edit' permissions are deprecated. "</span> |
| <span class="s2">"Please use 'can_read' and 'can_edit', respectively."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> |
| <span class="p">)</span> |
| |
| <span class="k">return</span> <span class="n">updated_access_control</span> |
| |
| <div class="viewcode-block" id="DAG.date_range"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.date_range">[docs]</a> <span class="k">def</span> <span class="nf">date_range</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="p">:</span> <span class="n">pendulum</span><span class="o">.</span><span class="n">DateTime</span><span class="p">,</span> |
| <span class="n">num</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">datetime</span><span class="p">]:</span> |
| <span class="n">message</span> <span class="o">=</span> <span class="s2">"`DAG.date_range()` is deprecated."</span> |
| <span class="k">if</span> <span class="n">num</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">,</span> <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> |
| <span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">():</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="s2">"ignore"</span><span class="p">,</span> <span class="ne">DeprecationWarning</span><span class="p">)</span> |
| <span class="k">return</span> <span class="n">utils_date_range</span><span class="p">(</span> |
| <span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span> <span class="n">num</span><span class="o">=</span><span class="n">num</span><span class="p">,</span> <span class="n">delta</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">normalized_schedule_interval</span> |
| <span class="p">)</span> |
| <span class="n">message</span> <span class="o">+=</span> <span class="s2">" Please use `DAG.iter_dagrun_infos_between(..., align=False)` instead."</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">,</span> <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">end_date</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">coerced_end_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">coerced_end_date</span> <span class="o">=</span> <span class="n">end_date</span> |
| <span class="n">it</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">iter_dagrun_infos_between</span><span class="p">(</span><span class="n">start_date</span><span class="p">,</span> <span class="n">pendulum</span><span class="o">.</span><span class="n">instance</span><span class="p">(</span><span class="n">coerced_end_date</span><span class="p">),</span> <span class="n">align</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> |
| <span class="k">return</span> <span class="p">[</span><span class="n">info</span><span class="o">.</span><span class="n">logical_date</span> <span class="k">for</span> <span class="n">info</span> <span class="ow">in</span> <span class="n">it</span><span class="p">]</span></div> |
| |
| <div class="viewcode-block" id="DAG.is_fixed_time_schedule"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.is_fixed_time_schedule">[docs]</a> <span class="k">def</span> <span class="nf">is_fixed_time_schedule</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"`DAG.is_fixed_time_schedule()` is deprecated."</span><span class="p">,</span> |
| <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="o">.</span><span class="n">_should_fix_dst</span> |
| <span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span> |
| <span class="k">return</span> <span class="kc">True</span></div> |
| |
| <div class="viewcode-block" id="DAG.following_schedule"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.following_schedule">[docs]</a> <span class="k">def</span> <span class="nf">following_schedule</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dttm</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Calculates the following schedule for this dag in UTC.</span> |
| |
| <span class="sd"> :param dttm: utc datetime</span> |
| <span class="sd"> :return: utc datetime</span> |
| <span class="sd"> """</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"`DAG.following_schedule()` is deprecated. Use `DAG.next_dagrun_info(restricted=False)` instead."</span><span class="p">,</span> |
| <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">data_interval</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">infer_automated_data_interval</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="n">dttm</span><span class="p">))</span> |
| <span class="n">next_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun_info</span><span class="p">(</span><span class="n">data_interval</span><span class="p">,</span> <span class="n">restricted</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">next_info</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">return</span> <span class="kc">None</span> |
| <span class="k">return</span> <span class="n">next_info</span><span class="o">.</span><span class="n">data_interval</span><span class="o">.</span><span class="n">start</span></div> |
| |
| <div class="viewcode-block" id="DAG.previous_schedule"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.previous_schedule">[docs]</a> <span class="k">def</span> <span class="nf">previous_schedule</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dttm</span><span class="p">):</span> |
| <span class="kn">from</span> <span class="nn">airflow.timetables.interval</span> <span class="kn">import</span> <span class="n">_DataIntervalTimetable</span> |
| |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"`DAG.previous_schedule()` is deprecated."</span><span class="p">,</span> |
| <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="p">,</span> <span class="n">_DataIntervalTimetable</span><span class="p">):</span> |
| <span class="k">return</span> <span class="kc">None</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="o">.</span><span class="n">_get_prev</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="n">dttm</span><span class="p">))</span></div> |
| |
| <div class="viewcode-block" id="DAG.get_next_data_interval"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_next_data_interval">[docs]</a> <span class="k">def</span> <span class="nf">get_next_data_interval</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dag_model</span><span class="p">:</span> <span class="s2">"DagModel"</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">DataInterval</span><span class="p">]:</span> |
| <span class="sd">"""Get the data interval of the next scheduled run.</span> |
| |
| <span class="sd"> For compatibility, this method infers the data interval from the DAG's</span> |
| <span class="sd"> schedule if the run does not have an explicit one set, which is possible</span> |
| <span class="sd"> for runs created prior to AIP-39.</span> |
| |
| <span class="sd"> This function is private to Airflow core and should not be depended as a</span> |
| <span class="sd"> part of the Python API.</span> |
| |
| <span class="sd"> :meta private:</span> |
| <span class="sd"> """</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">!=</span> <span class="n">dag_model</span><span class="o">.</span><span class="n">dag_id</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Arguments refer to different DAGs: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="si">}</span><span class="s2"> != </span><span class="si">{</span><span class="n">dag_model</span><span class="o">.</span><span class="n">dag_id</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">dag_model</span><span class="o">.</span><span class="n">next_dagrun</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> <span class="c1"># Next run not scheduled.</span> |
| <span class="k">return</span> <span class="kc">None</span> |
| <span class="n">data_interval</span> <span class="o">=</span> <span class="n">dag_model</span><span class="o">.</span><span class="n">next_dagrun_data_interval</span> |
| <span class="k">if</span> <span class="n">data_interval</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">data_interval</span> |
| |
| <span class="c1"># Compatibility: A run was scheduled without an explicit data interval.</span> |
| <span class="c1"># This means the run was scheduled before AIP-39 implementation. Try to</span> |
| <span class="c1"># infer from the logical date.</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">infer_automated_data_interval</span><span class="p">(</span><span class="n">dag_model</span><span class="o">.</span><span class="n">next_dagrun</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="DAG.get_run_data_interval"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_run_data_interval">[docs]</a> <span class="k">def</span> <span class="nf">get_run_data_interval</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">run</span><span class="p">:</span> <span class="n">DagRun</span><span class="p">)</span> <span class="o">-></span> <span class="n">DataInterval</span><span class="p">:</span> |
| <span class="sd">"""Get the data interval of this run.</span> |
| |
| <span class="sd"> For compatibility, this method infers the data interval from the DAG's</span> |
| <span class="sd"> schedule if the run does not have an explicit one set, which is possible for</span> |
| <span class="sd"> runs created prior to AIP-39.</span> |
| |
| <span class="sd"> This function is private to Airflow core and should not be depended as a</span> |
| <span class="sd"> part of the Python API.</span> |
| |
| <span class="sd"> :meta private:</span> |
| <span class="sd"> """</span> |
| <span class="k">if</span> <span class="n">run</span><span class="o">.</span><span class="n">dag_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">run</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Arguments refer to different DAGs: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="si">}</span><span class="s2"> != </span><span class="si">{</span><span class="n">run</span><span class="o">.</span><span class="n">dag_id</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <span class="n">data_interval</span> <span class="o">=</span> <span class="n">_get_model_data_interval</span><span class="p">(</span><span class="n">run</span><span class="p">,</span> <span class="s2">"data_interval_start"</span><span class="p">,</span> <span class="s2">"data_interval_end"</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">data_interval</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">data_interval</span> |
| <span class="c1"># Compatibility: runs created before AIP-39 implementation don't have an</span> |
| <span class="c1"># explicit data interval. Try to infer from the logical date.</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">infer_automated_data_interval</span><span class="p">(</span><span class="n">run</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="DAG.infer_automated_data_interval"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.infer_automated_data_interval">[docs]</a> <span class="k">def</span> <span class="nf">infer_automated_data_interval</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">logical_date</span><span class="p">:</span> <span class="n">datetime</span><span class="p">)</span> <span class="o">-></span> <span class="n">DataInterval</span><span class="p">:</span> |
| <span class="sd">"""Infer a data interval for a run against this DAG.</span> |
| |
| <span class="sd"> This method is used to bridge runs created prior to AIP-39</span> |
| <span class="sd"> implementation, which do not have an explicit data interval. Therefore,</span> |
| <span class="sd"> this method only considers ``schedule_interval`` values valid prior to</span> |
| <span class="sd"> Airflow 2.2.</span> |
| |
| <span class="sd"> DO NOT use this method is there is a known data interval.</span> |
| <span class="sd"> """</span> |
| <span class="n">timetable_type</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="p">)</span> |
| <span class="k">if</span> <span class="nb">issubclass</span><span class="p">(</span><span class="n">timetable_type</span><span class="p">,</span> <span class="p">(</span><span class="n">NullTimetable</span><span class="p">,</span> <span class="n">OnceTimetable</span><span class="p">)):</span> |
| <span class="k">return</span> <span class="n">DataInterval</span><span class="o">.</span><span class="n">exact</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="n">logical_date</span><span class="p">))</span> |
| <span class="n">start</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="n">logical_date</span><span class="p">)</span> |
| <span class="k">if</span> <span class="nb">issubclass</span><span class="p">(</span><span class="n">timetable_type</span><span class="p">,</span> <span class="n">CronDataIntervalTimetable</span><span class="p">):</span> |
| <span class="n">end</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">CronDataIntervalTimetable</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="p">)</span><span class="o">.</span><span class="n">_get_next</span><span class="p">(</span><span class="n">start</span><span class="p">)</span> |
| <span class="k">elif</span> <span class="nb">issubclass</span><span class="p">(</span><span class="n">timetable_type</span><span class="p">,</span> <span class="n">DeltaDataIntervalTimetable</span><span class="p">):</span> |
| <span class="n">end</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">DeltaDataIntervalTimetable</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="p">)</span><span class="o">.</span><span class="n">_get_next</span><span class="p">(</span><span class="n">start</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Not a valid timetable: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="si">!r}</span><span class="s2">"</span><span class="p">)</span> |
| <span class="k">return</span> <span class="n">DataInterval</span><span class="p">(</span><span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="DAG.next_dagrun_info"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.next_dagrun_info">[docs]</a> <span class="k">def</span> <span class="nf">next_dagrun_info</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">last_automated_dagrun</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">DataInterval</span><span class="p">],</span> |
| <span class="o">*</span><span class="p">,</span> |
| <span class="n">restricted</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">DagRunInfo</span><span class="p">]:</span> |
| <span class="sd">"""Get information about the next DagRun of this dag after ``date_last_automated_dagrun``.</span> |
| |
| <span class="sd"> This calculates what time interval the next DagRun should operate on</span> |
| <span class="sd"> (its execution date) and when it can be scheduled, according to the</span> |
| <span class="sd"> dag's timetable, start_date, end_date, etc. This doesn't check max</span> |
| <span class="sd"> active run or any other "max_active_tasks" type limits, but only</span> |
| <span class="sd"> performs calculations based on the various date and interval fields of</span> |
| <span class="sd"> this dag and its tasks.</span> |
| |
| <span class="sd"> :param last_automated_dagrun: The ``max(execution_date)`` of</span> |
| <span class="sd"> existing "automated" DagRuns for this dag (scheduled or backfill,</span> |
| <span class="sd"> but not manual).</span> |
| <span class="sd"> :param restricted: If set to *False* (default is *True*), ignore</span> |
| <span class="sd"> ``start_date``, ``end_date``, and ``catchup`` specified on the DAG</span> |
| <span class="sd"> or tasks.</span> |
| <span class="sd"> :return: DagRunInfo of the next dagrun, or None if a dagrun is not</span> |
| <span class="sd"> going to be scheduled.</span> |
| <span class="sd"> """</span> |
| <span class="c1"># Never schedule a subdag. It will be scheduled by its parent dag.</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_subdag</span><span class="p">:</span> |
| <span class="k">return</span> <span class="kc">None</span> |
| |
| <span class="n">data_interval</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">last_automated_dagrun</span><span class="p">,</span> <span class="n">datetime</span><span class="p">):</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"Passing a datetime to DAG.next_dagrun_info is deprecated. Use a DataInterval instead."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">data_interval</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">infer_automated_data_interval</span><span class="p">(</span> |
| <span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="n">last_automated_dagrun</span><span class="p">)</span> |
| <span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">data_interval</span> <span class="o">=</span> <span class="n">last_automated_dagrun</span> |
| <span class="k">if</span> <span class="n">restricted</span><span class="p">:</span> |
| <span class="n">restriction</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_time_restriction</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">restriction</span> <span class="o">=</span> <span class="n">TimeRestriction</span><span class="p">(</span><span class="n">earliest</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">latest</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">catchup</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="n">info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="o">.</span><span class="n">next_dagrun_info</span><span class="p">(</span> |
| <span class="n">last_automated_data_interval</span><span class="o">=</span><span class="n">data_interval</span><span class="p">,</span> |
| <span class="n">restriction</span><span class="o">=</span><span class="n">restriction</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span> |
| <span class="s2">"Failed to fetch run info after data interval </span><span class="si">%s</span><span class="s2"> for DAG </span><span class="si">%r</span><span class="s2">"</span><span class="p">,</span> |
| <span class="n">data_interval</span><span class="p">,</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">info</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">return</span> <span class="n">info</span></div> |
| |
| <div class="viewcode-block" id="DAG.next_dagrun_after_date"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.next_dagrun_after_date">[docs]</a> <span class="k">def</span> <span class="nf">next_dagrun_after_date</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">date_last_automated_dagrun</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">pendulum</span><span class="o">.</span><span class="n">DateTime</span><span class="p">]):</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"`DAG.next_dagrun_after_date()` is deprecated. Please use `DAG.next_dagrun_info()` instead."</span><span class="p">,</span> |
| <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">if</span> <span class="n">date_last_automated_dagrun</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">data_interval</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">data_interval</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">infer_automated_data_interval</span><span class="p">(</span><span class="n">date_last_automated_dagrun</span><span class="p">)</span> |
| <span class="n">info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun_info</span><span class="p">(</span><span class="n">data_interval</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">info</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">return</span> <span class="kc">None</span> |
| <span class="k">return</span> <span class="n">info</span><span class="o">.</span><span class="n">run_after</span></div> |
| |
| <span class="nd">@cached_property</span> |
| <span class="k">def</span> <span class="nf">_time_restriction</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">TimeRestriction</span><span class="p">:</span> |
| <span class="n">start_dates</span> <span class="o">=</span> <span class="p">[</span><span class="n">t</span><span class="o">.</span><span class="n">start_date</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">start_date</span><span class="p">]</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">start_dates</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">start_date</span><span class="p">)</span> |
| <span class="n">earliest</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">if</span> <span class="n">start_dates</span><span class="p">:</span> |
| <span class="n">earliest</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="nb">min</span><span class="p">(</span><span class="n">start_dates</span><span class="p">))</span> |
| <span class="n">latest</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> |
| <span class="n">end_dates</span> <span class="o">=</span> <span class="p">[</span><span class="n">t</span><span class="o">.</span><span class="n">end_date</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">end_date</span><span class="p">]</span> |
| <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">end_dates</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">):</span> <span class="c1"># not exists null end_date</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">end_dates</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">end_date</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">end_dates</span><span class="p">:</span> |
| <span class="n">latest</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="nb">max</span><span class="p">(</span><span class="n">end_dates</span><span class="p">))</span> |
| <span class="k">return</span> <span class="n">TimeRestriction</span><span class="p">(</span><span class="n">earliest</span><span class="p">,</span> <span class="n">latest</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">catchup</span><span class="p">)</span> |
| |
| <div class="viewcode-block" id="DAG.iter_dagrun_infos_between"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.iter_dagrun_infos_between">[docs]</a> <span class="k">def</span> <span class="nf">iter_dagrun_infos_between</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">earliest</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">pendulum</span><span class="o">.</span><span class="n">DateTime</span><span class="p">],</span> |
| <span class="n">latest</span><span class="p">:</span> <span class="n">pendulum</span><span class="o">.</span><span class="n">DateTime</span><span class="p">,</span> |
| <span class="o">*</span><span class="p">,</span> |
| <span class="n">align</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="n">Iterable</span><span class="p">[</span><span class="n">DagRunInfo</span><span class="p">]:</span> |
| <span class="sd">"""Yield DagRunInfo using this DAG's timetable between given interval.</span> |
| |
| <span class="sd"> DagRunInfo instances yielded if their ``logical_date`` is not earlier</span> |
| <span class="sd"> than ``earliest``, nor later than ``latest``. The instances are ordered</span> |
| <span class="sd"> by their ``logical_date`` from earliest to latest.</span> |
| |
| <span class="sd"> If ``align`` is ``False``, the first run will happen immediately on</span> |
| <span class="sd"> ``earliest``, even if it does not fall on the logical timetable schedule.</span> |
| <span class="sd"> The default is ``True``, but subdags will ignore this value and always</span> |
| <span class="sd"> behave as if this is set to ``False`` for backward compatibility.</span> |
| |
| <span class="sd"> Example: A DAG is scheduled to run every midnight (``0 0 * * *``). If</span> |
| <span class="sd"> ``earliest`` is ``2021-06-03 23:00:00``, the first DagRunInfo would be</span> |
| <span class="sd"> ``2021-06-03 23:00:00`` if ``align=False``, and ``2021-06-04 00:00:00``</span> |
| <span class="sd"> if ``align=True``.</span> |
| <span class="sd"> """</span> |
| <span class="k">if</span> <span class="n">earliest</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">earliest</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_time_restriction</span><span class="o">.</span><span class="n">earliest</span> |
| <span class="k">if</span> <span class="n">earliest</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"earliest was None and we had no value in time_restriction to fallback on"</span><span class="p">)</span> |
| <span class="n">earliest</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="n">earliest</span><span class="p">)</span> |
| <span class="n">latest</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="n">latest</span><span class="p">)</span> |
| |
| <span class="n">restriction</span> <span class="o">=</span> <span class="n">TimeRestriction</span><span class="p">(</span><span class="n">earliest</span><span class="p">,</span> <span class="n">latest</span><span class="p">,</span> <span class="n">catchup</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
| |
| <span class="c1"># HACK: Sub-DAGs are currently scheduled differently. For example, say</span> |
| <span class="c1"># the schedule is @daily and start is 2021-06-03 22:16:00, a top-level</span> |
| <span class="c1"># DAG should be first scheduled to run on midnight 2021-06-04, but a</span> |
| <span class="c1"># sub-DAG should be first scheduled to run RIGHT NOW. We can change</span> |
| <span class="c1"># this, but since sub-DAGs are going away in 3.0 anyway, let's keep</span> |
| <span class="c1"># compatibility for now and remove this entirely later.</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_subdag</span><span class="p">:</span> |
| <span class="n">align</span> <span class="o">=</span> <span class="kc">False</span> |
| |
| <span class="k">try</span><span class="p">:</span> |
| <span class="n">info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="o">.</span><span class="n">next_dagrun_info</span><span class="p">(</span> |
| <span class="n">last_automated_data_interval</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">restriction</span><span class="o">=</span><span class="n">restriction</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span> |
| <span class="s2">"Failed to fetch run info after data interval </span><span class="si">%s</span><span class="s2"> for DAG </span><span class="si">%r</span><span class="s2">"</span><span class="p">,</span> |
| <span class="kc">None</span><span class="p">,</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">info</span> <span class="o">=</span> <span class="kc">None</span> |
| |
| <span class="k">if</span> <span class="n">info</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="c1"># No runs to be scheduled between the user-supplied timeframe. But</span> |
| <span class="c1"># if align=False, "invent" a data interval for the timeframe itself.</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">align</span><span class="p">:</span> |
| <span class="k">yield</span> <span class="n">DagRunInfo</span><span class="o">.</span><span class="n">interval</span><span class="p">(</span><span class="n">earliest</span><span class="p">,</span> <span class="n">latest</span><span class="p">)</span> |
| <span class="k">return</span> |
| |
| <span class="c1"># If align=False and earliest does not fall on the timetable's logical</span> |
| <span class="c1"># schedule, "invent" a data interval for it.</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">align</span> <span class="ow">and</span> <span class="n">info</span><span class="o">.</span><span class="n">logical_date</span> <span class="o">!=</span> <span class="n">earliest</span><span class="p">:</span> |
| <span class="k">yield</span> <span class="n">DagRunInfo</span><span class="o">.</span><span class="n">interval</span><span class="p">(</span><span class="n">earliest</span><span class="p">,</span> <span class="n">info</span><span class="o">.</span><span class="n">data_interval</span><span class="o">.</span><span class="n">start</span><span class="p">)</span> |
| |
| <span class="c1"># Generate naturally according to schedule.</span> |
| <span class="k">while</span> <span class="n">info</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">yield</span> <span class="n">info</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="n">info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="o">.</span><span class="n">next_dagrun_info</span><span class="p">(</span> |
| <span class="n">last_automated_data_interval</span><span class="o">=</span><span class="n">info</span><span class="o">.</span><span class="n">data_interval</span><span class="p">,</span> |
| <span class="n">restriction</span><span class="o">=</span><span class="n">restriction</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span> |
| <span class="s2">"Failed to fetch run info after data interval </span><span class="si">%s</span><span class="s2"> for DAG </span><span class="si">%r</span><span class="s2">"</span><span class="p">,</span> |
| <span class="n">info</span><span class="o">.</span><span class="n">data_interval</span> <span class="k">if</span> <span class="n">info</span> <span class="k">else</span> <span class="s2">"<NONE>"</span><span class="p">,</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">break</span></div> |
| |
| <div class="viewcode-block" id="DAG.get_run_dates"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_run_dates">[docs]</a> <span class="k">def</span> <span class="nf">get_run_dates</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">start_date</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Returns a list of dates between the interval received as parameter using this</span> |
| <span class="sd"> dag's schedule interval. Returned dates can be used for execution dates.</span> |
| |
| <span class="sd"> :param start_date: The start date of the interval.</span> |
| <span class="sd"> :param end_date: The end date of the interval. Defaults to ``timezone.utcnow()``.</span> |
| <span class="sd"> :return: A list of dates within the interval following the dag's schedule.</span> |
| <span class="sd"> :rtype: list</span> |
| <span class="sd"> """</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"`DAG.get_run_dates()` is deprecated. Please use `DAG.iter_dagrun_infos_between()` instead."</span><span class="p">,</span> |
| <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">earliest</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="n">start_date</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">end_date</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">latest</span> <span class="o">=</span> <span class="n">pendulum</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">latest</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="n">end_date</span><span class="p">)</span> |
| <span class="k">return</span> <span class="p">[</span><span class="n">info</span><span class="o">.</span><span class="n">logical_date</span> <span class="k">for</span> <span class="n">info</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">iter_dagrun_infos_between</span><span class="p">(</span><span class="n">earliest</span><span class="p">,</span> <span class="n">latest</span><span class="p">)]</span></div> |
| |
| <div class="viewcode-block" id="DAG.normalize_schedule"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.normalize_schedule">[docs]</a> <span class="k">def</span> <span class="nf">normalize_schedule</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dttm</span><span class="p">):</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"`DAG.normalize_schedule()` is deprecated."</span><span class="p">,</span> |
| <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">():</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="s2">"ignore"</span><span class="p">,</span> <span class="ne">DeprecationWarning</span><span class="p">)</span> |
| <span class="n">following</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">following_schedule</span><span class="p">(</span><span class="n">dttm</span><span class="p">)</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">following</span><span class="p">:</span> <span class="c1"># in case of @once</span> |
| <span class="k">return</span> <span class="n">dttm</span> |
| <span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">():</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="s2">"ignore"</span><span class="p">,</span> <span class="ne">DeprecationWarning</span><span class="p">)</span> |
| <span class="n">previous_of_following</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">previous_schedule</span><span class="p">(</span><span class="n">following</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">previous_of_following</span> <span class="o">!=</span> <span class="n">dttm</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">following</span> |
| <span class="k">return</span> <span class="n">dttm</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.get_last_dagrun"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_last_dagrun">[docs]</a> <span class="k">def</span> <span class="nf">get_last_dagrun</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">,</span> <span class="n">include_externally_triggered</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">get_last_dagrun</span><span class="p">(</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span> <span class="n">include_externally_triggered</span><span class="o">=</span><span class="n">include_externally_triggered</span></div> |
| <span class="p">)</span> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.has_dag_runs"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.has_dag_runs">[docs]</a> <span class="k">def</span> <span class="nf">has_dag_runs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">,</span> <span class="n">include_externally_triggered</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> |
| <span class="k">return</span> <span class="p">(</span> |
| <span class="n">get_last_dagrun</span><span class="p">(</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span> <span class="n">include_externally_triggered</span><span class="o">=</span><span class="n">include_externally_triggered</span> |
| <span class="p">)</span> |
| <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span></div> |
| <span class="p">)</span> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.dag_id"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.dag_id">[docs]</a> <span class="k">def</span> <span class="nf">dag_id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dag_id</span></div> |
| |
| <span class="nd">@dag_id</span><span class="o">.</span><span class="n">setter</span> |
| <span class="k">def</span> <span class="nf">dag_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_dag_id</span> <span class="o">=</span> <span class="n">value</span> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.is_subdag"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.is_subdag">[docs]</a> <span class="k">def</span> <span class="nf">is_subdag</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">parent_dag</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.full_filepath"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.full_filepath">[docs]</a> <span class="k">def</span> <span class="nf">full_filepath</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <span class="sd">""":meta private:"""</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"DAG.full_filepath is deprecated in favour of fileloc"</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fileloc</span></div> |
| |
| <span class="nd">@full_filepath</span><span class="o">.</span><span class="n">setter</span> |
| <span class="k">def</span> <span class="nf">full_filepath</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"DAG.full_filepath is deprecated in favour of fileloc"</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">=</span> <span class="n">value</span> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.concurrency"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.concurrency">[docs]</a> <span class="k">def</span> <span class="nf">concurrency</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span> |
| <span class="c1"># TODO: Remove in Airflow 3.0</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"The 'DAG.concurrency' attribute is deprecated. Please use 'DAG.max_active_tasks'."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_max_active_tasks</span></div> |
| |
| <span class="nd">@concurrency</span><span class="o">.</span><span class="n">setter</span> |
| <span class="k">def</span> <span class="nf">concurrency</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_max_active_tasks</span> <span class="o">=</span> <span class="n">value</span> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.max_active_tasks"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.max_active_tasks">[docs]</a> <span class="k">def</span> <span class="nf">max_active_tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_max_active_tasks</span></div> |
| |
| <span class="nd">@max_active_tasks</span><span class="o">.</span><span class="n">setter</span> |
| <span class="k">def</span> <span class="nf">max_active_tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_max_active_tasks</span> <span class="o">=</span> <span class="n">value</span> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.access_control"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.access_control">[docs]</a> <span class="k">def</span> <span class="nf">access_control</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_access_control</span></div> |
| |
| <span class="nd">@access_control</span><span class="o">.</span><span class="n">setter</span> |
| <span class="k">def</span> <span class="nf">access_control</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_access_control</span> <span class="o">=</span> <span class="n">DAG</span><span class="o">.</span><span class="n">_upgrade_outdated_dag_access_control</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.description"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.description">[docs]</a> <span class="k">def</span> <span class="nf">description</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_description</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.default_view"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.default_view">[docs]</a> <span class="k">def</span> <span class="nf">default_view</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_default_view</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.pickle_id"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.pickle_id">[docs]</a> <span class="k">def</span> <span class="nf">pickle_id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]:</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pickle_id</span></div> |
| |
| <span class="nd">@pickle_id</span><span class="o">.</span><span class="n">setter</span> |
| <span class="k">def</span> <span class="nf">pickle_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_pickle_id</span> <span class="o">=</span> <span class="n">value</span> |
| |
| <div class="viewcode-block" id="DAG.param"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.param">[docs]</a> <span class="k">def</span> <span class="nf">param</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">Any</span> <span class="o">=</span> <span class="n">NOTSET</span><span class="p">)</span> <span class="o">-></span> <span class="n">DagParam</span><span class="p">:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Return a DagParam object for current dag.</span> |
| |
| <span class="sd"> :param name: dag parameter name.</span> |
| <span class="sd"> :param default: fallback value for dag parameter.</span> |
| <span class="sd"> :return: DagParam instance for specified name and current dag.</span> |
| <span class="sd"> """</span> |
| <span class="k">return</span> <span class="n">DagParam</span><span class="p">(</span><span class="n">current_dag</span><span class="o">=</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">default</span><span class="p">)</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.tasks"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.tasks">[docs]</a> <span class="k">def</span> <span class="nf">tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">Operator</span><span class="p">]:</span> |
| <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">values</span><span class="p">())</span></div> |
| |
| <span class="nd">@tasks</span><span class="o">.</span><span class="n">setter</span> |
| <span class="k">def</span> <span class="nf">tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">val</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="s1">'DAG.tasks can not be modified. Use dag.add_task() instead.'</span><span class="p">)</span> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.task_ids"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.task_ids">[docs]</a> <span class="k">def</span> <span class="nf">task_ids</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span> |
| <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.task_group"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.task_group">[docs]</a> <span class="k">def</span> <span class="nf">task_group</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"TaskGroup"</span><span class="p">:</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_task_group</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.filepath"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.filepath">[docs]</a> <span class="k">def</span> <span class="nf">filepath</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <span class="sd">""":meta private:"""</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"filepath is deprecated, use relative_fileloc instead"</span><span class="p">,</span> <span class="ne">DeprecationWarning</span><span class="p">,</span> <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">relative_fileloc</span><span class="p">)</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.relative_fileloc"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.relative_fileloc">[docs]</a> <span class="k">def</span> <span class="nf">relative_fileloc</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">:</span> |
| <span class="sd">"""File location of the importable dag 'file' relative to the configured DAGs folder."""</span> |
| <span class="n">path</span> <span class="o">=</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fileloc</span><span class="p">)</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">path</span><span class="o">.</span><span class="n">relative_to</span><span class="p">(</span><span class="n">settings</span><span class="o">.</span><span class="n">DAGS_FOLDER</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> |
| <span class="c1"># Not relative to DAGS_FOLDER.</span> |
| <span class="k">return</span> <span class="n">path</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.folder"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.folder">[docs]</a> <span class="k">def</span> <span class="nf">folder</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <span class="sd">"""Folder location of where the DAG object is instantiated."""</span> |
| <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fileloc</span><span class="p">)</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.owner"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.owner">[docs]</a> <span class="k">def</span> <span class="nf">owner</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Return list of all owners found in DAG tasks.</span> |
| |
| <span class="sd"> :return: Comma separated list of owners in DAG tasks</span> |
| <span class="sd"> :rtype: str</span> |
| <span class="sd"> """</span> |
| <span class="k">return</span> <span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">({</span><span class="n">t</span><span class="o">.</span><span class="n">owner</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">})</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.allow_future_exec_dates"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.allow_future_exec_dates">[docs]</a> <span class="k">def</span> <span class="nf">allow_future_exec_dates</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">settings</span><span class="o">.</span><span class="n">ALLOW_FUTURE_EXEC_DATES</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="o">.</span><span class="n">can_run</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.get_concurrency_reached"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_concurrency_reached">[docs]</a> <span class="k">def</span> <span class="nf">get_concurrency_reached</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Returns a boolean indicating whether the max_active_tasks limit for this DAG</span> |
| <span class="sd"> has been reached</span> |
| <span class="sd"> """</span> |
| <span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span> |
| <span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">func</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">task_id</span><span class="p">))</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span> |
| <span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> |
| <span class="n">TI</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="n">qry</span><span class="o">.</span><span class="n">scalar</span><span class="p">()</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_active_tasks</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.concurrency_reached"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.concurrency_reached">[docs]</a> <span class="k">def</span> <span class="nf">concurrency_reached</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="sd">"""This attribute is deprecated. Please use `airflow.models.DAG.get_concurrency_reached` method."""</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"This attribute is deprecated. Please use `airflow.models.DAG.get_concurrency_reached` method."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_concurrency_reached</span><span class="p">()</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.get_is_active"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_is_active">[docs]</a> <span class="k">def</span> <span class="nf">get_is_active</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="kc">None</span><span class="p">]:</span> |
| <span class="sd">"""Returns a boolean indicating whether this DAG is active"""</span> |
| <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">is_active</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">scalar</span><span class="p">()</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.get_is_paused"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_is_paused">[docs]</a> <span class="k">def</span> <span class="nf">get_is_paused</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="kc">None</span><span class="p">]:</span> |
| <span class="sd">"""Returns a boolean indicating whether this DAG is paused"""</span> |
| <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">is_paused</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">scalar</span><span class="p">()</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.is_paused"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.is_paused">[docs]</a> <span class="k">def</span> <span class="nf">is_paused</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="sd">"""This attribute is deprecated. Please use `airflow.models.DAG.get_is_paused` method."""</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"This attribute is deprecated. Please use `airflow.models.DAG.get_is_paused` method."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_is_paused</span><span class="p">()</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.normalized_schedule_interval"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.normalized_schedule_interval">[docs]</a> <span class="k">def</span> <span class="nf">normalized_schedule_interval</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">ScheduleInterval</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"DAG.normalized_schedule_interval() is deprecated."</span><span class="p">,</span> |
| <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schedule_interval</span><span class="p">,</span> <span class="nb">str</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">schedule_interval</span> <span class="ow">in</span> <span class="n">cron_presets</span><span class="p">:</span> |
| <span class="n">_schedule_interval</span><span class="p">:</span> <span class="n">ScheduleInterval</span> <span class="o">=</span> <span class="n">cron_presets</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schedule_interval</span><span class="p">)</span> |
| <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">schedule_interval</span> <span class="o">==</span> <span class="s1">'@once'</span><span class="p">:</span> |
| <span class="n">_schedule_interval</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">_schedule_interval</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">schedule_interval</span> |
| <span class="k">return</span> <span class="n">_schedule_interval</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.handle_callback"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.handle_callback">[docs]</a> <span class="k">def</span> <span class="nf">handle_callback</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dagrun</span><span class="p">,</span> <span class="n">success</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">reason</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Triggers the appropriate callback depending on the value of success, namely the</span> |
| <span class="sd"> on_failure_callback or on_success_callback. This method gets the context of a</span> |
| <span class="sd"> single TaskInstance part of this DagRun and passes that to the callable along</span> |
| <span class="sd"> with a 'reason', primarily to differentiate DagRun failures.</span> |
| |
| <span class="sd"> .. note: The logs end up in</span> |
| <span class="sd"> ``$AIRFLOW_HOME/logs/scheduler/latest/PROJECT/DAG_FILE.py.log``</span> |
| |
| <span class="sd"> :param dagrun: DagRun object</span> |
| <span class="sd"> :param success: Flag to specify if failure or success callback should be called</span> |
| <span class="sd"> :param reason: Completion reason</span> |
| <span class="sd"> :param session: Database session</span> |
| <span class="sd"> """</span> |
| <span class="n">callback</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">on_success_callback</span> <span class="k">if</span> <span class="n">success</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">on_failure_callback</span> |
| <span class="k">if</span> <span class="n">callback</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing dag callback function: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">callback</span><span class="p">)</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">dagrun</span><span class="o">.</span><span class="n">get_task_instances</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span> |
| <span class="n">ti</span> <span class="o">=</span> <span class="n">tis</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="c1"># get first TaskInstance of DagRun</span> |
| <span class="n">ti</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span> |
| <span class="n">context</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">get_template_context</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span> |
| <span class="n">context</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">'reason'</span><span class="p">:</span> <span class="n">reason</span><span class="p">})</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="n">callback</span><span class="p">(</span><span class="n">context</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">"failed to invoke dag state update callback"</span><span class="p">)</span> |
| <span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span><span class="s2">"dag.callback_exceptions"</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="DAG.get_active_runs"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_active_runs">[docs]</a> <span class="k">def</span> <span class="nf">get_active_runs</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Returns a list of dag run execution dates currently running</span> |
| |
| <span class="sd"> :return: List of execution dates</span> |
| <span class="sd"> """</span> |
| <span class="n">runs</span> <span class="o">=</span> <span class="n">DagRun</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">state</span><span class="o">=</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">)</span> |
| |
| <span class="n">active_dates</span> <span class="o">=</span> <span class="p">[]</span> |
| <span class="k">for</span> <span class="n">run</span> <span class="ow">in</span> <span class="n">runs</span><span class="p">:</span> |
| <span class="n">active_dates</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">run</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span> |
| |
| <span class="k">return</span> <span class="n">active_dates</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.get_num_active_runs"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_num_active_runs">[docs]</a> <span class="k">def</span> <span class="nf">get_num_active_runs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">external_trigger</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">only_running</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Returns the number of active "running" dag runs</span> |
| |
| <span class="sd"> :param external_trigger: True for externally triggered active dag runs</span> |
| <span class="sd"> :param session:</span> |
| <span class="sd"> :return: number greater than 0 for active dag runs</span> |
| <span class="sd"> """</span> |
| <span class="c1"># .count() is inefficient</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">func</span><span class="o">.</span><span class="n">count</span><span class="p">())</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">only_running</span><span class="p">:</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">in_</span><span class="p">({</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">QUEUED</span><span class="p">}))</span> |
| |
| <span class="k">if</span> <span class="n">external_trigger</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span> |
| <span class="n">DagRun</span><span class="o">.</span><span class="n">external_trigger</span> <span class="o">==</span> <span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">true</span><span class="p">()</span> <span class="k">if</span> <span class="n">external_trigger</span> <span class="k">else</span> <span class="n">expression</span><span class="o">.</span><span class="n">false</span><span class="p">())</span> |
| <span class="p">)</span> |
| |
| <span class="k">return</span> <span class="n">query</span><span class="o">.</span><span class="n">scalar</span><span class="p">()</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.get_dagrun"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_dagrun">[docs]</a> <span class="k">def</span> <span class="nf">get_dagrun</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">execution_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">run_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">session</span><span class="p">:</span> <span class="n">Session</span> <span class="o">=</span> <span class="n">NEW_SESSION</span><span class="p">,</span> |
| <span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Returns the dag run for a given execution date or run_id if it exists, otherwise</span> |
| <span class="sd"> none.</span> |
| |
| <span class="sd"> :param execution_date: The execution date of the DagRun to find.</span> |
| <span class="sd"> :param run_id: The run_id of the DagRun to find.</span> |
| <span class="sd"> :param session:</span> |
| <span class="sd"> :return: The DagRun if found, otherwise None.</span> |
| <span class="sd"> """</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">execution_date</span> <span class="ow">or</span> <span class="n">run_id</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"You must provide either the execution_date or the run_id"</span><span class="p">)</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">execution_date</span><span class="p">:</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">execution_date</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">run_id</span><span class="p">:</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">DagRun</span><span class="o">.</span><span class="n">run_id</span> <span class="o">==</span> <span class="n">run_id</span><span class="p">)</span> |
| <span class="k">return</span> <span class="n">query</span><span class="o">.</span><span class="n">first</span><span class="p">()</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.get_dagruns_between"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_dagruns_between">[docs]</a> <span class="k">def</span> <span class="nf">get_dagruns_between</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">start_date</span><span class="p">,</span> <span class="n">end_date</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Returns the list of dag runs between start_date (inclusive) and end_date (inclusive).</span> |
| |
| <span class="sd"> :param start_date: The starting execution date of the DagRun to find.</span> |
| <span class="sd"> :param end_date: The ending execution date of the DagRun to find.</span> |
| <span class="sd"> :param session:</span> |
| <span class="sd"> :return: The list of DagRuns found.</span> |
| <span class="sd"> """</span> |
| <span class="n">dagruns</span> <span class="o">=</span> <span class="p">(</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">filter</span><span class="p">(</span> |
| <span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> |
| <span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">>=</span> <span class="n">start_date</span><span class="p">,</span> |
| <span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o"><=</span> <span class="n">end_date</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="o">.</span><span class="n">all</span><span class="p">()</span> |
| <span class="p">)</span> |
| |
| <span class="k">return</span> <span class="n">dagruns</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.get_latest_execution_date"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_latest_execution_date">[docs]</a> <span class="k">def</span> <span class="nf">get_latest_execution_date</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">:</span> <span class="n">Session</span> <span class="o">=</span> <span class="n">NEW_SESSION</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">pendulum</span><span class="o">.</span><span class="n">DateTime</span><span class="p">]:</span> |
| <span class="sd">"""Returns the latest date for which at least one dag run exists"""</span> |
| <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">func</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span><span class="p">))</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">scalar</span><span class="p">()</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.latest_execution_date"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.latest_execution_date">[docs]</a> <span class="k">def</span> <span class="nf">latest_execution_date</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="sd">"""This attribute is deprecated. Please use `airflow.models.DAG.get_latest_execution_date`."""</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"This attribute is deprecated. Please use `airflow.models.DAG.get_latest_execution_date`."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_latest_execution_date</span><span class="p">()</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.subdags"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.subdags">[docs]</a> <span class="k">def</span> <span class="nf">subdags</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="sd">"""Returns a list of the subdag objects associated to this DAG"""</span> |
| <span class="c1"># Check SubDag for class but don't check class directly</span> |
| <span class="kn">from</span> <span class="nn">airflow.operators.subdag</span> <span class="kn">import</span> <span class="n">SubDagOperator</span> |
| |
| <span class="n">subdag_lst</span> <span class="o">=</span> <span class="p">[]</span> |
| <span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">:</span> |
| <span class="k">if</span> <span class="p">(</span> |
| <span class="nb">isinstance</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="n">SubDagOperator</span><span class="p">)</span> |
| <span class="ow">or</span> |
| <span class="c1"># TODO remove in Airflow 2.0</span> |
| <span class="nb">type</span><span class="p">(</span><span class="n">task</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'SubDagOperator'</span> |
| <span class="ow">or</span> <span class="n">task</span><span class="o">.</span><span class="n">task_type</span> <span class="o">==</span> <span class="s1">'SubDagOperator'</span> |
| <span class="p">):</span> |
| <span class="n">subdag_lst</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">subdag</span><span class="p">)</span> |
| <span class="n">subdag_lst</span> <span class="o">+=</span> <span class="n">task</span><span class="o">.</span><span class="n">subdag</span><span class="o">.</span><span class="n">subdags</span> |
| <span class="k">return</span> <span class="n">subdag_lst</span></div> |
| |
| <div class="viewcode-block" id="DAG.resolve_template_files"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.resolve_template_files">[docs]</a> <span class="k">def</span> <span class="nf">resolve_template_files</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">:</span> |
| <span class="n">t</span><span class="o">.</span><span class="n">resolve_template_files</span><span class="p">()</span></div> |
| |
| <div class="viewcode-block" id="DAG.get_template_env"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_template_env">[docs]</a> <span class="k">def</span> <span class="nf">get_template_env</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">force_sandboxed</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">jinja2</span><span class="o">.</span><span class="n">Environment</span><span class="p">:</span> |
| <span class="sd">"""Build a Jinja2 environment."""</span> |
| <span class="c1"># Collect directories to search for template files</span> |
| <span class="n">searchpath</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">folder</span><span class="p">]</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">template_searchpath</span><span class="p">:</span> |
| <span class="n">searchpath</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">template_searchpath</span> |
| |
| <span class="c1"># Default values (for backward compatibility)</span> |
| <span class="n">jinja_env_options</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="s1">'loader'</span><span class="p">:</span> <span class="n">jinja2</span><span class="o">.</span><span class="n">FileSystemLoader</span><span class="p">(</span><span class="n">searchpath</span><span class="p">),</span> |
| <span class="s1">'undefined'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">template_undefined</span><span class="p">,</span> |
| <span class="s1">'extensions'</span><span class="p">:</span> <span class="p">[</span><span class="s2">"jinja2.ext.do"</span><span class="p">],</span> |
| <span class="s1">'cache_size'</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> |
| <span class="p">}</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">jinja_environment_kwargs</span><span class="p">:</span> |
| <span class="n">jinja_env_options</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">jinja_environment_kwargs</span><span class="p">)</span> |
| <span class="n">env</span><span class="p">:</span> <span class="n">jinja2</span><span class="o">.</span><span class="n">Environment</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">render_template_as_native_obj</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">force_sandboxed</span><span class="p">:</span> |
| <span class="n">env</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">templates</span><span class="o">.</span><span class="n">NativeEnvironment</span><span class="p">(</span><span class="o">**</span><span class="n">jinja_env_options</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">env</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">templates</span><span class="o">.</span><span class="n">SandboxedEnvironment</span><span class="p">(</span><span class="o">**</span><span class="n">jinja_env_options</span><span class="p">)</span> |
| |
| <span class="c1"># Add any user defined items. Safe to edit globals as long as no templates are rendered yet.</span> |
| <span class="c1"># http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment.globals</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">user_defined_macros</span><span class="p">:</span> |
| <span class="n">env</span><span class="o">.</span><span class="n">globals</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">user_defined_macros</span><span class="p">)</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">user_defined_filters</span><span class="p">:</span> |
| <span class="n">env</span><span class="o">.</span><span class="n">filters</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">user_defined_filters</span><span class="p">)</span> |
| |
| <span class="k">return</span> <span class="n">env</span></div> |
| |
| <div class="viewcode-block" id="DAG.set_dependency"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.set_dependency">[docs]</a> <span class="k">def</span> <span class="nf">set_dependency</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">upstream_task_id</span><span class="p">,</span> <span class="n">downstream_task_id</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Simple utility method to set dependency between two tasks that</span> |
| <span class="sd"> already have been added to the DAG using add_task()</span> |
| <span class="sd"> """</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">upstream_task_id</span><span class="p">)</span><span class="o">.</span><span class="n">set_downstream</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">downstream_task_id</span><span class="p">))</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.get_task_instances_before"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_task_instances_before">[docs]</a> <span class="k">def</span> <span class="nf">get_task_instances_before</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">base_date</span><span class="p">:</span> <span class="n">datetime</span><span class="p">,</span> |
| <span class="n">num</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> |
| <span class="o">*</span><span class="p">,</span> |
| <span class="n">session</span><span class="p">:</span> <span class="n">Session</span> <span class="o">=</span> <span class="n">NEW_SESSION</span><span class="p">,</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">TaskInstance</span><span class="p">]:</span> |
| <span class="sd">"""Get ``num`` task instances before (including) ``base_date``.</span> |
| |
| <span class="sd"> The returned list may contain exactly ``num`` task instances. It can</span> |
| <span class="sd"> have less if there are less than ``num`` scheduled DAG runs before</span> |
| <span class="sd"> ``base_date``, or more if there are manual task runs between the</span> |
| <span class="sd"> requested period, which does not count toward ``num``.</span> |
| <span class="sd"> """</span> |
| <span class="n">min_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">filter</span><span class="p">(</span> |
| <span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> |
| <span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o"><=</span> <span class="n">base_date</span><span class="p">,</span> |
| <span class="n">DagRun</span><span class="o">.</span><span class="n">run_type</span> <span class="o">!=</span> <span class="n">DagRunType</span><span class="o">.</span><span class="n">MANUAL</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">desc</span><span class="p">())</span> |
| <span class="o">.</span><span class="n">offset</span><span class="p">(</span><span class="n">num</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">limit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">scalar</span><span class="p">()</span> |
| <span class="p">)</span> |
| <span class="k">if</span> <span class="n">min_date</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">min_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utc_epoch</span><span class="p">()</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_task_instances</span><span class="p">(</span><span class="n">start_date</span><span class="o">=</span><span class="n">min_date</span><span class="p">,</span> <span class="n">end_date</span><span class="o">=</span><span class="n">base_date</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.get_task_instances"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_task_instances">[docs]</a> <span class="k">def</span> <span class="nf">get_task_instances</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">state</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">TaskInstanceState</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">session</span><span class="p">:</span> <span class="n">Session</span> <span class="o">=</span> <span class="n">NEW_SESSION</span><span class="p">,</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">TaskInstance</span><span class="p">]:</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">start_date</span><span class="p">:</span> |
| <span class="n">start_date</span> <span class="o">=</span> <span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="mi">30</span><span class="p">))</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span> |
| <span class="n">hour</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">minute</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">second</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">microsecond</span><span class="o">=</span><span class="mi">0</span> |
| <span class="p">)</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_task_instances</span><span class="p">(</span> |
| <span class="n">task_ids</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span> |
| <span class="n">run_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">state</span><span class="o">=</span><span class="n">state</span> <span class="ow">or</span> <span class="p">(),</span> |
| <span class="n">include_subdags</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">include_parentdag</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">include_dependent_dags</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">exclude_task_ids</span><span class="o">=</span><span class="p">(),</span> |
| <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">Query</span><span class="p">,</span> <span class="n">query</span><span class="p">)</span><span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">()</span></div> |
| |
| <span class="nd">@overload</span> |
| <span class="k">def</span> <span class="nf">_get_task_instances</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="o">*</span><span class="p">,</span> |
| <span class="n">task_ids</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Collection</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]],</span> |
| <span class="n">start_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">],</span> |
| <span class="n">end_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">],</span> |
| <span class="n">run_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> |
| <span class="n">state</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">TaskInstanceState</span><span class="p">,</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">TaskInstanceState</span><span class="p">]],</span> |
| <span class="n">include_subdags</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span> |
| <span class="n">include_parentdag</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span> |
| <span class="n">include_dependent_dags</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span> |
| <span class="n">exclude_task_ids</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Collection</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]],</span> |
| <span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">,</span> |
| <span class="n">dag_bag</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="s2">"DagBag"</span><span class="p">]</span> <span class="o">=</span> <span class="o">...</span><span class="p">,</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="n">Iterable</span><span class="p">[</span><span class="n">TaskInstance</span><span class="p">]:</span> |
| <span class="o">...</span> <span class="c1"># pragma: no cover</span> |
| |
| <span class="nd">@overload</span> |
| <span class="k">def</span> <span class="nf">_get_task_instances</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="o">*</span><span class="p">,</span> |
| <span class="n">task_ids</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Collection</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]],</span> |
| <span class="n">as_pk_tuple</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="kc">True</span><span class="p">],</span> |
| <span class="n">start_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">],</span> |
| <span class="n">end_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">],</span> |
| <span class="n">run_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> |
| <span class="n">state</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">TaskInstanceState</span><span class="p">,</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">TaskInstanceState</span><span class="p">]],</span> |
| <span class="n">include_subdags</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span> |
| <span class="n">include_parentdag</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span> |
| <span class="n">include_dependent_dags</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span> |
| <span class="n">exclude_task_ids</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Collection</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]],</span> |
| <span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">,</span> |
| <span class="n">dag_bag</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="s2">"DagBag"</span><span class="p">]</span> <span class="o">=</span> <span class="o">...</span><span class="p">,</span> |
| <span class="n">recursion_depth</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="o">...</span><span class="p">,</span> |
| <span class="n">max_recursion_depth</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="o">...</span><span class="p">,</span> |
| <span class="n">visited_external_tis</span><span class="p">:</span> <span class="n">Set</span><span class="p">[</span><span class="n">TaskInstanceKey</span><span class="p">]</span> <span class="o">=</span> <span class="o">...</span><span class="p">,</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="n">Set</span><span class="p">[</span><span class="s2">"TaskInstanceKey"</span><span class="p">]:</span> |
| <span class="o">...</span> <span class="c1"># pragma: no cover</span> |
| |
| <span class="k">def</span> <span class="nf">_get_task_instances</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="o">*</span><span class="p">,</span> |
| <span class="n">task_ids</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Collection</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]],</span> |
| <span class="n">as_pk_tuple</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="kc">True</span><span class="p">,</span> <span class="kc">None</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">],</span> |
| <span class="n">end_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">],</span> |
| <span class="n">run_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> |
| <span class="n">state</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">TaskInstanceState</span><span class="p">,</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">TaskInstanceState</span><span class="p">]],</span> |
| <span class="n">include_subdags</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span> |
| <span class="n">include_parentdag</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span> |
| <span class="n">include_dependent_dags</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span> |
| <span class="n">exclude_task_ids</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Collection</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]],</span> |
| <span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">,</span> |
| <span class="n">dag_bag</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="s2">"DagBag"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">recursion_depth</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span> |
| <span class="n">max_recursion_depth</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">visited_external_tis</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Set</span><span class="p">[</span><span class="n">TaskInstanceKey</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="n">Union</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="n">TaskInstance</span><span class="p">],</span> <span class="n">Set</span><span class="p">[</span><span class="n">TaskInstanceKey</span><span class="p">]]:</span> |
| <span class="n">TI</span> <span class="o">=</span> <span class="n">TaskInstance</span> |
| |
| <span class="c1"># If we are looking at subdags/dependent dags we want to avoid UNION calls</span> |
| <span class="c1"># in SQL (it doesn't play nice with fields that have no equality operator,</span> |
| <span class="c1"># like JSON types), we instead build our result set separately.</span> |
| <span class="c1">#</span> |
| <span class="c1"># This will be empty if we are only looking at one dag, in which case</span> |
| <span class="c1"># we can return the filtered TI query object directly.</span> |
| <span class="n">result</span><span class="p">:</span> <span class="n">Set</span><span class="p">[</span><span class="n">TaskInstanceKey</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span> |
| |
| <span class="c1"># Do we want full objects, or just the primary columns?</span> |
| <span class="k">if</span> <span class="n">as_pk_tuple</span><span class="p">:</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">TI</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">TI</span><span class="o">.</span><span class="n">run_id</span><span class="p">,</span> <span class="n">TI</span><span class="o">.</span><span class="n">map_index</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TaskInstance</span><span class="p">)</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">dag_run</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">include_subdags</span><span class="p">:</span> |
| <span class="c1"># Crafting the right filter for dag_id and task_ids combo</span> |
| <span class="n">conditions</span> <span class="o">=</span> <span class="p">[]</span> |
| <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">subdags</span> <span class="o">+</span> <span class="p">[</span><span class="bp">self</span><span class="p">]:</span> |
| <span class="n">conditions</span><span class="o">.</span><span class="n">append</span><span class="p">(</span> |
| <span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span> <span class="o">&</span> <span class="n">TaskInstance</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">task_ids</span><span class="p">)</span> |
| <span class="p">)</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">or_</span><span class="p">(</span><span class="o">*</span><span class="n">conditions</span><span class="p">))</span> |
| <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">partial</span><span class="p">:</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">TaskInstance</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_ids</span><span class="p">))</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">run_id</span><span class="p">:</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">run_id</span> <span class="o">==</span> <span class="n">run_id</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">start_date</span><span class="p">:</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">>=</span> <span class="n">start_date</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">task_ids</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">ti_selector_condition</span><span class="p">(</span><span class="n">task_ids</span><span class="p">))</span> |
| |
| <span class="c1"># This allows allow_trigger_in_future config to take affect, rather than mandating exec_date <= UTC</span> |
| <span class="k">if</span> <span class="n">end_date</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">allow_future_exec_dates</span><span class="p">:</span> |
| <span class="n">end_date</span> <span class="o">=</span> <span class="n">end_date</span> <span class="ow">or</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o"><=</span> <span class="n">end_date</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">state</span><span class="p">:</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">state</span><span class="p">,</span> <span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="n">TaskInstanceState</span><span class="p">)):</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">state</span><span class="p">)</span> |
| <span class="k">elif</span> <span class="nb">len</span><span class="p">(</span><span class="n">state</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">state</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="c1"># this is required to deal with NULL values</span> |
| <span class="k">if</span> <span class="kc">None</span> <span class="ow">in</span> <span class="n">state</span><span class="p">:</span> |
| <span class="k">if</span> <span class="nb">all</span><span class="p">(</span><span class="n">x</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">state</span><span class="p">):</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">is_</span><span class="p">(</span><span class="kc">None</span><span class="p">))</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">not_none_state</span> <span class="o">=</span> <span class="p">[</span><span class="n">s</span> <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">state</span> <span class="k">if</span> <span class="n">s</span><span class="p">]</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span> |
| <span class="n">or_</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">not_none_state</span><span class="p">),</span> <span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">is_</span><span class="p">(</span><span class="kc">None</span><span class="p">))</span> |
| <span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">state</span><span class="p">))</span> |
| |
| <span class="c1"># Next, get any of them from our parent DAG (if there is one)</span> |
| <span class="k">if</span> <span class="n">include_parentdag</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">parent_dag</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| |
| <span class="k">if</span> <span class="n">visited_external_tis</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">visited_external_tis</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span> |
| |
| <span class="n">p_dag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parent_dag</span><span class="o">.</span><span class="n">partial_subset</span><span class="p">(</span> |
| <span class="n">task_ids_or_regex</span><span class="o">=</span><span class="sa">r</span><span class="s2">"^</span><span class="si">{}</span><span class="s2">$"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)[</span><span class="mi">1</span><span class="p">]),</span> |
| <span class="n">include_upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">include_downstream</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">result</span><span class="o">.</span><span class="n">update</span><span class="p">(</span> |
| <span class="n">p_dag</span><span class="o">.</span><span class="n">_get_task_instances</span><span class="p">(</span> |
| <span class="n">task_ids</span><span class="o">=</span><span class="n">task_ids</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span> |
| <span class="n">run_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">state</span><span class="o">=</span><span class="n">state</span><span class="p">,</span> |
| <span class="n">include_subdags</span><span class="o">=</span><span class="n">include_subdags</span><span class="p">,</span> |
| <span class="n">include_parentdag</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">include_dependent_dags</span><span class="o">=</span><span class="n">include_dependent_dags</span><span class="p">,</span> |
| <span class="n">as_pk_tuple</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> |
| <span class="n">exclude_task_ids</span><span class="o">=</span><span class="n">exclude_task_ids</span><span class="p">,</span> |
| <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span> |
| <span class="n">dag_bag</span><span class="o">=</span><span class="n">dag_bag</span><span class="p">,</span> |
| <span class="n">recursion_depth</span><span class="o">=</span><span class="n">recursion_depth</span><span class="p">,</span> |
| <span class="n">max_recursion_depth</span><span class="o">=</span><span class="n">max_recursion_depth</span><span class="p">,</span> |
| <span class="n">visited_external_tis</span><span class="o">=</span><span class="n">visited_external_tis</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">include_dependent_dags</span><span class="p">:</span> |
| <span class="c1"># Recursively find external tasks indicated by ExternalTaskMarker</span> |
| <span class="kn">from</span> <span class="nn">airflow.sensors.external_task</span> <span class="kn">import</span> <span class="n">ExternalTaskMarker</span> |
| |
| <span class="n">query</span> <span class="o">=</span> <span class="n">tis</span> |
| <span class="k">if</span> <span class="n">as_pk_tuple</span><span class="p">:</span> |
| <span class="n">condition</span> <span class="o">=</span> <span class="n">TI</span><span class="o">.</span><span class="n">filter_for_tis</span><span class="p">(</span><span class="n">TaskInstanceKey</span><span class="p">(</span><span class="o">*</span><span class="n">cols</span><span class="p">)</span> <span class="k">for</span> <span class="n">cols</span> <span class="ow">in</span> <span class="n">tis</span><span class="o">.</span><span class="n">all</span><span class="p">())</span> |
| <span class="k">if</span> <span class="n">condition</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">condition</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">visited_external_tis</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">visited_external_tis</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span> |
| |
| <span class="k">for</span> <span class="n">ti</span> <span class="ow">in</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">operator</span> <span class="o">==</span> <span class="n">ExternalTaskMarker</span><span class="o">.</span><span class="vm">__name__</span><span class="p">):</span> |
| <span class="n">ti_key</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">key</span><span class="o">.</span><span class="n">primary</span> |
| <span class="k">if</span> <span class="n">ti_key</span> <span class="ow">in</span> <span class="n">visited_external_tis</span><span class="p">:</span> |
| <span class="k">continue</span> |
| |
| <span class="n">visited_external_tis</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">ti_key</span><span class="p">)</span> |
| |
| <span class="n">task</span><span class="p">:</span> <span class="n">ExternalTaskMarker</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">ExternalTaskMarker</span><span class="p">,</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)))</span> |
| <span class="n">ti</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">task</span> |
| |
| <span class="k">if</span> <span class="n">max_recursion_depth</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="c1"># Maximum recursion depth allowed is the recursion_depth of the first</span> |
| <span class="c1"># ExternalTaskMarker in the tasks to be visited.</span> |
| <span class="n">max_recursion_depth</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">recursion_depth</span> |
| |
| <span class="k">if</span> <span class="n">recursion_depth</span> <span class="o">+</span> <span class="mi">1</span> <span class="o">></span> <span class="n">max_recursion_depth</span><span class="p">:</span> |
| <span class="c1"># Prevent cycles or accidents.</span> |
| <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span> |
| <span class="sa">f</span><span class="s2">"Maximum recursion depth </span><span class="si">{</span><span class="n">max_recursion_depth</span><span class="si">}</span><span class="s2"> reached for "</span> |
| <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">ExternalTaskMarker</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="si">}</span><span class="s2">. "</span> |
| <span class="sa">f</span><span class="s2">"Attempted to clear too many tasks or there may be a cyclic dependency."</span> |
| <span class="p">)</span> |
| <span class="n">ti</span><span class="o">.</span><span class="n">render_templates</span><span class="p">()</span> |
| <span class="n">external_tis</span> <span class="o">=</span> <span class="p">(</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">dag_run</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">filter</span><span class="p">(</span> |
| <span class="n">TI</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">task</span><span class="o">.</span><span class="n">external_dag_id</span><span class="p">,</span> |
| <span class="n">TI</span><span class="o">.</span><span class="n">task_id</span> <span class="o">==</span> <span class="n">task</span><span class="o">.</span><span class="n">external_task_id</span><span class="p">,</span> |
| <span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">pendulum</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">execution_date</span><span class="p">),</span> |
| <span class="p">)</span> |
| <span class="p">)</span> |
| |
| <span class="k">for</span> <span class="n">tii</span> <span class="ow">in</span> <span class="n">external_tis</span><span class="p">:</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">dag_bag</span><span class="p">:</span> |
| <span class="n">dag_bag</span> <span class="o">=</span> <span class="n">DagBag</span><span class="p">(</span><span class="n">read_dags_from_db</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
| <span class="n">external_dag</span> <span class="o">=</span> <span class="n">dag_bag</span><span class="o">.</span><span class="n">get_dag</span><span class="p">(</span><span class="n">tii</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">external_dag</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Could not find dag </span><span class="si">{</span><span class="n">tii</span><span class="o">.</span><span class="n">dag_id</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <span class="n">downstream</span> <span class="o">=</span> <span class="n">external_dag</span><span class="o">.</span><span class="n">partial_subset</span><span class="p">(</span> |
| <span class="n">task_ids_or_regex</span><span class="o">=</span><span class="p">[</span><span class="n">tii</span><span class="o">.</span><span class="n">task_id</span><span class="p">],</span> |
| <span class="n">include_upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">include_downstream</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">result</span><span class="o">.</span><span class="n">update</span><span class="p">(</span> |
| <span class="n">downstream</span><span class="o">.</span><span class="n">_get_task_instances</span><span class="p">(</span> |
| <span class="n">task_ids</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">run_id</span><span class="o">=</span><span class="n">tii</span><span class="o">.</span><span class="n">run_id</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">state</span><span class="o">=</span><span class="n">state</span><span class="p">,</span> |
| <span class="n">include_subdags</span><span class="o">=</span><span class="n">include_subdags</span><span class="p">,</span> |
| <span class="n">include_dependent_dags</span><span class="o">=</span><span class="n">include_dependent_dags</span><span class="p">,</span> |
| <span class="n">include_parentdag</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">as_pk_tuple</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> |
| <span class="n">exclude_task_ids</span><span class="o">=</span><span class="n">exclude_task_ids</span><span class="p">,</span> |
| <span class="n">dag_bag</span><span class="o">=</span><span class="n">dag_bag</span><span class="p">,</span> |
| <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span> |
| <span class="n">recursion_depth</span><span class="o">=</span><span class="n">recursion_depth</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> |
| <span class="n">max_recursion_depth</span><span class="o">=</span><span class="n">max_recursion_depth</span><span class="p">,</span> |
| <span class="n">visited_external_tis</span><span class="o">=</span><span class="n">visited_external_tis</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">result</span> <span class="ow">or</span> <span class="n">as_pk_tuple</span><span class="p">:</span> |
| <span class="c1"># Only execute the `ti` query if we have also collected some other results (i.e. subdags etc.)</span> |
| <span class="k">if</span> <span class="n">as_pk_tuple</span><span class="p">:</span> |
| <span class="n">result</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">TaskInstanceKey</span><span class="p">(</span><span class="o">*</span><span class="n">cols</span><span class="p">)</span> <span class="k">for</span> <span class="n">cols</span> <span class="ow">in</span> <span class="n">tis</span><span class="o">.</span><span class="n">all</span><span class="p">())</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">result</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">key</span> <span class="k">for</span> <span class="n">ti</span> <span class="ow">in</span> <span class="n">tis</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">exclude_task_ids</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">result</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="n">task</span> |
| <span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">result</span> |
| <span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">exclude_task_ids</span> |
| <span class="ow">and</span> <span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">task</span><span class="o">.</span><span class="n">map_index</span><span class="p">)</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">exclude_task_ids</span> |
| <span class="p">}</span> |
| |
| <span class="k">if</span> <span class="n">as_pk_tuple</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">result</span> |
| <span class="k">if</span> <span class="n">result</span><span class="p">:</span> |
| <span class="c1"># We've been asked for objects, lets combine it all back in to a result set</span> |
| <span class="n">ti_filters</span> <span class="o">=</span> <span class="n">TI</span><span class="o">.</span><span class="n">filter_for_tis</span><span class="p">(</span><span class="n">result</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">ti_filters</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">TI</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">ti_filters</span><span class="p">)</span> |
| <span class="k">elif</span> <span class="n">exclude_task_ids</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">pass</span> <span class="c1"># Disable filter if not set.</span> |
| <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="nb">next</span><span class="p">(</span><span class="nb">iter</span><span class="p">(</span><span class="n">exclude_task_ids</span><span class="p">),</span> <span class="kc">None</span><span class="p">),</span> <span class="nb">str</span><span class="p">):</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">notin_</span><span class="p">(</span><span class="n">exclude_task_ids</span><span class="p">))</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">not_</span><span class="p">(</span><span class="n">tuple_in_condition</span><span class="p">((</span><span class="n">TI</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">TI</span><span class="o">.</span><span class="n">map_index</span><span class="p">),</span> <span class="n">exclude_task_ids</span><span class="p">)))</span> |
| |
| <span class="k">return</span> <span class="n">tis</span> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.set_task_instance_state"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.set_task_instance_state">[docs]</a> <span class="k">def</span> <span class="nf">set_task_instance_state</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="o">*</span><span class="p">,</span> |
| <span class="n">task_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
| <span class="n">map_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Collection</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">execution_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">run_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">state</span><span class="p">:</span> <span class="n">TaskInstanceState</span><span class="p">,</span> |
| <span class="n">upstream</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> |
| <span class="n">downstream</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> |
| <span class="n">future</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> |
| <span class="n">past</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> |
| <span class="n">commit</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">,</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">TaskInstance</span><span class="p">]:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Set the state of a TaskInstance to the given state, and clear its downstream tasks that are</span> |
| <span class="sd"> in failed or upstream_failed state.</span> |
| |
| <span class="sd"> :param task_id: Task ID of the TaskInstance</span> |
| <span class="sd"> :param map_indexes: Only set TaskInstance if its map_index matches.</span> |
| <span class="sd"> If None (default), all mapped TaskInstances of the task are set.</span> |
| <span class="sd"> :param execution_date: Execution date of the TaskInstance</span> |
| <span class="sd"> :param run_id: The run_id of the TaskInstance</span> |
| <span class="sd"> :param state: State to set the TaskInstance to</span> |
| <span class="sd"> :param upstream: Include all upstream tasks of the given task_id</span> |
| <span class="sd"> :param downstream: Include all downstream tasks of the given task_id</span> |
| <span class="sd"> :param future: Include all future TaskInstances of the given task_id</span> |
| <span class="sd"> :param commit: Commit changes</span> |
| <span class="sd"> :param past: Include all past TaskInstances of the given task_id</span> |
| <span class="sd"> """</span> |
| <span class="kn">from</span> <span class="nn">airflow.api.common.mark_tasks</span> <span class="kn">import</span> <span class="n">set_state</span> |
| |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">exactly_one</span><span class="p">(</span><span class="n">execution_date</span><span class="p">,</span> <span class="n">run_id</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Exactly one of execution_date or run_id must be provided"</span><span class="p">)</span> |
| |
| <span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">task_id</span><span class="p">)</span> |
| <span class="n">task</span><span class="o">.</span><span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span> |
| |
| <span class="n">tasks_to_set_state</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Operator</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Operator</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> |
| <span class="k">if</span> <span class="n">map_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">tasks_to_set_state</span> <span class="o">=</span> <span class="p">[</span><span class="n">task</span><span class="p">]</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">tasks_to_set_state</span> <span class="o">=</span> <span class="p">[(</span><span class="n">task</span><span class="p">,</span> <span class="n">map_index</span><span class="p">)</span> <span class="k">for</span> <span class="n">map_index</span> <span class="ow">in</span> <span class="n">map_indexes</span><span class="p">]</span> |
| |
| <span class="n">altered</span> <span class="o">=</span> <span class="n">set_state</span><span class="p">(</span> |
| <span class="n">tasks</span><span class="o">=</span><span class="n">tasks_to_set_state</span><span class="p">,</span> |
| <span class="n">execution_date</span><span class="o">=</span><span class="n">execution_date</span><span class="p">,</span> |
| <span class="n">run_id</span><span class="o">=</span><span class="n">run_id</span><span class="p">,</span> |
| <span class="n">upstream</span><span class="o">=</span><span class="n">upstream</span><span class="p">,</span> |
| <span class="n">downstream</span><span class="o">=</span><span class="n">downstream</span><span class="p">,</span> |
| <span class="n">future</span><span class="o">=</span><span class="n">future</span><span class="p">,</span> |
| <span class="n">past</span><span class="o">=</span><span class="n">past</span><span class="p">,</span> |
| <span class="n">state</span><span class="o">=</span><span class="n">state</span><span class="p">,</span> |
| <span class="n">commit</span><span class="o">=</span><span class="n">commit</span><span class="p">,</span> |
| <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span> |
| <span class="p">)</span> |
| |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">commit</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">altered</span> |
| |
| <span class="c1"># Clear downstream tasks that are in failed/upstream_failed state to resume them.</span> |
| <span class="c1"># Flush the session so that the tasks marked success are reflected in the db.</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span> |
| <span class="n">subdag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">partial_subset</span><span class="p">(</span> |
| <span class="n">task_ids_or_regex</span><span class="o">=</span><span class="p">{</span><span class="n">task_id</span><span class="p">},</span> |
| <span class="n">include_downstream</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> |
| <span class="n">include_upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">execution_date</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">dag_run</span> <span class="o">=</span> <span class="p">(</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">run_id</span> <span class="o">==</span> <span class="n">run_id</span><span class="p">,</span> <span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">one</span><span class="p">()</span> |
| <span class="p">)</span> <span class="c1"># Raises an error if not found</span> |
| <span class="n">resolve_execution_date</span> <span class="o">=</span> <span class="n">dag_run</span><span class="o">.</span><span class="n">execution_date</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">resolve_execution_date</span> <span class="o">=</span> <span class="n">execution_date</span> |
| |
| <span class="n">end_date</span> <span class="o">=</span> <span class="n">resolve_execution_date</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">future</span> <span class="k">else</span> <span class="kc">None</span> |
| <span class="n">start_date</span> <span class="o">=</span> <span class="n">resolve_execution_date</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">past</span> <span class="k">else</span> <span class="kc">None</span> |
| |
| <span class="n">subdag</span><span class="o">.</span><span class="n">clear</span><span class="p">(</span> |
| <span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span> |
| <span class="n">include_subdags</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> |
| <span class="n">include_parentdag</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> |
| <span class="n">only_failed</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> |
| <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span> |
| <span class="c1"># Exclude the task itself from being cleared</span> |
| <span class="n">exclude_task_ids</span><span class="o">=</span><span class="p">{</span><span class="n">task_id</span><span class="p">},</span> |
| <span class="p">)</span> |
| |
| <span class="k">return</span> <span class="n">altered</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.roots"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.roots">[docs]</a> <span class="k">def</span> <span class="nf">roots</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">Operator</span><span class="p">]:</span> |
| <span class="sd">"""Return nodes with no parents. These are first to execute and are called roots or root nodes."""</span> |
| <span class="k">return</span> <span class="p">[</span><span class="n">task</span> <span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">task</span><span class="o">.</span><span class="n">upstream_list</span><span class="p">]</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.leaves"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.leaves">[docs]</a> <span class="k">def</span> <span class="nf">leaves</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">Operator</span><span class="p">]:</span> |
| <span class="sd">"""Return nodes with no children. These are last to execute and are called leaves or leaf nodes."""</span> |
| <span class="k">return</span> <span class="p">[</span><span class="n">task</span> <span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">task</span><span class="o">.</span><span class="n">downstream_list</span><span class="p">]</span></div> |
| |
| <div class="viewcode-block" id="DAG.topological_sort"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.topological_sort">[docs]</a> <span class="k">def</span> <span class="nf">topological_sort</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">include_subdag_tasks</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Sorts tasks in topographical order, such that a task comes after any of its</span> |
| <span class="sd"> upstream dependencies.</span> |
| |
| <span class="sd"> Deprecated in place of ``task_group.topological_sort``</span> |
| <span class="sd"> """</span> |
| <span class="kn">from</span> <span class="nn">airflow.utils.task_group</span> <span class="kn">import</span> <span class="n">TaskGroup</span> |
| |
| <span class="k">def</span> <span class="nf">nested_topo</span><span class="p">(</span><span class="n">group</span><span class="p">):</span> |
| <span class="k">for</span> <span class="n">node</span> <span class="ow">in</span> <span class="n">group</span><span class="o">.</span><span class="n">topological_sort</span><span class="p">(</span><span class="n">_include_subdag_tasks</span><span class="o">=</span><span class="n">include_subdag_tasks</span><span class="p">):</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">node</span><span class="p">,</span> <span class="n">TaskGroup</span><span class="p">):</span> |
| <span class="k">yield from</span> <span class="n">nested_topo</span><span class="p">(</span><span class="n">node</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">yield</span> <span class="n">node</span> |
| |
| <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">nested_topo</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_group</span><span class="p">))</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.set_dag_runs_state"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.set_dag_runs_state">[docs]</a> <span class="k">def</span> <span class="nf">set_dag_runs_state</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">state</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">,</span> |
| <span class="n">session</span><span class="p">:</span> <span class="n">Session</span> <span class="o">=</span> <span class="n">NEW_SESSION</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">dag_ids</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[],</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"This method is deprecated and will be removed in a future version."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">dag_ids</span> <span class="o">=</span> <span class="n">dag_ids</span> <span class="ow">or</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">]</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">dag_ids</span><span class="p">))</span> |
| <span class="k">if</span> <span class="n">start_date</span><span class="p">:</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">>=</span> <span class="n">start_date</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">end_date</span><span class="p">:</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o"><=</span> <span class="n">end_date</span><span class="p">)</span> |
| <span class="n">query</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">DagRun</span><span class="o">.</span><span class="n">state</span><span class="p">:</span> <span class="n">state</span><span class="p">},</span> <span class="n">synchronize_session</span><span class="o">=</span><span class="s1">'fetch'</span><span class="p">)</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.clear"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.clear">[docs]</a> <span class="k">def</span> <span class="nf">clear</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">task_ids</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">Collection</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]],</span> <span class="kc">None</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">only_failed</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> |
| <span class="n">only_running</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> |
| <span class="n">confirm_prompt</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> |
| <span class="n">include_subdags</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <span class="n">include_parentdag</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> |
| <span class="n">dag_run_state</span><span class="p">:</span> <span class="n">DagRunState</span> <span class="o">=</span> <span class="n">DagRunState</span><span class="o">.</span><span class="n">QUEUED</span><span class="p">,</span> |
| <span class="n">dry_run</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> |
| <span class="n">session</span><span class="p">:</span> <span class="n">Session</span> <span class="o">=</span> <span class="n">NEW_SESSION</span><span class="p">,</span> |
| <span class="n">get_tis</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> |
| <span class="n">recursion_depth</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span> |
| <span class="n">max_recursion_depth</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">dag_bag</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="s2">"DagBag"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">exclude_task_ids</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">FrozenSet</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">FrozenSet</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]],</span> <span class="kc">None</span><span class="p">]</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">(),</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">TaskInstance</span><span class="p">]]:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Clears a set of task instances associated with the current dag for</span> |
| <span class="sd"> a specified date range.</span> |
| |
| <span class="sd"> :param task_ids: List of task ids or (``task_id``, ``map_index``) tuples to clear</span> |
| <span class="sd"> :param start_date: The minimum execution_date to clear</span> |
| <span class="sd"> :param end_date: The maximum execution_date to clear</span> |
| <span class="sd"> :param only_failed: Only clear failed tasks</span> |
| <span class="sd"> :param only_running: Only clear running tasks.</span> |
| <span class="sd"> :param confirm_prompt: Ask for confirmation</span> |
| <span class="sd"> :param include_subdags: Clear tasks in subdags and clear external tasks</span> |
| <span class="sd"> indicated by ExternalTaskMarker</span> |
| <span class="sd"> :param include_parentdag: Clear tasks in the parent dag of the subdag.</span> |
| <span class="sd"> :param dag_run_state: state to set DagRun to. If set to False, dagrun state will not</span> |
| <span class="sd"> be changed.</span> |
| <span class="sd"> :param dry_run: Find the tasks to clear but don't clear them.</span> |
| <span class="sd"> :param session: The sqlalchemy session to use</span> |
| <span class="sd"> :param dag_bag: The DagBag used to find the dags subdags (Optional)</span> |
| <span class="sd"> :param exclude_task_ids: A set of ``task_id`` or (``task_id``, ``map_index``)</span> |
| <span class="sd"> tuples that should not be cleared</span> |
| <span class="sd"> """</span> |
| <span class="k">if</span> <span class="n">get_tis</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"Passing `get_tis` to dag.clear() is deprecated. Use `dry_run` parameter instead."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">dry_run</span> <span class="o">=</span> <span class="kc">True</span> |
| |
| <span class="k">if</span> <span class="n">recursion_depth</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"Passing `recursion_depth` to dag.clear() is deprecated."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">if</span> <span class="n">max_recursion_depth</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"Passing `max_recursion_depth` to dag.clear() is deprecated."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| |
| <span class="n">state</span> <span class="o">=</span> <span class="p">[]</span> |
| <span class="k">if</span> <span class="n">only_failed</span><span class="p">:</span> |
| <span class="n">state</span> <span class="o">+=</span> <span class="p">[</span><span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">UPSTREAM_FAILED</span><span class="p">]</span> |
| <span class="k">if</span> <span class="n">only_running</span><span class="p">:</span> |
| <span class="c1"># Yes, having `+=` doesn't make sense, but this was the existing behaviour</span> |
| <span class="n">state</span> <span class="o">+=</span> <span class="p">[</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">]</span> |
| |
| <span class="n">tis</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_task_instances</span><span class="p">(</span> |
| <span class="n">task_ids</span><span class="o">=</span><span class="n">task_ids</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span> |
| <span class="n">run_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">state</span><span class="o">=</span><span class="n">state</span><span class="p">,</span> |
| <span class="n">include_subdags</span><span class="o">=</span><span class="n">include_subdags</span><span class="p">,</span> |
| <span class="n">include_parentdag</span><span class="o">=</span><span class="n">include_parentdag</span><span class="p">,</span> |
| <span class="n">include_dependent_dags</span><span class="o">=</span><span class="n">include_subdags</span><span class="p">,</span> <span class="c1"># compat, yes this is not a typo</span> |
| <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span> |
| <span class="n">dag_bag</span><span class="o">=</span><span class="n">dag_bag</span><span class="p">,</span> |
| <span class="n">exclude_task_ids</span><span class="o">=</span><span class="n">exclude_task_ids</span><span class="p">,</span> |
| <span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">dry_run</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">tis</span> |
| |
| <span class="n">tis</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">tis</span><span class="p">)</span> |
| |
| <span class="n">count</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">tis</span><span class="p">)</span> |
| <span class="n">do_it</span> <span class="o">=</span> <span class="kc">True</span> |
| <span class="k">if</span> <span class="n">count</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> |
| <span class="k">return</span> <span class="mi">0</span> |
| <span class="k">if</span> <span class="n">confirm_prompt</span><span class="p">:</span> |
| <span class="n">ti_list</span> <span class="o">=</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">tis</span><span class="p">)</span> |
| <span class="n">question</span> <span class="o">=</span> <span class="p">(</span> |
| <span class="s2">"You are about to delete these </span><span class="si">{count}</span><span class="s2"> tasks:</span><span class="se">\n</span><span class="si">{ti_list}</span><span class="se">\n\n</span><span class="s2">Are you sure? [y/n]"</span> |
| <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">count</span><span class="o">=</span><span class="n">count</span><span class="p">,</span> <span class="n">ti_list</span><span class="o">=</span><span class="n">ti_list</span><span class="p">)</span> |
| <span class="n">do_it</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">helpers</span><span class="o">.</span><span class="n">ask_yesno</span><span class="p">(</span><span class="n">question</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">do_it</span><span class="p">:</span> |
| <span class="n">clear_task_instances</span><span class="p">(</span> |
| <span class="n">tis</span><span class="p">,</span> |
| <span class="n">session</span><span class="p">,</span> |
| <span class="n">dag</span><span class="o">=</span><span class="bp">self</span><span class="p">,</span> |
| <span class="n">dag_run_state</span><span class="o">=</span><span class="n">dag_run_state</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">count</span> <span class="o">=</span> <span class="mi">0</span> |
| <span class="nb">print</span><span class="p">(</span><span class="s2">"Cancelled, nothing was cleared."</span><span class="p">)</span> |
| |
| <span class="n">session</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span> |
| <span class="k">return</span> <span class="n">count</span></div> |
| |
| <span class="nd">@classmethod</span> |
| <div class="viewcode-block" id="DAG.clear_dags"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.clear_dags">[docs]</a> <span class="k">def</span> <span class="nf">clear_dags</span><span class="p">(</span> |
| <span class="bp">cls</span><span class="p">,</span> |
| <span class="n">dags</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">only_failed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">only_running</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">confirm_prompt</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">include_subdags</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> |
| <span class="n">include_parentdag</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">dag_run_state</span><span class="o">=</span><span class="n">DagRunState</span><span class="o">.</span><span class="n">QUEUED</span><span class="p">,</span> |
| <span class="n">dry_run</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="p">):</span> |
| <span class="n">all_tis</span> <span class="o">=</span> <span class="p">[]</span> |
| <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">dags</span><span class="p">:</span> |
| <span class="n">tis</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">clear</span><span class="p">(</span> |
| <span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span> |
| <span class="n">only_failed</span><span class="o">=</span><span class="n">only_failed</span><span class="p">,</span> |
| <span class="n">only_running</span><span class="o">=</span><span class="n">only_running</span><span class="p">,</span> |
| <span class="n">confirm_prompt</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">include_subdags</span><span class="o">=</span><span class="n">include_subdags</span><span class="p">,</span> |
| <span class="n">include_parentdag</span><span class="o">=</span><span class="n">include_parentdag</span><span class="p">,</span> |
| <span class="n">dag_run_state</span><span class="o">=</span><span class="n">dag_run_state</span><span class="p">,</span> |
| <span class="n">dry_run</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">all_tis</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">tis</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">dry_run</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">all_tis</span> |
| |
| <span class="n">count</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">all_tis</span><span class="p">)</span> |
| <span class="n">do_it</span> <span class="o">=</span> <span class="kc">True</span> |
| <span class="k">if</span> <span class="n">count</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> |
| <span class="nb">print</span><span class="p">(</span><span class="s2">"Nothing to clear."</span><span class="p">)</span> |
| <span class="k">return</span> <span class="mi">0</span> |
| <span class="k">if</span> <span class="n">confirm_prompt</span><span class="p">:</span> |
| <span class="n">ti_list</span> <span class="o">=</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">all_tis</span><span class="p">)</span> |
| <span class="n">question</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"You are about to delete these </span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2"> tasks:</span><span class="se">\n</span><span class="si">{</span><span class="n">ti_list</span><span class="si">}</span><span class="se">\n\n</span><span class="s2">Are you sure? [y/n]"</span> |
| <span class="n">do_it</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">helpers</span><span class="o">.</span><span class="n">ask_yesno</span><span class="p">(</span><span class="n">question</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">do_it</span><span class="p">:</span> |
| <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">dags</span><span class="p">:</span> |
| <span class="n">dag</span><span class="o">.</span><span class="n">clear</span><span class="p">(</span> |
| <span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span> |
| <span class="n">only_failed</span><span class="o">=</span><span class="n">only_failed</span><span class="p">,</span> |
| <span class="n">only_running</span><span class="o">=</span><span class="n">only_running</span><span class="p">,</span> |
| <span class="n">confirm_prompt</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">include_subdags</span><span class="o">=</span><span class="n">include_subdags</span><span class="p">,</span> |
| <span class="n">dag_run_state</span><span class="o">=</span><span class="n">dag_run_state</span><span class="p">,</span> |
| <span class="n">dry_run</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">count</span> <span class="o">=</span> <span class="mi">0</span> |
| <span class="nb">print</span><span class="p">(</span><span class="s2">"Cancelled, nothing was cleared."</span><span class="p">)</span> |
| <span class="k">return</span> <span class="n">count</span></div> |
| |
| <div class="viewcode-block" id="DAG.__deepcopy__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.__deepcopy__">[docs]</a> <span class="k">def</span> <span class="nf">__deepcopy__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">memo</span><span class="p">):</span> |
| <span class="c1"># Switcharoo to go around deepcopying objects coming through the</span> |
| <span class="c1"># backdoor</span> |
| <span class="bp">cls</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span> |
| <span class="n">result</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span> |
| <span class="n">memo</span><span class="p">[</span><span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="p">)]</span> <span class="o">=</span> <span class="n">result</span> |
| <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> |
| <span class="k">if</span> <span class="n">k</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'user_defined_macros'</span><span class="p">,</span> <span class="s1">'user_defined_filters'</span><span class="p">,</span> <span class="s1">'_log'</span><span class="p">):</span> |
| <span class="nb">setattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">k</span><span class="p">,</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">memo</span><span class="p">))</span> |
| |
| <span class="n">result</span><span class="o">.</span><span class="n">user_defined_macros</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">user_defined_macros</span> |
| <span class="n">result</span><span class="o">.</span><span class="n">user_defined_filters</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">user_defined_filters</span> |
| <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">'_log'</span><span class="p">):</span> |
| <span class="n">result</span><span class="o">.</span><span class="n">_log</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_log</span> |
| <span class="k">return</span> <span class="n">result</span></div> |
| |
| <div class="viewcode-block" id="DAG.sub_dag"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.sub_dag">[docs]</a> <span class="k">def</span> <span class="nf">sub_dag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> |
| <span class="sd">"""This method is deprecated in favor of partial_subset"""</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"This method is deprecated and will be removed in a future version. Please use partial_subset"</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">partial_subset</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="DAG.partial_subset"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.partial_subset">[docs]</a> <span class="k">def</span> <span class="nf">partial_subset</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">task_ids_or_regex</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">RePatternType</span><span class="p">,</span> <span class="n">Iterable</span><span class="p">[</span><span class="nb">str</span><span class="p">]],</span> |
| <span class="n">include_downstream</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">include_upstream</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> |
| <span class="n">include_direct_upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Returns a subset of the current dag as a deep copy of the current dag</span> |
| <span class="sd"> based on a regex that should match one or many tasks, and includes</span> |
| <span class="sd"> upstream and downstream neighbours based on the flag passed.</span> |
| |
| <span class="sd"> :param task_ids_or_regex: Either a list of task_ids, or a regex to</span> |
| <span class="sd"> match against task ids (as a string, or compiled regex pattern).</span> |
| <span class="sd"> :param include_downstream: Include all downstream tasks of matched</span> |
| <span class="sd"> tasks, in addition to matched tasks.</span> |
| <span class="sd"> :param include_upstream: Include all upstream tasks of matched tasks,</span> |
| <span class="sd"> in addition to matched tasks.</span> |
| <span class="sd"> :param include_direct_upstream: Include all tasks directly upstream of matched</span> |
| <span class="sd"> and downstream (if include_downstream = True) tasks</span> |
| <span class="sd"> """</span> |
| |
| <span class="kn">from</span> <span class="nn">airflow.models.baseoperator</span> <span class="kn">import</span> <span class="n">BaseOperator</span> |
| <span class="kn">from</span> <span class="nn">airflow.models.mappedoperator</span> <span class="kn">import</span> <span class="n">MappedOperator</span> |
| |
| <span class="c1"># deep-copying self.task_dict and self._task_group takes a long time, and we don't want all</span> |
| <span class="c1"># the tasks anyway, so we copy the tasks manually later</span> |
| <span class="n">memo</span> <span class="o">=</span> <span class="p">{</span><span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">):</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_task_group</span><span class="p">):</span> <span class="kc">None</span><span class="p">}</span> |
| <span class="n">dag</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">memo</span><span class="p">)</span> <span class="c1"># type: ignore</span> |
| |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">task_ids_or_regex</span><span class="p">,</span> <span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="n">RePatternType</span><span class="p">)):</span> |
| <span class="n">matched_tasks</span> <span class="o">=</span> <span class="p">[</span><span class="n">t</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span> <span class="k">if</span> <span class="n">re</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">task_ids_or_regex</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">task_id</span><span class="p">)]</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">matched_tasks</span> <span class="o">=</span> <span class="p">[</span><span class="n">t</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span> <span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">task_id</span> <span class="ow">in</span> <span class="n">task_ids_or_regex</span><span class="p">]</span> |
| |
| <span class="n">also_include</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Operator</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span> |
| <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">matched_tasks</span><span class="p">:</span> |
| <span class="k">if</span> <span class="n">include_downstream</span><span class="p">:</span> |
| <span class="n">also_include</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">get_flat_relatives</span><span class="p">(</span><span class="n">upstream</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span> |
| <span class="k">if</span> <span class="n">include_upstream</span><span class="p">:</span> |
| <span class="n">also_include</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">get_flat_relatives</span><span class="p">(</span><span class="n">upstream</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span> |
| |
| <span class="n">direct_upstreams</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Operator</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span> |
| <span class="k">if</span> <span class="n">include_direct_upstream</span><span class="p">:</span> |
| <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">itertools</span><span class="o">.</span><span class="n">chain</span><span class="p">(</span><span class="n">matched_tasks</span><span class="p">,</span> <span class="n">also_include</span><span class="p">):</span> |
| <span class="n">upstream</span> <span class="o">=</span> <span class="p">(</span><span class="n">u</span> <span class="k">for</span> <span class="n">u</span> <span class="ow">in</span> <span class="n">t</span><span class="o">.</span><span class="n">upstream_list</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">u</span><span class="p">,</span> <span class="p">(</span><span class="n">BaseOperator</span><span class="p">,</span> <span class="n">MappedOperator</span><span class="p">)))</span> |
| <span class="n">direct_upstreams</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">upstream</span><span class="p">)</span> |
| |
| <span class="c1"># Compiling the unique list of tasks that made the cut</span> |
| <span class="c1"># Make sure to not recursively deepcopy the dag or task_group while copying the task.</span> |
| <span class="c1"># task_group is reset later</span> |
| <span class="k">def</span> <span class="nf">_deepcopy_task</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"Operator"</span><span class="p">:</span> |
| <span class="n">memo</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="nb">id</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">task_group</span><span class="p">),</span> <span class="kc">None</span><span class="p">)</span> |
| <span class="k">return</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">memo</span><span class="p">)</span> |
| |
| <span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span> <span class="o">=</span> <span class="p">{</span> |
| <span class="n">t</span><span class="o">.</span><span class="n">task_id</span><span class="p">:</span> <span class="n">_deepcopy_task</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> |
| <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">itertools</span><span class="o">.</span><span class="n">chain</span><span class="p">(</span><span class="n">matched_tasks</span><span class="p">,</span> <span class="n">also_include</span><span class="p">,</span> <span class="n">direct_upstreams</span><span class="p">)</span> |
| <span class="p">}</span> |
| |
| <span class="k">def</span> <span class="nf">filter_task_group</span><span class="p">(</span><span class="n">group</span><span class="p">,</span> <span class="n">parent_group</span><span class="p">):</span> |
| <span class="sd">"""Exclude tasks not included in the subdag from the given TaskGroup."""</span> |
| <span class="n">copied</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">group</span><span class="p">)</span> |
| <span class="n">copied</span><span class="o">.</span><span class="n">used_group_ids</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">copied</span><span class="o">.</span><span class="n">used_group_ids</span><span class="p">)</span> |
| <span class="n">copied</span><span class="o">.</span><span class="n">_parent_group</span> <span class="o">=</span> <span class="n">parent_group</span> |
| |
| <span class="n">copied</span><span class="o">.</span><span class="n">children</span> <span class="o">=</span> <span class="p">{}</span> |
| |
| <span class="k">for</span> <span class="n">child</span> <span class="ow">in</span> <span class="n">group</span><span class="o">.</span><span class="n">children</span><span class="o">.</span><span class="n">values</span><span class="p">():</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">child</span><span class="p">,</span> <span class="n">AbstractOperator</span><span class="p">):</span> |
| <span class="k">if</span> <span class="n">child</span><span class="o">.</span><span class="n">task_id</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">:</span> |
| <span class="n">task</span> <span class="o">=</span> <span class="n">copied</span><span class="o">.</span><span class="n">children</span><span class="p">[</span><span class="n">child</span><span class="o">.</span><span class="n">task_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">[</span><span class="n">child</span><span class="o">.</span><span class="n">task_id</span><span class="p">]</span> |
| <span class="n">task</span><span class="o">.</span><span class="n">task_group</span> <span class="o">=</span> <span class="n">weakref</span><span class="o">.</span><span class="n">proxy</span><span class="p">(</span><span class="n">copied</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">copied</span><span class="o">.</span><span class="n">used_group_ids</span><span class="o">.</span><span class="n">discard</span><span class="p">(</span><span class="n">child</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">filtered_child</span> <span class="o">=</span> <span class="n">filter_task_group</span><span class="p">(</span><span class="n">child</span><span class="p">,</span> <span class="n">copied</span><span class="p">)</span> |
| |
| <span class="c1"># Only include this child TaskGroup if it is non-empty.</span> |
| <span class="k">if</span> <span class="n">filtered_child</span><span class="o">.</span><span class="n">children</span><span class="p">:</span> |
| <span class="n">copied</span><span class="o">.</span><span class="n">children</span><span class="p">[</span><span class="n">child</span><span class="o">.</span><span class="n">group_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">filtered_child</span> |
| |
| <span class="k">return</span> <span class="n">copied</span> |
| |
| <span class="n">dag</span><span class="o">.</span><span class="n">_task_group</span> <span class="o">=</span> <span class="n">filter_task_group</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_task_group</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> |
| |
| <span class="c1"># Removing upstream/downstream references to tasks and TaskGroups that did not make</span> |
| <span class="c1"># the cut.</span> |
| <span class="n">subdag_task_groups</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">task_group</span><span class="o">.</span><span class="n">get_task_group_dict</span><span class="p">()</span> |
| <span class="k">for</span> <span class="n">group</span> <span class="ow">in</span> <span class="n">subdag_task_groups</span><span class="o">.</span><span class="n">values</span><span class="p">():</span> |
| <span class="n">group</span><span class="o">.</span><span class="n">upstream_group_ids</span><span class="o">.</span><span class="n">intersection_update</span><span class="p">(</span><span class="n">subdag_task_groups</span><span class="p">)</span> |
| <span class="n">group</span><span class="o">.</span><span class="n">downstream_group_ids</span><span class="o">.</span><span class="n">intersection_update</span><span class="p">(</span><span class="n">subdag_task_groups</span><span class="p">)</span> |
| <span class="n">group</span><span class="o">.</span><span class="n">upstream_task_ids</span><span class="o">.</span><span class="n">intersection_update</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">)</span> |
| <span class="n">group</span><span class="o">.</span><span class="n">downstream_task_ids</span><span class="o">.</span><span class="n">intersection_update</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">)</span> |
| |
| <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">tasks</span><span class="p">:</span> |
| <span class="c1"># Removing upstream/downstream references to tasks that did not</span> |
| <span class="c1"># make the cut</span> |
| <span class="n">t</span><span class="o">.</span><span class="n">upstream_task_ids</span><span class="o">.</span><span class="n">intersection_update</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">)</span> |
| <span class="n">t</span><span class="o">.</span><span class="n">downstream_task_ids</span><span class="o">.</span><span class="n">intersection_update</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">tasks</span><span class="p">)</span> <span class="o"><</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">):</span> |
| <span class="n">dag</span><span class="o">.</span><span class="n">partial</span> <span class="o">=</span> <span class="kc">True</span> |
| |
| <span class="k">return</span> <span class="n">dag</span></div> |
| |
| <div class="viewcode-block" id="DAG.has_task"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.has_task">[docs]</a> <span class="k">def</span> <span class="nf">has_task</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">task_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span></div> |
| |
| <div class="viewcode-block" id="DAG.get_task"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_task">[docs]</a> <span class="k">def</span> <span class="nf">get_task</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">include_subdags</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">Operator</span><span class="p">:</span> |
| <span class="k">if</span> <span class="n">task_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">:</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">[</span><span class="n">task_id</span><span class="p">]</span> |
| <span class="k">if</span> <span class="n">include_subdags</span><span class="p">:</span> |
| <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">subdags</span><span class="p">:</span> |
| <span class="k">if</span> <span class="n">task_id</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">dag</span><span class="o">.</span><span class="n">task_dict</span><span class="p">[</span><span class="n">task_id</span><span class="p">]</span> |
| <span class="k">raise</span> <span class="n">TaskNotFound</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Task </span><span class="si">{</span><span class="n">task_id</span><span class="si">}</span><span class="s2"> not found"</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="DAG.pickle_info"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.pickle_info">[docs]</a> <span class="k">def</span> <span class="nf">pickle_info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="n">d</span> <span class="o">=</span> <span class="p">{}</span> |
| <span class="n">d</span><span class="p">[</span><span class="s1">'is_picklable'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="n">dttm</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> |
| <span class="n">pickled</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| <span class="n">d</span><span class="p">[</span><span class="s1">'pickle_len'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">pickled</span><span class="p">)</span> |
| <span class="n">d</span><span class="p">[</span><span class="s1">'pickling_duration'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">dttm</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> |
| <span class="n">d</span><span class="p">[</span><span class="s1">'is_picklable'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span> |
| <span class="n">d</span><span class="p">[</span><span class="s1">'stacktrace'</span><span class="p">]</span> <span class="o">=</span> <span class="n">traceback</span><span class="o">.</span><span class="n">format_exc</span><span class="p">()</span> |
| <span class="k">return</span> <span class="n">d</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.pickle"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.pickle">[docs]</a> <span class="k">def</span> <span class="nf">pickle</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">)</span> <span class="o">-></span> <span class="n">DagPickle</span><span class="p">:</span> |
| <span class="n">dag</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span> |
| <span class="n">dp</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">if</span> <span class="n">dag</span> <span class="ow">and</span> <span class="n">dag</span><span class="o">.</span><span class="n">pickle_id</span><span class="p">:</span> |
| <span class="n">dp</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagPickle</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagPickle</span><span class="o">.</span><span class="n">id</span> <span class="o">==</span> <span class="n">dag</span><span class="o">.</span><span class="n">pickle_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">dp</span> <span class="ow">or</span> <span class="n">dp</span><span class="o">.</span><span class="n">pickle</span> <span class="o">!=</span> <span class="bp">self</span><span class="p">:</span> |
| <span class="n">dp</span> <span class="o">=</span> <span class="n">DagPickle</span><span class="p">(</span><span class="n">dag</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">dp</span><span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">last_pickled</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">pickle_id</span> <span class="o">=</span> <span class="n">dp</span><span class="o">.</span><span class="n">id</span> |
| |
| <span class="k">return</span> <span class="n">dp</span></div> |
| |
| <div class="viewcode-block" id="DAG.tree_view"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.tree_view">[docs]</a> <span class="k">def</span> <span class="nf">tree_view</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <span class="sd">"""Print an ASCII tree representation of the DAG."""</span> |
| |
| <span class="k">def</span> <span class="nf">get_downstream</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span> |
| <span class="nb">print</span><span class="p">((</span><span class="s2">" "</span> <span class="o">*</span> <span class="n">level</span> <span class="o">*</span> <span class="mi">4</span><span class="p">)</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">task</span><span class="p">))</span> |
| <span class="n">level</span> <span class="o">+=</span> <span class="mi">1</span> |
| <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">task</span><span class="o">.</span><span class="n">downstream_list</span><span class="p">:</span> |
| <span class="n">get_downstream</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">level</span><span class="p">)</span> |
| |
| <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">roots</span><span class="p">:</span> |
| <span class="n">get_downstream</span><span class="p">(</span><span class="n">t</span><span class="p">)</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DAG.task"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.task">[docs]</a> <span class="k">def</span> <span class="nf">task</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"TaskDecoratorCollection"</span><span class="p">:</span> |
| <span class="kn">from</span> <span class="nn">airflow.decorators</span> <span class="kn">import</span> <span class="n">task</span> |
| |
| <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="s2">"TaskDecoratorCollection"</span><span class="p">,</span> <span class="n">functools</span><span class="o">.</span><span class="n">partial</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="bp">self</span><span class="p">))</span></div> |
| |
| <div class="viewcode-block" id="DAG.add_task"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.add_task">[docs]</a> <span class="k">def</span> <span class="nf">add_task</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task</span><span class="p">:</span> <span class="n">Operator</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Add a task to the DAG</span> |
| |
| <span class="sd"> :param task: the task you want to add</span> |
| <span class="sd"> """</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">task</span><span class="o">.</span><span class="n">start_date</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"DAG is missing the start_date parameter"</span><span class="p">)</span> |
| <span class="c1"># if the task has no start date, assign it the same as the DAG</span> |
| <span class="k">elif</span> <span class="ow">not</span> <span class="n">task</span><span class="o">.</span><span class="n">start_date</span><span class="p">:</span> |
| <span class="n">task</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span> |
| <span class="c1"># otherwise, the task will start on the later of its own start date and</span> |
| <span class="c1"># the DAG's start date</span> |
| <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span><span class="p">:</span> |
| <span class="n">task</span><span class="o">.</span><span class="n">start_date</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">start_date</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">start_date</span><span class="p">)</span> |
| |
| <span class="c1"># if the task has no end date, assign it the same as the dag</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">task</span><span class="o">.</span><span class="n">end_date</span><span class="p">:</span> |
| <span class="n">task</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> |
| <span class="c1"># otherwise, the task will end on the earlier of its own end date and</span> |
| <span class="c1"># the DAG's end date</span> |
| <span class="k">elif</span> <span class="n">task</span><span class="o">.</span><span class="n">end_date</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span><span class="p">:</span> |
| <span class="n">task</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">end_date</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">end_date</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="p">(</span> |
| <span class="n">task</span><span class="o">.</span><span class="n">task_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">[</span><span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">task</span> |
| <span class="p">)</span> <span class="ow">or</span> <span class="n">task</span><span class="o">.</span><span class="n">task_id</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_task_group</span><span class="o">.</span><span class="n">used_group_ids</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="n">DuplicateTaskIdFound</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Task id '</span><span class="si">{</span><span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="si">}</span><span class="s2">' has already been added to the DAG"</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">[</span><span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">task</span> |
| <span class="n">task</span><span class="o">.</span><span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span> |
| <span class="c1"># Add task_id to used_group_ids to prevent group_id and task_id collisions.</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">_task_group</span><span class="o">.</span><span class="n">used_group_ids</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span> |
| |
| <span class="bp">self</span><span class="o">.</span><span class="n">task_count</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="DAG.add_tasks"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.add_tasks">[docs]</a> <span class="k">def</span> <span class="nf">add_tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tasks</span><span class="p">:</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Operator</span><span class="p">])</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Add a list of tasks to the DAG</span> |
| |
| <span class="sd"> :param tasks: a lit of tasks you want to add</span> |
| <span class="sd"> """</span> |
| <span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">tasks</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">add_task</span><span class="p">(</span><span class="n">task</span><span class="p">)</span></div> |
| |
| <span class="k">def</span> <span class="nf">_remove_task</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <span class="c1"># This is "private" as removing could leave a hole in dependencies if done incorrectly, and this</span> |
| <span class="c1"># doesn't guard against that</span> |
| <span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="n">task_id</span><span class="p">)</span> |
| <span class="n">tg</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="s1">'task_group'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">tg</span><span class="p">:</span> |
| <span class="n">tg</span><span class="o">.</span><span class="n">_remove</span><span class="p">(</span><span class="n">task</span><span class="p">)</span> |
| |
| <span class="bp">self</span><span class="o">.</span><span class="n">task_count</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="p">)</span> |
| |
| <div class="viewcode-block" id="DAG.run"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.run">[docs]</a> <span class="k">def</span> <span class="nf">run</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">mark_success</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">local</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">executor</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">donot_pickle</span><span class="o">=</span><span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'donot_pickle'</span><span class="p">),</span> |
| <span class="n">ignore_task_deps</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">ignore_first_depends_on_past</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> |
| <span class="n">pool</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">delay_on_limit_secs</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> |
| <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">conf</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
| <span class="n">rerun_failed_tasks</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">run_backwards</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">run_at_least_once</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="n">continue_on_failures</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> |
| <span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Runs the DAG.</span> |
| |
| <span class="sd"> :param start_date: the start date of the range to run</span> |
| <span class="sd"> :param end_date: the end date of the range to run</span> |
| <span class="sd"> :param mark_success: True to mark jobs as succeeded without running them</span> |
| <span class="sd"> :param local: True to run the tasks using the LocalExecutor</span> |
| <span class="sd"> :param executor: The executor instance to run the tasks</span> |
| <span class="sd"> :param donot_pickle: True to avoid pickling DAG object and send to workers</span> |
| <span class="sd"> :param ignore_task_deps: True to skip upstream tasks</span> |
| <span class="sd"> :param ignore_first_depends_on_past: True to ignore depends_on_past</span> |
| <span class="sd"> dependencies for the first set of tasks only</span> |
| <span class="sd"> :param pool: Resource pool to use</span> |
| <span class="sd"> :param delay_on_limit_secs: Time in seconds to wait before next attempt to run</span> |
| <span class="sd"> dag run when max_active_runs limit has been reached</span> |
| <span class="sd"> :param verbose: Make logging output more verbose</span> |
| <span class="sd"> :param conf: user defined dictionary passed from CLI</span> |
| <span class="sd"> :param rerun_failed_tasks:</span> |
| <span class="sd"> :param run_backwards:</span> |
| <span class="sd"> :param run_at_least_once: If true, always run the DAG at least once even</span> |
| <span class="sd"> if no logical run exists within the time range.</span> |
| <span class="sd"> """</span> |
| <span class="kn">from</span> <span class="nn">airflow.jobs.backfill_job</span> <span class="kn">import</span> <span class="n">BackfillJob</span> |
| |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">executor</span> <span class="ow">and</span> <span class="n">local</span><span class="p">:</span> |
| <span class="kn">from</span> <span class="nn">airflow.executors.local_executor</span> <span class="kn">import</span> <span class="n">LocalExecutor</span> |
| |
| <span class="n">executor</span> <span class="o">=</span> <span class="n">LocalExecutor</span><span class="p">()</span> |
| <span class="k">elif</span> <span class="ow">not</span> <span class="n">executor</span><span class="p">:</span> |
| <span class="kn">from</span> <span class="nn">airflow.executors.executor_loader</span> <span class="kn">import</span> <span class="n">ExecutorLoader</span> |
| |
| <span class="n">executor</span> <span class="o">=</span> <span class="n">ExecutorLoader</span><span class="o">.</span><span class="n">get_default_executor</span><span class="p">()</span> |
| <span class="n">job</span> <span class="o">=</span> <span class="n">BackfillJob</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span> |
| <span class="n">end_date</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span> |
| <span class="n">mark_success</span><span class="o">=</span><span class="n">mark_success</span><span class="p">,</span> |
| <span class="n">executor</span><span class="o">=</span><span class="n">executor</span><span class="p">,</span> |
| <span class="n">donot_pickle</span><span class="o">=</span><span class="n">donot_pickle</span><span class="p">,</span> |
| <span class="n">ignore_task_deps</span><span class="o">=</span><span class="n">ignore_task_deps</span><span class="p">,</span> |
| <span class="n">ignore_first_depends_on_past</span><span class="o">=</span><span class="n">ignore_first_depends_on_past</span><span class="p">,</span> |
| <span class="n">pool</span><span class="o">=</span><span class="n">pool</span><span class="p">,</span> |
| <span class="n">delay_on_limit_secs</span><span class="o">=</span><span class="n">delay_on_limit_secs</span><span class="p">,</span> |
| <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span> |
| <span class="n">conf</span><span class="o">=</span><span class="n">conf</span><span class="p">,</span> |
| <span class="n">rerun_failed_tasks</span><span class="o">=</span><span class="n">rerun_failed_tasks</span><span class="p">,</span> |
| <span class="n">run_backwards</span><span class="o">=</span><span class="n">run_backwards</span><span class="p">,</span> |
| <span class="n">run_at_least_once</span><span class="o">=</span><span class="n">run_at_least_once</span><span class="p">,</span> |
| <span class="n">continue_on_failures</span><span class="o">=</span><span class="n">continue_on_failures</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">job</span><span class="o">.</span><span class="n">run</span><span class="p">()</span></div> |
| |
| <div class="viewcode-block" id="DAG.cli"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.cli">[docs]</a> <span class="k">def</span> <span class="nf">cli</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="sd">"""Exposes a CLI specific to this DAG"""</span> |
| <span class="n">check_cycle</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> |
| |
| <span class="kn">from</span> <span class="nn">airflow.cli</span> <span class="kn">import</span> <span class="n">cli_parser</span> |
| |
| <span class="n">parser</span> <span class="o">=</span> <span class="n">cli_parser</span><span class="o">.</span><span class="n">get_parser</span><span class="p">(</span><span class="n">dag_parser</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
| <span class="n">args</span> <span class="o">=</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse_args</span><span class="p">()</span> |
| <span class="n">args</span><span class="o">.</span><span class="n">func</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.create_dagrun"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.create_dagrun">[docs]</a> <span class="k">def</span> <span class="nf">create_dagrun</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">state</span><span class="p">:</span> <span class="n">DagRunState</span><span class="p">,</span> |
| <span class="n">execution_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">run_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">external_trigger</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> |
| <span class="n">conf</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">run_type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DagRunType</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">,</span> |
| <span class="n">dag_hash</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">creating_job_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="n">data_interval</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="n">datetime</span><span class="p">,</span> <span class="n">datetime</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> |
| <span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Creates a dag run from this dag including the tasks associated with this dag.</span> |
| <span class="sd"> Returns the dag run.</span> |
| |
| <span class="sd"> :param run_id: defines the run id for this dag run</span> |
| <span class="sd"> :param run_type: type of DagRun</span> |
| <span class="sd"> :param execution_date: the execution date of this dag run</span> |
| <span class="sd"> :param state: the state of the dag run</span> |
| <span class="sd"> :param start_date: the date this dag run should be evaluated</span> |
| <span class="sd"> :param external_trigger: whether this dag run is externally triggered</span> |
| <span class="sd"> :param conf: Dict containing configuration/parameters to pass to the DAG</span> |
| <span class="sd"> :param creating_job_id: id of the job creating this DagRun</span> |
| <span class="sd"> :param session: database session</span> |
| <span class="sd"> :param dag_hash: Hash of Serialized DAG</span> |
| <span class="sd"> :param data_interval: Data interval of the DagRun</span> |
| <span class="sd"> """</span> |
| <span class="k">if</span> <span class="n">run_id</span><span class="p">:</span> <span class="c1"># Infer run_type from run_id if needed.</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">run_id</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"`run_id` should be a str, not </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">run_id</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">run_type</span><span class="p">:</span> |
| <span class="n">run_type</span> <span class="o">=</span> <span class="n">DagRunType</span><span class="o">.</span><span class="n">from_run_id</span><span class="p">(</span><span class="n">run_id</span><span class="p">)</span> |
| <span class="k">elif</span> <span class="n">run_type</span> <span class="ow">and</span> <span class="n">execution_date</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> <span class="c1"># Generate run_id from run_type and execution_date.</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">run_type</span><span class="p">,</span> <span class="n">DagRunType</span><span class="p">):</span> |
| <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"`run_type` should be a DagRunType, not </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">run_type</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> |
| <span class="n">run_id</span> <span class="o">=</span> <span class="n">DagRun</span><span class="o">.</span><span class="n">generate_run_id</span><span class="p">(</span><span class="n">run_type</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span> |
| <span class="s2">"Creating DagRun needs either `run_id` or both `run_type` and `execution_date`"</span> |
| <span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">run_id</span> <span class="ow">and</span> <span class="s2">"/"</span> <span class="ow">in</span> <span class="n">run_id</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"Using forward slash ('/') in a DAG run ID is deprecated. Note that this character "</span> |
| <span class="s2">"also makes the run impossible to retrieve via Airflow's REST API."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> |
| <span class="p">)</span> |
| |
| <span class="n">logical_date</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">coerce_datetime</span><span class="p">(</span><span class="n">execution_date</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">data_interval</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">logical_date</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"Calling `DAG.create_dagrun()` without an explicit data interval is deprecated"</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">if</span> <span class="n">run_type</span> <span class="o">==</span> <span class="n">DagRunType</span><span class="o">.</span><span class="n">MANUAL</span><span class="p">:</span> |
| <span class="n">data_interval</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="o">.</span><span class="n">infer_manual_data_interval</span><span class="p">(</span><span class="n">run_after</span><span class="o">=</span><span class="n">logical_date</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">data_interval</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">infer_automated_data_interval</span><span class="p">(</span><span class="n">logical_date</span><span class="p">)</span> |
| |
| <span class="c1"># create a copy of params before validating</span> |
| <span class="n">copied_params</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">)</span> |
| <span class="n">copied_params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">conf</span> <span class="ow">or</span> <span class="p">{})</span> |
| <span class="n">copied_params</span><span class="o">.</span><span class="n">validate</span><span class="p">()</span> |
| |
| <span class="n">run</span> <span class="o">=</span> <span class="n">DagRun</span><span class="p">(</span> |
| <span class="n">dag_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> |
| <span class="n">run_id</span><span class="o">=</span><span class="n">run_id</span><span class="p">,</span> |
| <span class="n">execution_date</span><span class="o">=</span><span class="n">logical_date</span><span class="p">,</span> |
| <span class="n">start_date</span><span class="o">=</span><span class="n">start_date</span><span class="p">,</span> |
| <span class="n">external_trigger</span><span class="o">=</span><span class="n">external_trigger</span><span class="p">,</span> |
| <span class="n">conf</span><span class="o">=</span><span class="n">conf</span><span class="p">,</span> |
| <span class="n">state</span><span class="o">=</span><span class="n">state</span><span class="p">,</span> |
| <span class="n">run_type</span><span class="o">=</span><span class="n">run_type</span><span class="p">,</span> |
| <span class="n">dag_hash</span><span class="o">=</span><span class="n">dag_hash</span><span class="p">,</span> |
| <span class="n">creating_job_id</span><span class="o">=</span><span class="n">creating_job_id</span><span class="p">,</span> |
| <span class="n">data_interval</span><span class="o">=</span><span class="n">data_interval</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">run</span><span class="p">)</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span> |
| |
| <span class="n">run</span><span class="o">.</span><span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span> |
| |
| <span class="c1"># create the associated task instances</span> |
| <span class="c1"># state is None at the moment of creation</span> |
| <span class="n">run</span><span class="o">.</span><span class="n">verify_integrity</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span> |
| |
| <span class="k">return</span> <span class="n">run</span></div> |
| |
| <span class="nd">@classmethod</span> |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.bulk_sync_to_db"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.bulk_sync_to_db">[docs]</a> <span class="k">def</span> <span class="nf">bulk_sync_to_db</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">dags</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="s2">"DAG"</span><span class="p">],</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">):</span> |
| <span class="sd">"""This method is deprecated in favor of bulk_write_to_db"""</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"This method is deprecated and will be removed in a future version. Please use bulk_write_to_db"</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">bulk_write_to_db</span><span class="p">(</span><span class="n">dags</span><span class="p">,</span> <span class="n">session</span><span class="p">)</span></div> |
| |
| <span class="nd">@classmethod</span> |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.bulk_write_to_db"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.bulk_write_to_db">[docs]</a> <span class="k">def</span> <span class="nf">bulk_write_to_db</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">dags</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="s2">"DAG"</span><span class="p">],</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Ensure the DagModel rows for the given dags are up-to-date in the dag table in the DB, including</span> |
| <span class="sd"> calculated fields.</span> |
| |
| <span class="sd"> Note that this method can be called for both DAGs and SubDAGs. A SubDag is actually a SubDagOperator.</span> |
| |
| <span class="sd"> :param dags: the DAG objects to save to the DB</span> |
| <span class="sd"> :return: None</span> |
| <span class="sd"> """</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">dags</span><span class="p">:</span> |
| <span class="k">return</span> |
| |
| <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Sync </span><span class="si">%s</span><span class="s2"> DAGs"</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">dags</span><span class="p">))</span> |
| <span class="n">dag_by_ids</span> <span class="o">=</span> <span class="p">{</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">:</span> <span class="n">dag</span> <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">dags</span><span class="p">}</span> |
| <span class="n">dag_ids</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">dag_by_ids</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span> |
| <span class="n">query</span> <span class="o">=</span> <span class="p">(</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">options</span><span class="p">(</span><span class="n">joinedload</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">tags</span><span class="p">,</span> <span class="n">innerjoin</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span> |
| <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">dag_ids</span><span class="p">))</span> |
| <span class="p">)</span> |
| <span class="n">orm_dags</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">DagModel</span><span class="p">]</span> <span class="o">=</span> <span class="n">with_row_locks</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">of</span><span class="o">=</span><span class="n">DagModel</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">()</span> |
| |
| <span class="n">existing_dag_ids</span> <span class="o">=</span> <span class="p">{</span><span class="n">orm_dag</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">for</span> <span class="n">orm_dag</span> <span class="ow">in</span> <span class="n">orm_dags</span><span class="p">}</span> |
| <span class="n">missing_dag_ids</span> <span class="o">=</span> <span class="n">dag_ids</span><span class="o">.</span><span class="n">difference</span><span class="p">(</span><span class="n">existing_dag_ids</span><span class="p">)</span> |
| |
| <span class="k">for</span> <span class="n">missing_dag_id</span> <span class="ow">in</span> <span class="n">missing_dag_ids</span><span class="p">:</span> |
| <span class="n">orm_dag</span> <span class="o">=</span> <span class="n">DagModel</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="n">missing_dag_id</span><span class="p">)</span> |
| <span class="n">dag</span> <span class="o">=</span> <span class="n">dag_by_ids</span><span class="p">[</span><span class="n">missing_dag_id</span><span class="p">]</span> |
| <span class="k">if</span> <span class="n">dag</span><span class="o">.</span><span class="n">is_paused_upon_creation</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">is_paused</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">is_paused_upon_creation</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">tags</span> <span class="o">=</span> <span class="p">[]</span> |
| <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Creating ORM DAG for </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">orm_dag</span><span class="p">)</span> |
| <span class="n">orm_dags</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">orm_dag</span><span class="p">)</span> |
| |
| <span class="c1"># Get the latest dag run for each existing dag as a single query (avoid n+1 query)</span> |
| <span class="n">most_recent_subq</span> <span class="o">=</span> <span class="p">(</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">func</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span><span class="p">)</span><span class="o">.</span><span class="n">label</span><span class="p">(</span><span class="s2">"max_execution_date"</span><span class="p">))</span> |
| <span class="o">.</span><span class="n">filter</span><span class="p">(</span> |
| <span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">existing_dag_ids</span><span class="p">),</span> |
| <span class="n">or_</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">run_type</span> <span class="o">==</span> <span class="n">DagRunType</span><span class="o">.</span><span class="n">BACKFILL_JOB</span><span class="p">,</span> <span class="n">DagRun</span><span class="o">.</span><span class="n">run_type</span> <span class="o">==</span> <span class="n">DagRunType</span><span class="o">.</span><span class="n">SCHEDULED</span><span class="p">),</span> |
| <span class="p">)</span> |
| <span class="o">.</span><span class="n">group_by</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">subquery</span><span class="p">()</span> |
| <span class="p">)</span> |
| <span class="n">most_recent_runs_iter</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span> |
| <span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">most_recent_subq</span><span class="o">.</span><span class="n">c</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> |
| <span class="n">DagRun</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">==</span> <span class="n">most_recent_subq</span><span class="o">.</span><span class="n">c</span><span class="o">.</span><span class="n">max_execution_date</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">most_recent_runs</span> <span class="o">=</span> <span class="p">{</span><span class="n">run</span><span class="o">.</span><span class="n">dag_id</span><span class="p">:</span> <span class="n">run</span> <span class="k">for</span> <span class="n">run</span> <span class="ow">in</span> <span class="n">most_recent_runs_iter</span><span class="p">}</span> |
| |
| <span class="c1"># Get number of active dagruns for all dags we are processing as a single query.</span> |
| |
| <span class="n">num_active_runs</span> <span class="o">=</span> <span class="n">DagRun</span><span class="o">.</span><span class="n">active_runs_of_dags</span><span class="p">(</span><span class="n">dag_ids</span><span class="o">=</span><span class="n">existing_dag_ids</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span> |
| |
| <span class="n">filelocs</span> <span class="o">=</span> <span class="p">[]</span> |
| |
| <span class="k">for</span> <span class="n">orm_dag</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">orm_dags</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span><span class="o">.</span><span class="n">dag_id</span><span class="p">):</span> |
| <span class="n">dag</span> <span class="o">=</span> <span class="n">dag_by_ids</span><span class="p">[</span><span class="n">orm_dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">]</span> |
| <span class="n">filelocs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">fileloc</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">dag</span><span class="o">.</span><span class="n">is_subdag</span><span class="p">:</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="kc">True</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">parent_dag</span><span class="o">.</span><span class="n">fileloc</span> <span class="c1"># type: ignore</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">root_dag_id</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">parent_dag</span><span class="o">.</span><span class="n">dag_id</span> <span class="c1"># type: ignore</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">owners</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">parent_dag</span><span class="o">.</span><span class="n">owner</span> <span class="c1"># type: ignore</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">is_subdag</span> <span class="o">=</span> <span class="kc">False</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">fileloc</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">owners</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">owner</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">is_active</span> <span class="o">=</span> <span class="kc">True</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">has_import_errors</span> <span class="o">=</span> <span class="kc">False</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">last_parsed_time</span> <span class="o">=</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">default_view</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">default_view</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">description</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">description</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">schedule_interval</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">schedule_interval</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">max_active_tasks</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">max_active_tasks</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">max_active_runs</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">max_active_runs</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">has_task_concurrency_limits</span> <span class="o">=</span> <span class="nb">any</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">max_active_tis_per_dag</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">tasks</span><span class="p">)</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">timetable_description</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">timetable</span><span class="o">.</span><span class="n">description</span> |
| |
| <span class="n">run</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DagRun</span><span class="p">]</span> <span class="o">=</span> <span class="n">most_recent_runs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">run</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="n">data_interval</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">data_interval</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">get_run_data_interval</span><span class="p">(</span><span class="n">run</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">num_active_runs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span> <span class="o">>=</span> <span class="n">orm_dag</span><span class="o">.</span><span class="n">max_active_runs</span><span class="p">:</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">next_dagrun_create_after</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">calculate_dagrun_date_fields</span><span class="p">(</span><span class="n">dag</span><span class="p">,</span> <span class="n">data_interval</span><span class="p">)</span> |
| |
| <span class="n">dag_tags</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">tags</span> <span class="ow">or</span> <span class="p">{})</span> |
| <span class="n">orm_dag_tags</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">orm_dag</span><span class="o">.</span><span class="n">tags</span> <span class="ow">or</span> <span class="p">[])</span> |
| <span class="k">for</span> <span class="n">orm_tag</span> <span class="ow">in</span> <span class="n">orm_dag_tags</span><span class="p">:</span> |
| <span class="k">if</span> <span class="n">orm_tag</span><span class="o">.</span><span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">dag_tags</span><span class="p">:</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">orm_tag</span><span class="p">)</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">tags</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">orm_tag</span><span class="p">)</span> |
| <span class="n">orm_tag_names</span> <span class="o">=</span> <span class="p">{</span><span class="n">t</span><span class="o">.</span><span class="n">name</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">orm_dag_tags</span><span class="p">}</span> |
| <span class="k">for</span> <span class="n">dag_tag</span> <span class="ow">in</span> <span class="n">dag_tags</span><span class="p">:</span> |
| <span class="k">if</span> <span class="n">dag_tag</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">orm_tag_names</span><span class="p">:</span> |
| <span class="n">dag_tag_orm</span> <span class="o">=</span> <span class="n">DagTag</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">dag_tag</span><span class="p">,</span> <span class="n">dag_id</span><span class="o">=</span><span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span> |
| <span class="n">orm_dag</span><span class="o">.</span><span class="n">tags</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dag_tag_orm</span><span class="p">)</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">dag_tag_orm</span><span class="p">)</span> |
| |
| <span class="n">DagCode</span><span class="o">.</span><span class="n">bulk_sync_to_db</span><span class="p">(</span><span class="n">filelocs</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span> |
| |
| <span class="c1"># Issue SQL/finish "Unit of Work", but let @provide_session commit (or if passed a session, let caller</span> |
| <span class="c1"># decide when to commit</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span> |
| |
| <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">dags</span><span class="p">:</span> |
| <span class="bp">cls</span><span class="o">.</span><span class="n">bulk_write_to_db</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">subdags</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.sync_to_db"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.sync_to_db">[docs]</a> <span class="k">def</span> <span class="nf">sync_to_db</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Save attributes about this DAG to the DB. Note that this method</span> |
| <span class="sd"> can be called for both DAGs and SubDAGs. A SubDag is actually a</span> |
| <span class="sd"> SubDagOperator.</span> |
| |
| <span class="sd"> :return: None</span> |
| <span class="sd"> """</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">bulk_write_to_db</span><span class="p">([</span><span class="bp">self</span><span class="p">],</span> <span class="n">session</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="DAG.get_default_view"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_default_view">[docs]</a> <span class="k">def</span> <span class="nf">get_default_view</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="sd">"""This is only there for backward compatible jinja2 templates"""</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_view</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'webserver'</span><span class="p">,</span> <span class="s1">'dag_default_view'</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_view</span></div> |
| |
| <span class="nd">@staticmethod</span> |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.deactivate_unknown_dags"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.deactivate_unknown_dags">[docs]</a> <span class="k">def</span> <span class="nf">deactivate_unknown_dags</span><span class="p">(</span><span class="n">active_dag_ids</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Given a list of known DAGs, deactivate any other DAGs that are</span> |
| <span class="sd"> marked as active in the ORM</span> |
| |
| <span class="sd"> :param active_dag_ids: list of DAG IDs that are active</span> |
| <span class="sd"> :return: None</span> |
| <span class="sd"> """</span> |
| <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">active_dag_ids</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> |
| <span class="k">return</span> |
| <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="o">~</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">active_dag_ids</span><span class="p">))</span><span class="o">.</span><span class="n">all</span><span class="p">():</span> |
| <span class="n">dag</span><span class="o">.</span><span class="n">is_active</span> <span class="o">=</span> <span class="kc">False</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">dag</span><span class="p">)</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div> |
| |
| <span class="nd">@staticmethod</span> |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.deactivate_stale_dags"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.deactivate_stale_dags">[docs]</a> <span class="k">def</span> <span class="nf">deactivate_stale_dags</span><span class="p">(</span><span class="n">expiration_date</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Deactivate any DAGs that were last touched by the scheduler before</span> |
| <span class="sd"> the expiration date. These DAGs were likely deleted.</span> |
| |
| <span class="sd"> :param expiration_date: set inactive DAGs that were touched before this</span> |
| <span class="sd"> time</span> |
| <span class="sd"> :return: None</span> |
| <span class="sd"> """</span> |
| <span class="k">for</span> <span class="n">dag</span> <span class="ow">in</span> <span class="p">(</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">last_parsed_time</span> <span class="o"><</span> <span class="n">expiration_date</span><span class="p">,</span> <span class="n">DagModel</span><span class="o">.</span><span class="n">is_active</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">all</span><span class="p">()</span> |
| <span class="p">):</span> |
| <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> |
| <span class="s2">"Deactivating DAG ID </span><span class="si">%s</span><span class="s2"> since it was last touched by the scheduler at </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> |
| <span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> |
| <span class="n">dag</span><span class="o">.</span><span class="n">last_parsed_time</span><span class="o">.</span><span class="n">isoformat</span><span class="p">(),</span> |
| <span class="p">)</span> |
| <span class="n">dag</span><span class="o">.</span><span class="n">is_active</span> <span class="o">=</span> <span class="kc">False</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">dag</span><span class="p">)</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div> |
| |
| <span class="nd">@staticmethod</span> |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DAG.get_num_task_instances"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_num_task_instances">[docs]</a> <span class="k">def</span> <span class="nf">get_num_task_instances</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">task_ids</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">states</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Returns the number of task instances in the given DAG.</span> |
| |
| <span class="sd"> :param session: ORM session</span> |
| <span class="sd"> :param dag_id: ID of the DAG to get the task concurrency of</span> |
| <span class="sd"> :param task_ids: A list of valid task IDs for the given DAG</span> |
| <span class="sd"> :param states: A list of states to filter by if supplied</span> |
| <span class="sd"> :return: The number of running tasks</span> |
| <span class="sd"> :rtype: int</span> |
| <span class="sd"> """</span> |
| <span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">func</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">task_id</span><span class="p">))</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span> |
| <span class="n">TaskInstance</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="k">if</span> <span class="n">task_ids</span><span class="p">:</span> |
| <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span> |
| <span class="n">TaskInstance</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">task_ids</span><span class="p">),</span> |
| <span class="p">)</span> |
| |
| <span class="k">if</span> <span class="n">states</span><span class="p">:</span> |
| <span class="k">if</span> <span class="kc">None</span> <span class="ow">in</span> <span class="n">states</span><span class="p">:</span> |
| <span class="k">if</span> <span class="nb">all</span><span class="p">(</span><span class="n">x</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">states</span><span class="p">):</span> |
| <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">is_</span><span class="p">(</span><span class="kc">None</span><span class="p">))</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">not_none_states</span> <span class="o">=</span> <span class="p">[</span><span class="n">state</span> <span class="k">for</span> <span class="n">state</span> <span class="ow">in</span> <span class="n">states</span> <span class="k">if</span> <span class="n">state</span><span class="p">]</span> |
| <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span> |
| <span class="n">or_</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">not_none_states</span><span class="p">),</span> <span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">is_</span><span class="p">(</span><span class="kc">None</span><span class="p">))</span> |
| <span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TaskInstance</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">states</span><span class="p">))</span> |
| <span class="k">return</span> <span class="n">qry</span><span class="o">.</span><span class="n">scalar</span><span class="p">()</span></div> |
| |
| <span class="nd">@classmethod</span> |
| <div class="viewcode-block" id="DAG.get_serialized_fields"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_serialized_fields">[docs]</a> <span class="k">def</span> <span class="nf">get_serialized_fields</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span> |
| <span class="sd">"""Stringified DAGs and operators contain exactly these fields."""</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="bp">cls</span><span class="o">.</span><span class="n">__serialized_fields</span><span class="p">:</span> |
| <span class="bp">cls</span><span class="o">.</span><span class="n">__serialized_fields</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">(</span><span class="nb">vars</span><span class="p">(</span><span class="n">DAG</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="s1">'test'</span><span class="p">))</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span> <span class="o">-</span> <span class="p">{</span> |
| <span class="s1">'parent_dag'</span><span class="p">,</span> |
| <span class="s1">'_old_context_manager_dags'</span><span class="p">,</span> |
| <span class="s1">'safe_dag_id'</span><span class="p">,</span> |
| <span class="s1">'last_loaded'</span><span class="p">,</span> |
| <span class="s1">'user_defined_filters'</span><span class="p">,</span> |
| <span class="s1">'user_defined_macros'</span><span class="p">,</span> |
| <span class="s1">'partial'</span><span class="p">,</span> |
| <span class="s1">'params'</span><span class="p">,</span> |
| <span class="s1">'_pickle_id'</span><span class="p">,</span> |
| <span class="s1">'_log'</span><span class="p">,</span> |
| <span class="s1">'task_dict'</span><span class="p">,</span> |
| <span class="s1">'template_searchpath'</span><span class="p">,</span> |
| <span class="s1">'sla_miss_callback'</span><span class="p">,</span> |
| <span class="s1">'on_success_callback'</span><span class="p">,</span> |
| <span class="s1">'on_failure_callback'</span><span class="p">,</span> |
| <span class="s1">'template_undefined'</span><span class="p">,</span> |
| <span class="s1">'jinja_environment_kwargs'</span><span class="p">,</span> |
| <span class="c1"># has_on_*_callback are only stored if the value is True, as the default is False</span> |
| <span class="s1">'has_on_success_callback'</span><span class="p">,</span> |
| <span class="s1">'has_on_failure_callback'</span><span class="p">,</span> |
| <span class="p">}</span> |
| <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">__serialized_fields</span></div> |
| |
| <div class="viewcode-block" id="DAG.get_edge_info"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.get_edge_info">[docs]</a> <span class="k">def</span> <span class="nf">get_edge_info</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">upstream_task_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">downstream_task_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">EdgeInfoType</span><span class="p">:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Returns edge information for the given pair of tasks if present, and</span> |
| <span class="sd"> an empty edge if there is no information.</span> |
| <span class="sd"> """</span> |
| <span class="c1"># Note - older serialized DAGs may not have edge_info being a dict at all</span> |
| <span class="n">empty</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">EdgeInfoType</span><span class="p">,</span> <span class="p">{})</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">edge_info</span><span class="p">:</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">edge_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">upstream_task_id</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">downstream_task_id</span><span class="p">,</span> <span class="n">empty</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">empty</span></div> |
| |
| <div class="viewcode-block" id="DAG.set_edge_info"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.set_edge_info">[docs]</a> <span class="k">def</span> <span class="nf">set_edge_info</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">upstream_task_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">downstream_task_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">info</span><span class="p">:</span> <span class="n">EdgeInfoType</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Sets the given edge information on the DAG. Note that this will overwrite,</span> |
| <span class="sd"> rather than merge with, existing info.</span> |
| <span class="sd"> """</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">edge_info</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">upstream_task_id</span><span class="p">,</span> <span class="p">{})[</span><span class="n">downstream_task_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">info</span></div> |
| |
| <div class="viewcode-block" id="DAG.validate_schedule_and_params"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DAG.validate_schedule_and_params">[docs]</a> <span class="k">def</span> <span class="nf">validate_schedule_and_params</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Validates & raise exception if there are any Params in the DAG which neither have a default value nor</span> |
| <span class="sd"> have the null in schema['type'] list, but the DAG have a schedule_interval which is not None.</span> |
| <span class="sd"> """</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">timetable</span><span class="o">.</span><span class="n">can_run</span><span class="p">:</span> |
| <span class="k">return</span> |
| |
| <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> |
| <span class="c1"># As type can be an array, we would check if `null` is an allowed type or not</span> |
| <span class="k">if</span> <span class="ow">not</span> <span class="n">v</span><span class="o">.</span><span class="n">has_value</span> <span class="ow">and</span> <span class="p">(</span><span class="s2">"type"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">v</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="s2">"null"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">v</span><span class="o">.</span><span class="n">schema</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]):</span> |
| <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span> |
| <span class="s2">"DAG Schedule must be None, if there are any required params without default values"</span></div></div> |
| <span class="p">)</span> |
| |
| |
| <div class="viewcode-block" id="DagTag"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagTag">[docs]</a><span class="k">class</span> <span class="nc">DagTag</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span> |
| <span class="sd">"""A tag name per dag, to allow quick filtering in the DAG view."""</span> |
| |
| <div class="viewcode-block" id="DagTag.__tablename__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagTag.__tablename__">[docs]</a> <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">"dag_tag"</span></div> |
| <div class="viewcode-block" id="DagTag.name"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagTag.name">[docs]</a> <span class="n">name</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">100</span><span class="p">),</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div> |
| <div class="viewcode-block" id="DagTag.dag_id"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagTag.dag_id">[docs]</a> <span class="n">dag_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span> |
| <span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> |
| <span class="n">ForeignKey</span><span class="p">(</span><span class="s1">'dag.dag_id'</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="s1">'dag_tag_dag_id_fkey'</span><span class="p">,</span> <span class="n">ondelete</span><span class="o">=</span><span class="s1">'CASCADE'</span><span class="p">),</span> |
| <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span></div> |
| <span class="p">)</span> |
| |
| <div class="viewcode-block" id="DagTag.__repr__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagTag.__repr__">[docs]</a> <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span></div></div> |
| |
| |
| <div class="viewcode-block" id="DagModel"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel">[docs]</a><span class="k">class</span> <span class="nc">DagModel</span><span class="p">(</span><span class="n">Base</span><span class="p">):</span> |
| <span class="sd">"""Table containing DAG properties"""</span> |
| |
| <div class="viewcode-block" id="DagModel.__tablename__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.__tablename__">[docs]</a> <span class="n">__tablename__</span> <span class="o">=</span> <span class="s2">"dag"</span></div> |
| <span class="sd">"""</span> |
| <span class="sd"> These items are stored in the database for state related information</span> |
| <span class="sd"> """</span> |
| <div class="viewcode-block" id="DagModel.dag_id"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.dag_id">[docs]</a> <span class="n">dag_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">),</span> <span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div> |
| <div class="viewcode-block" id="DagModel.root_dag_id"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.root_dag_id">[docs]</a> <span class="n">root_dag_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="n">ID_LEN</span><span class="p">))</span></div> |
| <span class="c1"># A DAG can be paused from the UI / DB</span> |
| <span class="c1"># Set this default value of is_paused based on a configuration value!</span> |
| <div class="viewcode-block" id="DagModel.is_paused_at_creation"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.is_paused_at_creation">[docs]</a> <span class="n">is_paused_at_creation</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'dags_are_paused_at_creation'</span><span class="p">)</span></div> |
| <div class="viewcode-block" id="DagModel.is_paused"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.is_paused">[docs]</a> <span class="n">is_paused</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">is_paused_at_creation</span><span class="p">)</span></div> |
| <span class="c1"># Whether the DAG is a subdag</span> |
| <div class="viewcode-block" id="DagModel.is_subdag"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.is_subdag">[docs]</a> <span class="n">is_subdag</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div> |
| <span class="c1"># Whether that DAG was seen on the last DagBag load</span> |
| <div class="viewcode-block" id="DagModel.is_active"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.is_active">[docs]</a> <span class="n">is_active</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div> |
| <span class="c1"># Last time the scheduler started</span> |
| <div class="viewcode-block" id="DagModel.last_parsed_time"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.last_parsed_time">[docs]</a> <span class="n">last_parsed_time</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div> |
| <span class="c1"># Last time this DAG was pickled</span> |
| <div class="viewcode-block" id="DagModel.last_pickled"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.last_pickled">[docs]</a> <span class="n">last_pickled</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div> |
| <span class="c1"># Time when the DAG last received a refresh signal</span> |
| <span class="c1"># (e.g. the DAG's "refresh" button was clicked in the web UI)</span> |
| <div class="viewcode-block" id="DagModel.last_expired"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.last_expired">[docs]</a> <span class="n">last_expired</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div> |
| <span class="c1"># Whether (one of) the scheduler is scheduling this DAG at the moment</span> |
| <div class="viewcode-block" id="DagModel.scheduler_lock"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.scheduler_lock">[docs]</a> <span class="n">scheduler_lock</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">)</span></div> |
| <span class="c1"># Foreign key to the latest pickle_id</span> |
| <div class="viewcode-block" id="DagModel.pickle_id"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.pickle_id">[docs]</a> <span class="n">pickle_id</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">)</span></div> |
| <span class="c1"># The location of the file containing the DAG object</span> |
| <span class="c1"># Note: Do not depend on fileloc pointing to a file; in the case of a</span> |
| <span class="c1"># packaged DAG, it will point to the subpath of the DAG within the</span> |
| <span class="c1"># associated zip.</span> |
| <div class="viewcode-block" id="DagModel.fileloc"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.fileloc">[docs]</a> <span class="n">fileloc</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">2000</span><span class="p">))</span></div> |
| <span class="c1"># String representing the owners</span> |
| <div class="viewcode-block" id="DagModel.owners"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.owners">[docs]</a> <span class="n">owners</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">2000</span><span class="p">))</span></div> |
| <span class="c1"># Description of the dag</span> |
| <div class="viewcode-block" id="DagModel.description"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.description">[docs]</a> <span class="n">description</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Text</span><span class="p">)</span></div> |
| <span class="c1"># Default view of the DAG inside the webserver</span> |
| <div class="viewcode-block" id="DagModel.default_view"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.default_view">[docs]</a> <span class="n">default_view</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">25</span><span class="p">))</span></div> |
| <span class="c1"># Schedule interval</span> |
| <div class="viewcode-block" id="DagModel.schedule_interval"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.schedule_interval">[docs]</a> <span class="n">schedule_interval</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Interval</span><span class="p">)</span></div> |
| <span class="c1"># Timetable/Schedule Interval description</span> |
| <div class="viewcode-block" id="DagModel.timetable_description"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.timetable_description">[docs]</a> <span class="n">timetable_description</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">(</span><span class="mi">1000</span><span class="p">),</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div> |
| |
| <span class="c1"># Tags for view filter</span> |
| <div class="viewcode-block" id="DagModel.tags"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.tags">[docs]</a> <span class="n">tags</span> <span class="o">=</span> <span class="n">relationship</span><span class="p">(</span><span class="s1">'DagTag'</span><span class="p">,</span> <span class="n">cascade</span><span class="o">=</span><span class="s1">'all, delete, delete-orphan'</span><span class="p">,</span> <span class="n">backref</span><span class="o">=</span><span class="n">backref</span><span class="p">(</span><span class="s1">'dag'</span><span class="p">))</span></div> |
| |
| <div class="viewcode-block" id="DagModel.max_active_tasks"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.max_active_tasks">[docs]</a> <span class="n">max_active_tasks</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div> |
| <div class="viewcode-block" id="DagModel.max_active_runs"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.max_active_runs">[docs]</a> <span class="n">max_active_runs</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="DagModel.has_task_concurrency_limits"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.has_task_concurrency_limits">[docs]</a> <span class="n">has_task_concurrency_limits</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">,</span> <span class="n">nullable</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div> |
| <div class="viewcode-block" id="DagModel.has_import_errors"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.has_import_errors">[docs]</a> <span class="n">has_import_errors</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">Boolean</span><span class="p">(),</span> <span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span></div> |
| |
| <span class="c1"># The logical date of the next dag run.</span> |
| <div class="viewcode-block" id="DagModel.next_dagrun"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.next_dagrun">[docs]</a> <span class="n">next_dagrun</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div> |
| |
| <span class="c1"># Must be either both NULL or both datetime.</span> |
| <div class="viewcode-block" id="DagModel.next_dagrun_data_interval_start"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.next_dagrun_data_interval_start">[docs]</a> <span class="n">next_dagrun_data_interval_start</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div> |
| <div class="viewcode-block" id="DagModel.next_dagrun_data_interval_end"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.next_dagrun_data_interval_end">[docs]</a> <span class="n">next_dagrun_data_interval_end</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div> |
| |
| <span class="c1"># Earliest time at which this ``next_dagrun`` can be created.</span> |
| <div class="viewcode-block" id="DagModel.next_dagrun_create_after"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.next_dagrun_create_after">[docs]</a> <span class="n">next_dagrun_create_after</span> <span class="o">=</span> <span class="n">Column</span><span class="p">(</span><span class="n">UtcDateTime</span><span class="p">)</span></div> |
| |
| <div class="viewcode-block" id="DagModel.__table_args__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.__table_args__">[docs]</a> <span class="n">__table_args__</span> <span class="o">=</span> <span class="p">(</span> |
| <span class="n">Index</span><span class="p">(</span><span class="s1">'idx_root_dag_id'</span><span class="p">,</span> <span class="n">root_dag_id</span><span class="p">,</span> <span class="n">unique</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span> |
| <span class="n">Index</span><span class="p">(</span><span class="s1">'idx_next_dagrun_create_after'</span><span class="p">,</span> <span class="n">next_dagrun_create_after</span><span class="p">,</span> <span class="n">unique</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span></div> |
| <span class="p">)</span> |
| |
| <div class="viewcode-block" id="DagModel.parent_dag"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.parent_dag">[docs]</a> <span class="n">parent_dag</span> <span class="o">=</span> <span class="n">relationship</span><span class="p">(</span> |
| <span class="s2">"DagModel"</span><span class="p">,</span> <span class="n">remote_side</span><span class="o">=</span><span class="p">[</span><span class="n">dag_id</span><span class="p">],</span> <span class="n">primaryjoin</span><span class="o">=</span><span class="n">root_dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">foreign_keys</span><span class="o">=</span><span class="p">[</span><span class="n">root_dag_id</span><span class="p">]</span></div> |
| <span class="p">)</span> |
| |
| <div class="viewcode-block" id="DagModel.NUM_DAGS_PER_DAGRUN_QUERY"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.NUM_DAGS_PER_DAGRUN_QUERY">[docs]</a> <span class="n">NUM_DAGS_PER_DAGRUN_QUERY</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'scheduler'</span><span class="p">,</span> <span class="s1">'max_dagruns_to_create_per_loop'</span><span class="p">,</span> <span class="n">fallback</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span></div> |
| |
| <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">concurrency</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> |
| <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_active_tasks</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">if</span> <span class="n">concurrency</span><span class="p">:</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"The 'DagModel.concurrency' parameter is deprecated. Please use 'max_active_tasks'."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">max_active_tasks</span> <span class="o">=</span> <span class="n">concurrency</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">max_active_tasks</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'max_active_tasks_per_dag'</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_active_runs</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">max_active_runs</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'max_active_runs_per_dag'</span><span class="p">)</span> |
| |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_task_concurrency_limits</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="c1"># Be safe -- this will be updated later once the DAG is parsed</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">has_task_concurrency_limits</span> <span class="o">=</span> <span class="kc">True</span> |
| |
| <div class="viewcode-block" id="DagModel.__repr__"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.__repr__">[docs]</a> <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="k">return</span> <span class="sa">f</span><span class="s2">"<DAG: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="si">}</span><span class="s2">>"</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DagModel.next_dagrun_data_interval"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.next_dagrun_data_interval">[docs]</a> <span class="k">def</span> <span class="nf">next_dagrun_data_interval</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">DataInterval</span><span class="p">]:</span> |
| <span class="k">return</span> <span class="n">_get_model_data_interval</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="s2">"next_dagrun_data_interval_start"</span><span class="p">,</span> |
| <span class="s2">"next_dagrun_data_interval_end"</span><span class="p">,</span></div> |
| <span class="p">)</span> |
| |
| <span class="nd">@next_dagrun_data_interval</span><span class="o">.</span><span class="n">setter</span> |
| <span class="k">def</span> <span class="nf">next_dagrun_data_interval</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="n">datetime</span><span class="p">,</span> <span class="n">datetime</span><span class="p">]])</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun_data_interval_start</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun_data_interval_end</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun_data_interval_start</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun_data_interval_end</span> <span class="o">=</span> <span class="n">value</span> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DagModel.timezone"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.timezone">[docs]</a> <span class="k">def</span> <span class="nf">timezone</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">settings</span><span class="o">.</span><span class="n">TIMEZONE</span></div> |
| |
| <span class="nd">@staticmethod</span> |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DagModel.get_dagmodel"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.get_dagmodel">[docs]</a> <span class="k">def</span> <span class="nf">get_dagmodel</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">options</span><span class="p">(</span><span class="n">joinedload</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">parent_dag</span><span class="p">))</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">dag_id</span><span class="p">)</span></div> |
| |
| <span class="nd">@classmethod</span> |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DagModel.get_current"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.get_current">[docs]</a> <span class="k">def</span> <span class="nf">get_current</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DagModel.get_last_dagrun"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.get_last_dagrun">[docs]</a> <span class="k">def</span> <span class="nf">get_last_dagrun</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">,</span> <span class="n">include_externally_triggered</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> |
| <span class="k">return</span> <span class="n">get_last_dagrun</span><span class="p">(</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span> <span class="n">include_externally_triggered</span><span class="o">=</span><span class="n">include_externally_triggered</span></div> |
| <span class="p">)</span> |
| |
| <div class="viewcode-block" id="DagModel.get_is_paused"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.get_is_paused">[docs]</a> <span class="k">def</span> <span class="nf">get_is_paused</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">session</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Session</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> |
| <span class="sd">"""Provide interface compatibility to 'DAG'."""</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_paused</span></div> |
| |
| <span class="nd">@staticmethod</span> |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DagModel.get_paused_dag_ids"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.get_paused_dag_ids">[docs]</a> <span class="k">def</span> <span class="nf">get_paused_dag_ids</span><span class="p">(</span><span class="n">dag_ids</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">session</span><span class="p">:</span> <span class="n">Session</span> <span class="o">=</span> <span class="n">NEW_SESSION</span><span class="p">)</span> <span class="o">-></span> <span class="n">Set</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Given a list of dag_ids, get a set of Paused Dag Ids</span> |
| |
| <span class="sd"> :param dag_ids: List of Dag ids</span> |
| <span class="sd"> :param session: ORM Session</span> |
| <span class="sd"> :return: Paused Dag_ids</span> |
| <span class="sd"> """</span> |
| <span class="n">paused_dag_ids</span> <span class="o">=</span> <span class="p">(</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">is_paused</span> <span class="o">==</span> <span class="n">expression</span><span class="o">.</span><span class="n">true</span><span class="p">())</span> |
| <span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">dag_ids</span><span class="p">))</span> |
| <span class="o">.</span><span class="n">all</span><span class="p">()</span> |
| <span class="p">)</span> |
| |
| <span class="n">paused_dag_ids</span> <span class="o">=</span> <span class="p">{</span><span class="n">paused_dag_id</span> <span class="k">for</span> <span class="n">paused_dag_id</span><span class="p">,</span> <span class="ow">in</span> <span class="n">paused_dag_ids</span><span class="p">}</span> |
| <span class="k">return</span> <span class="n">paused_dag_ids</span></div> |
| |
| <div class="viewcode-block" id="DagModel.get_default_view"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.get_default_view">[docs]</a> <span class="k">def</span> <span class="nf">get_default_view</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Get the Default DAG View, returns the default config value if DagModel does not</span> |
| <span class="sd"> have a value</span> |
| <span class="sd"> """</span> |
| <span class="c1"># This is for backwards-compatibility with old dags that don't have None as default_view</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_view</span> <span class="ow">or</span> <span class="n">conf</span><span class="o">.</span><span class="n">get_mandatory_value</span><span class="p">(</span><span class="s1">'webserver'</span><span class="p">,</span> <span class="s1">'dag_default_view'</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DagModel.safe_dag_id"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.safe_dag_id">[docs]</a> <span class="k">def</span> <span class="nf">safe_dag_id</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> |
| <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="s1">'__dot__'</span><span class="p">)</span></div> |
| |
| <span class="nd">@property</span> |
| <div class="viewcode-block" id="DagModel.relative_fileloc"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.relative_fileloc">[docs]</a> <span class="k">def</span> <span class="nf">relative_fileloc</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">]:</span> |
| <span class="sd">"""File location of the importable dag 'file' relative to the configured DAGs folder."""</span> |
| <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">fileloc</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">return</span> <span class="kc">None</span> |
| <span class="n">path</span> <span class="o">=</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fileloc</span><span class="p">)</span> |
| <span class="k">try</span><span class="p">:</span> |
| <span class="k">return</span> <span class="n">path</span><span class="o">.</span><span class="n">relative_to</span><span class="p">(</span><span class="n">settings</span><span class="o">.</span><span class="n">DAGS_FOLDER</span><span class="p">)</span> |
| <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> |
| <span class="c1"># Not relative to DAGS_FOLDER.</span> |
| <span class="k">return</span> <span class="n">path</span></div> |
| |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DagModel.set_is_paused"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.set_is_paused">[docs]</a> <span class="k">def</span> <span class="nf">set_is_paused</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">is_paused</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span> <span class="n">including_subdags</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Pause/Un-pause a DAG.</span> |
| |
| <span class="sd"> :param is_paused: Is the DAG paused</span> |
| <span class="sd"> :param including_subdags: whether to include the DAG's subdags</span> |
| <span class="sd"> :param session: session</span> |
| <span class="sd"> """</span> |
| <span class="n">filter_query</span> <span class="o">=</span> <span class="p">[</span> |
| <span class="n">DagModel</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> |
| <span class="p">]</span> |
| <span class="k">if</span> <span class="n">including_subdags</span><span class="p">:</span> |
| <span class="n">filter_query</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">DagModel</span><span class="o">.</span><span class="n">root_dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">)</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">or_</span><span class="p">(</span><span class="o">*</span><span class="n">filter_query</span><span class="p">))</span><span class="o">.</span><span class="n">update</span><span class="p">(</span> |
| <span class="p">{</span><span class="n">DagModel</span><span class="o">.</span><span class="n">is_paused</span><span class="p">:</span> <span class="n">is_paused</span><span class="p">},</span> <span class="n">synchronize_session</span><span class="o">=</span><span class="s1">'fetch'</span> |
| <span class="p">)</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span></div> |
| |
| <span class="nd">@classmethod</span> |
| <span class="nd">@provide_session</span> |
| <div class="viewcode-block" id="DagModel.deactivate_deleted_dags"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.deactivate_deleted_dags">[docs]</a> <span class="k">def</span> <span class="nf">deactivate_deleted_dags</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">alive_dag_filelocs</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">session</span><span class="o">=</span><span class="n">NEW_SESSION</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Set ``is_active=False`` on the DAGs for which the DAG files have been removed.</span> |
| |
| <span class="sd"> :param alive_dag_filelocs: file paths of alive DAGs</span> |
| <span class="sd"> :param session: ORM Session</span> |
| <span class="sd"> """</span> |
| <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Deactivating DAGs (for which DAG files are deleted) from </span><span class="si">%s</span><span class="s2"> table "</span><span class="p">,</span> <span class="bp">cls</span><span class="o">.</span><span class="n">__tablename__</span><span class="p">)</span> |
| |
| <span class="n">dag_models</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span><span class="o">.</span><span class="n">all</span><span class="p">()</span> |
| <span class="k">for</span> <span class="n">dag_model</span> <span class="ow">in</span> <span class="n">dag_models</span><span class="p">:</span> |
| <span class="k">if</span> <span class="n">dag_model</span><span class="o">.</span><span class="n">fileloc</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="k">if</span> <span class="n">correct_maybe_zipped</span><span class="p">(</span><span class="n">dag_model</span><span class="o">.</span><span class="n">fileloc</span><span class="p">)</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">alive_dag_filelocs</span><span class="p">:</span> |
| <span class="n">dag_model</span><span class="o">.</span><span class="n">is_active</span> <span class="o">=</span> <span class="kc">False</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="k">continue</span></div> |
| |
| <span class="nd">@classmethod</span> |
| <div class="viewcode-block" id="DagModel.dags_needing_dagruns"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.dags_needing_dagruns">[docs]</a> <span class="k">def</span> <span class="nf">dags_needing_dagruns</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">session</span><span class="p">:</span> <span class="n">Session</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Return (and lock) a list of Dag objects that are due to create a new DagRun.</span> |
| |
| <span class="sd"> This will return a resultset of rows that is row-level-locked with a "SELECT ... FOR UPDATE" query,</span> |
| <span class="sd"> you should ensure that any scheduling decisions are made in a single transaction -- as soon as the</span> |
| <span class="sd"> transaction is committed it will be unlocked.</span> |
| <span class="sd"> """</span> |
| <span class="c1"># TODO[HA]: Bake this query, it is run _A lot_</span> |
| <span class="c1"># We limit so that _one_ scheduler doesn't try to do all the creation</span> |
| <span class="c1"># of dag runs</span> |
| |
| <span class="n">query</span> <span class="o">=</span> <span class="p">(</span> |
| <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">filter</span><span class="p">(</span> |
| <span class="bp">cls</span><span class="o">.</span><span class="n">is_paused</span> <span class="o">==</span> <span class="n">expression</span><span class="o">.</span><span class="n">false</span><span class="p">(),</span> |
| <span class="bp">cls</span><span class="o">.</span><span class="n">is_active</span> <span class="o">==</span> <span class="n">expression</span><span class="o">.</span><span class="n">true</span><span class="p">(),</span> |
| <span class="bp">cls</span><span class="o">.</span><span class="n">has_import_errors</span> <span class="o">==</span> <span class="n">expression</span><span class="o">.</span><span class="n">false</span><span class="p">(),</span> |
| <span class="bp">cls</span><span class="o">.</span><span class="n">next_dagrun_create_after</span> <span class="o"><=</span> <span class="n">func</span><span class="o">.</span><span class="n">now</span><span class="p">(),</span> |
| <span class="p">)</span> |
| <span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">next_dagrun_create_after</span><span class="p">)</span> |
| <span class="o">.</span><span class="n">limit</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">NUM_DAGS_PER_DAGRUN_QUERY</span><span class="p">)</span> |
| <span class="p">)</span> |
| |
| <span class="k">return</span> <span class="n">with_row_locks</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">of</span><span class="o">=</span><span class="bp">cls</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">,</span> <span class="o">**</span><span class="n">skip_locked</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">))</span></div> |
| |
| <div class="viewcode-block" id="DagModel.calculate_dagrun_date_fields"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagModel.calculate_dagrun_date_fields">[docs]</a> <span class="k">def</span> <span class="nf">calculate_dagrun_date_fields</span><span class="p">(</span> |
| <span class="bp">self</span><span class="p">,</span> |
| <span class="n">dag</span><span class="p">:</span> <span class="n">DAG</span><span class="p">,</span> |
| <span class="n">most_recent_dag_run</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">DataInterval</span><span class="p">],</span> |
| <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Calculate ``next_dagrun`` and `next_dagrun_create_after``</span> |
| |
| <span class="sd"> :param dag: The DAG object</span> |
| <span class="sd"> :param most_recent_dag_run: DataInterval (or datetime) of most recent run of this dag, or none</span> |
| <span class="sd"> if not yet scheduled.</span> |
| <span class="sd"> """</span> |
| <span class="n">most_recent_data_interval</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DataInterval</span><span class="p">]</span> |
| <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">most_recent_dag_run</span><span class="p">,</span> <span class="n">datetime</span><span class="p">):</span> |
| <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> |
| <span class="s2">"Passing a datetime to `DagModel.calculate_dagrun_date_fields` is deprecated. "</span> |
| <span class="s2">"Provide a data interval instead."</span><span class="p">,</span> |
| <span class="ne">DeprecationWarning</span><span class="p">,</span> |
| <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> |
| <span class="p">)</span> |
| <span class="n">most_recent_data_interval</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">infer_automated_data_interval</span><span class="p">(</span><span class="n">most_recent_dag_run</span><span class="p">)</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="n">most_recent_data_interval</span> <span class="o">=</span> <span class="n">most_recent_dag_run</span> |
| <span class="n">next_dagrun_info</span> <span class="o">=</span> <span class="n">dag</span><span class="o">.</span><span class="n">next_dagrun_info</span><span class="p">(</span><span class="n">most_recent_data_interval</span><span class="p">)</span> |
| <span class="k">if</span> <span class="n">next_dagrun_info</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun_data_interval</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun_create_after</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun_data_interval</span> <span class="o">=</span> <span class="n">next_dagrun_info</span><span class="o">.</span><span class="n">data_interval</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun</span> <span class="o">=</span> <span class="n">next_dagrun_info</span><span class="o">.</span><span class="n">logical_date</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun_create_after</span> <span class="o">=</span> <span class="n">next_dagrun_info</span><span class="o">.</span><span class="n">run_after</span> |
| |
| <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> |
| <span class="s2">"Setting next_dagrun for </span><span class="si">%s</span><span class="s2"> to </span><span class="si">%s</span><span class="s2">, run_after=</span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> |
| <span class="n">dag</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun</span><span class="p">,</span> |
| <span class="bp">self</span><span class="o">.</span><span class="n">next_dagrun_create_after</span><span class="p">,</span></div></div> |
| <span class="p">)</span> |
| |
| |
| <div class="viewcode-block" id="dag"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.dag">[docs]</a><span class="k">def</span> <span class="nf">dag</span><span class="p">(</span><span class="o">*</span><span class="n">dag_args</span><span class="p">,</span> <span class="o">**</span><span class="n">dag_kwargs</span><span class="p">):</span> |
| <span class="sd">"""</span> |
| <span class="sd"> Python dag decorator. Wraps a function into an Airflow DAG.</span> |
| <span class="sd"> Accepts kwargs for operator kwarg. Can be used to parameterize DAGs.</span> |
| |
| <span class="sd"> :param dag_args: Arguments for DAG object</span> |
| <span class="sd"> :param dag_kwargs: Kwargs for DAG object.</span> |
| <span class="sd"> """</span> |
| |
| <span class="k">def</span> <span class="nf">wrapper</span><span class="p">(</span><span class="n">f</span><span class="p">:</span> <span class="n">Callable</span><span class="p">):</span> |
| <span class="c1"># Get dag initializer signature and bind it to validate that dag_args, and dag_kwargs are correct</span> |
| <span class="n">dag_sig</span> <span class="o">=</span> <span class="n">signature</span><span class="p">(</span><span class="n">DAG</span><span class="o">.</span><span class="fm">__init__</span><span class="p">)</span> |
| <span class="n">dag_bound_args</span> <span class="o">=</span> <span class="n">dag_sig</span><span class="o">.</span><span class="n">bind_partial</span><span class="p">(</span><span class="o">*</span><span class="n">dag_args</span><span class="p">,</span> <span class="o">**</span><span class="n">dag_kwargs</span><span class="p">)</span> |
| |
| <span class="nd">@functools</span><span class="o">.</span><span class="n">wraps</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> |
| <span class="k">def</span> <span class="nf">factory</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> |
| <span class="c1"># Generate signature for decorated function and bind the arguments when called</span> |
| <span class="c1"># we do this to extract parameters so we can annotate them on the DAG object.</span> |
| <span class="c1"># In addition, this fails if we are missing any args/kwargs with TypeError as expected.</span> |
| <span class="n">f_sig</span> <span class="o">=</span> <span class="n">signature</span><span class="p">(</span><span class="n">f</span><span class="p">)</span><span class="o">.</span><span class="n">bind</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> |
| <span class="c1"># Apply defaults to capture default values if set.</span> |
| <span class="n">f_sig</span><span class="o">.</span><span class="n">apply_defaults</span><span class="p">()</span> |
| |
| <span class="c1"># Set function name as dag_id if not set</span> |
| <span class="n">dag_id</span> <span class="o">=</span> <span class="n">dag_bound_args</span><span class="o">.</span><span class="n">arguments</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'dag_id'</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span> |
| <span class="n">dag_bound_args</span><span class="o">.</span><span class="n">arguments</span><span class="p">[</span><span class="s1">'dag_id'</span><span class="p">]</span> <span class="o">=</span> <span class="n">dag_id</span> |
| |
| <span class="c1"># Initialize DAG with bound arguments</span> |
| <span class="k">with</span> <span class="n">DAG</span><span class="p">(</span><span class="o">*</span><span class="n">dag_bound_args</span><span class="o">.</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">dag_bound_args</span><span class="o">.</span><span class="n">kwargs</span><span class="p">)</span> <span class="k">as</span> <span class="n">dag_obj</span><span class="p">:</span> |
| <span class="c1"># Set DAG documentation from function documentation.</span> |
| <span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="vm">__doc__</span><span class="p">:</span> |
| <span class="n">dag_obj</span><span class="o">.</span><span class="n">doc_md</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="vm">__doc__</span> |
| |
| <span class="c1"># Generate DAGParam for each function arg/kwarg and replace it for calling the function.</span> |
| <span class="c1"># All args/kwargs for function will be DAGParam object and replaced on execution time.</span> |
| <span class="n">f_kwargs</span> <span class="o">=</span> <span class="p">{}</span> |
| <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">f_sig</span><span class="o">.</span><span class="n">arguments</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> |
| <span class="n">f_kwargs</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">dag_obj</span><span class="o">.</span><span class="n">param</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span> |
| |
| <span class="c1"># set file location to caller source path</span> |
| <span class="n">back</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">_getframe</span><span class="p">()</span><span class="o">.</span><span class="n">f_back</span> |
| <span class="n">dag_obj</span><span class="o">.</span><span class="n">fileloc</span> <span class="o">=</span> <span class="n">back</span><span class="o">.</span><span class="n">f_code</span><span class="o">.</span><span class="n">co_filename</span> <span class="k">if</span> <span class="n">back</span> <span class="k">else</span> <span class="s2">""</span> |
| |
| <span class="c1"># Invoke function to create operators in the DAG scope.</span> |
| <span class="n">f</span><span class="p">(</span><span class="o">**</span><span class="n">f_kwargs</span><span class="p">)</span> |
| |
| <span class="c1"># Return dag object such that it's accessible in Globals.</span> |
| <span class="k">return</span> <span class="n">dag_obj</span> |
| |
| <span class="k">return</span> <span class="n">factory</span> |
| |
| <span class="k">return</span> <span class="n">wrapper</span></div> |
| |
| |
| <span class="n">STATICA_HACK</span> <span class="o">=</span> <span class="kc">True</span> |
| <span class="nb">globals</span><span class="p">()[</span><span class="s1">'kcah_acitats'</span><span class="p">[::</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()]</span> <span class="o">=</span> <span class="kc">False</span> |
| <span class="k">if</span> <span class="n">STATICA_HACK</span><span class="p">:</span> <span class="c1"># pragma: no cover</span> |
| |
| <span class="kn">from</span> <span class="nn">airflow.models.serialized_dag</span> <span class="kn">import</span> <span class="n">SerializedDagModel</span> |
| |
| <span class="n">DagModel</span><span class="o">.</span><span class="n">serialized_dag</span> <span class="o">=</span> <span class="n">relationship</span><span class="p">(</span><span class="n">SerializedDagModel</span><span class="p">)</span> |
| <span class="sd">""":sphinx-autoapi-skip:"""</span> |
| |
| |
| <div class="viewcode-block" id="DagContext"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagContext">[docs]</a><span class="k">class</span> <span class="nc">DagContext</span><span class="p">:</span> |
| <span class="sd">"""</span> |
| <span class="sd"> DAG context is used to keep the current DAG when DAG is used as ContextManager.</span> |
| |
| <span class="sd"> You can use DAG as context:</span> |
| |
| <span class="sd"> .. code-block:: python</span> |
| |
| <span class="sd"> with DAG(</span> |
| <span class="sd"> dag_id="example_dag",</span> |
| <span class="sd"> default_args=default_args,</span> |
| <span class="sd"> schedule_interval="0 0 * * *",</span> |
| <span class="sd"> dagrun_timeout=timedelta(minutes=60),</span> |
| <span class="sd"> ) as dag:</span> |
| <span class="sd"> ...</span> |
| |
| <span class="sd"> If you do this the context stores the DAG and whenever new task is created, it will use</span> |
| <span class="sd"> such stored DAG as the parent DAG.</span> |
| |
| <span class="sd"> """</span> |
| |
| <span class="n">_context_managed_dag</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DAG</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="n">_previous_context_managed_dags</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">DAG</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span> |
| |
| <span class="nd">@classmethod</span> |
| <div class="viewcode-block" id="DagContext.push_context_managed_dag"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagContext.push_context_managed_dag">[docs]</a> <span class="k">def</span> <span class="nf">push_context_managed_dag</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">dag</span><span class="p">:</span> <span class="n">DAG</span><span class="p">):</span> |
| <span class="k">if</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_context_managed_dag</span><span class="p">:</span> |
| <span class="bp">cls</span><span class="o">.</span><span class="n">_previous_context_managed_dags</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">_context_managed_dag</span><span class="p">)</span> |
| <span class="bp">cls</span><span class="o">.</span><span class="n">_context_managed_dag</span> <span class="o">=</span> <span class="n">dag</span></div> |
| |
| <span class="nd">@classmethod</span> |
| <div class="viewcode-block" id="DagContext.pop_context_managed_dag"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagContext.pop_context_managed_dag">[docs]</a> <span class="k">def</span> <span class="nf">pop_context_managed_dag</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">DAG</span><span class="p">]:</span> |
| <span class="n">old_dag</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_context_managed_dag</span> |
| <span class="k">if</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_previous_context_managed_dags</span><span class="p">:</span> |
| <span class="bp">cls</span><span class="o">.</span><span class="n">_context_managed_dag</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_previous_context_managed_dags</span><span class="o">.</span><span class="n">pop</span><span class="p">()</span> |
| <span class="k">else</span><span class="p">:</span> |
| <span class="bp">cls</span><span class="o">.</span><span class="n">_context_managed_dag</span> <span class="o">=</span> <span class="kc">None</span> |
| <span class="k">return</span> <span class="n">old_dag</span></div> |
| |
| <span class="nd">@classmethod</span> |
| <div class="viewcode-block" id="DagContext.get_current_dag"><a class="viewcode-back" href="../../../_api/airflow/models/dag/index.html#airflow.models.dag.DagContext.get_current_dag">[docs]</a> <span class="k">def</span> <span class="nf">get_current_dag</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span> <span class="o">-></span> <span class="n">Optional</span><span class="p">[</span><span class="n">DAG</span><span class="p">]:</span> |
| <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_context_managed_dag</span></div></div> |
| </pre></div> |
| |
| <div class="pager" role="navigation" aria-label="related navigation"><button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" disabled>Previous</button><button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" disabled>Next</button> |
| </div> |
| </div> |
| |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="rating-container"> |
| <p class="bodytext__medium--greyish-brown font-weight-500">Was this entry helpful?</p> |
| <div class="rating"> |
| |
| <div id="rate-star-5" class="rate-star"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="21.05" height="20.02" viewBox="0 0 21.05 20.02"> |
| <g id="Group_806" data-name="Group 806" transform="translate(-774.404 -13.178)"> |
| <path id="Path_715" d="M791.434 33.2l-6.5-3.42-6.5 3.42 1.242-7.243-5.262-5.13 7.273-1.057 3.252-6.59 3.252 6.59 7.273 1.057-5.262 5.13z" fill="#017cee" data-name="Path 715"></path> |
| </g> |
| </svg> |
| |
| </div> |
| |
| <div id="rate-star-4" class="rate-star"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="21.05" height="20.02" viewBox="0 0 21.05 20.02"> |
| <g id="Group_806" data-name="Group 806" transform="translate(-774.404 -13.178)"> |
| <path id="Path_715" d="M791.434 33.2l-6.5-3.42-6.5 3.42 1.242-7.243-5.262-5.13 7.273-1.057 3.252-6.59 3.252 6.59 7.273 1.057-5.262 5.13z" fill="#017cee" data-name="Path 715"></path> |
| </g> |
| </svg> |
| |
| </div> |
| |
| <div id="rate-star-3" class="rate-star"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="21.05" height="20.02" viewBox="0 0 21.05 20.02"> |
| <g id="Group_806" data-name="Group 806" transform="translate(-774.404 -13.178)"> |
| <path id="Path_715" d="M791.434 33.2l-6.5-3.42-6.5 3.42 1.242-7.243-5.262-5.13 7.273-1.057 3.252-6.59 3.252 6.59 7.273 1.057-5.262 5.13z" fill="#017cee" data-name="Path 715"></path> |
| </g> |
| </svg> |
| |
| </div> |
| |
| <div id="rate-star-2" class="rate-star"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="21.05" height="20.02" viewBox="0 0 21.05 20.02"> |
| <g id="Group_806" data-name="Group 806" transform="translate(-774.404 -13.178)"> |
| <path id="Path_715" d="M791.434 33.2l-6.5-3.42-6.5 3.42 1.242-7.243-5.262-5.13 7.273-1.057 3.252-6.59 3.252 6.59 7.273 1.057-5.262 5.13z" fill="#017cee" data-name="Path 715"></path> |
| </g> |
| </svg> |
| |
| </div> |
| |
| <div id="rate-star-1" class="rate-star"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="21.05" height="20.02" viewBox="0 0 21.05 20.02"> |
| <g id="Group_806" data-name="Group 806" transform="translate(-774.404 -13.178)"> |
| <path id="Path_715" d="M791.434 33.2l-6.5-3.42-6.5 3.42 1.242-7.243-5.262-5.13 7.273-1.057 3.252-6.59 3.252 6.59 7.273 1.057-5.262 5.13z" fill="#017cee" data-name="Path 715"></path> |
| </g> |
| </svg> |
| |
| </div> |
| |
| </div> |
| </div> |
| |
| </main> |
| |
| |
| |
| |
| <nav class="wy-nav-side-toc"> |
| <div class="wy-menu-vertical"> |
| |
| </div> |
| </nav> |
| |
| </div> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </div> |
| |
| |
| |
| |
| <footer> |
| <div class="footer-section footer-section__media-section"> |
| <div class="d-flex align-items-center"> |
| |
| <a class="footer-section__media-section--link" target="_blank" href="https://github.com/apache/airflow"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="46.14" height="45" viewBox="0 0 46.14 45"> |
| <path id="Path_207" d="M228.962 1078.578a23.072 23.072 0 0 0-7.29 44.96c1.154.212 1.574-.5 1.574-1.112 0-.548-.02-2-.031-3.924-6.417 1.394-7.771-3.093-7.771-3.093a6.109 6.109 0 0 0-2.562-3.375c-2.095-1.431.159-1.4.159-1.4a4.846 4.846 0 0 1 3.533 2.377c2.058 3.525 5.4 2.507 6.714 1.917a4.926 4.926 0 0 1 1.464-3.084c-5.123-.582-10.508-2.562-10.508-11.4a8.919 8.919 0 0 1 2.374-6.191 8.3 8.3 0 0 1 .226-6.105s1.937-.62 6.344 2.365a21.857 21.857 0 0 1 11.551 0c4.4-2.985 6.338-2.365 6.338-2.365a8.284 8.284 0 0 1 .23 6.105 8.9 8.9 0 0 1 2.371 6.191c0 8.862-5.393 10.812-10.533 11.384a5.506 5.506 0 0 1 1.566 4.272c0 3.084-.028 5.572-.028 6.329 0 .617.415 1.334 1.586 1.109a23.073 23.073 0 0 0-7.308-44.958z" fill="#fff" fill-rule="evenodd" data-name="Path 207" transform="translate(-205.894 -1078.578)"></path> |
| </svg> |
| |
| </a> |
| |
| <a class="footer-section__media-section--link" target="_blank" href="https://github.com/apache/airflow/issues"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="45" height="45" viewBox="0 0 45 45"> |
| <g id="Group_210" data-name="Group 210" transform="translate(-339.789 -1315.282)"> |
| <path id="Path_218" d="M394.82 1315.282h-21.671a9.784 9.784 0 0 0 9.784 9.778h3.986v3.857a9.784 9.784 0 0 0 9.784 9.771v-21.523a1.884 1.884 0 0 0-1.883-1.883z" fill="#fff" data-name="Path 218" transform="translate(-11.914)"></path> |
| <path id="Path_219" d="M378.14 1332.072h-21.671a9.778 9.778 0 0 0 9.778 9.778h4.018v3.857a9.784 9.784 0 0 0 9.752 9.778v-21.536a1.877 1.877 0 0 0-1.877-1.877z" fill="#fff" data-name="Path 219" transform="translate(-5.957 -5.996)"></path> |
| <path id="Path_220" d="M361.46 1348.862h-21.671a9.778 9.778 0 0 0 9.778 9.778h3.992v3.857a9.778 9.778 0 0 0 9.778 9.778v-21.529a1.883 1.883 0 0 0-1.877-1.884z" fill="#fff" data-name="Path 220" transform="translate(0 -11.993)"></path> |
| </g> |
| </svg> |
| |
| </a> |
| |
| <a class="footer-section__media-section--link" target="_blank" href="https://s.apache.org/airflow-slack"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="45.073" height="45.073" viewBox="0 0 45.073 45.073"> |
| <g id="Group_208" data-name="Group 208" transform="translate(-661.145 -806.287)"> |
| <g id="Group_204" data-name="Group 204" transform="translate(661.145 830.01)"> |
| <path id="Path_208" d="M670.634 856.859a4.744 4.744 0 1 1-4.744-4.744h4.744z" fill="#fff" data-name="Path 208" transform="translate(-661.145 -852.115)"></path> |
| <path id="Path_209" d="M684.059 856.859a4.744 4.744 0 0 1 9.489 0v11.861a4.744 4.744 0 1 1-9.489 0z" fill="#fff" data-name="Path 209" transform="translate(-672.198 -852.115)"></path> |
| </g> |
| <g id="Group_205" data-name="Group 205" transform="translate(661.145 806.287)"> |
| <path id="Path_210" d="M688.8 815.776a4.744 4.744 0 1 1 4.744-4.745v4.745z" fill="#fff" data-name="Path 210" transform="translate(-672.198 -806.287)"></path> |
| <path id="Path_211" d="M677.751 829.2a4.744 4.744 0 0 1 0 9.489H665.89a4.744 4.744 0 1 1 0-9.489z" fill="#fff" data-name="Path 211" transform="translate(-661.145 -817.34)"></path> |
| </g> |
| <g id="Group_206" data-name="Group 206" transform="translate(684.868 806.287)"> |
| <path id="Path_212" d="M729.887 833.945a4.744 4.744 0 1 1 4.745 4.745h-4.745z" fill="#fff" data-name="Path 212" transform="translate(-718.026 -817.34)"></path> |
| <path id="Path_213" d="M716.462 822.893a4.744 4.744 0 1 1-9.489 0v-11.862a4.744 4.744 0 0 1 9.489 0z" fill="#fff" data-name="Path 213" transform="translate(-706.973 -806.287)"></path> |
| </g> |
| <g id="Group_207" data-name="Group 207" transform="translate(684.868 830.01)"> |
| <path id="Path_214" d="M711.718 875.029a4.744 4.744 0 1 1-4.745 4.744v-4.744z" fill="#fff" data-name="Path 214" transform="translate(-706.973 -863.168)"></path> |
| <path id="Path_215" d="M711.718 861.6a4.744 4.744 0 1 1 0-9.489h11.861a4.744 4.744 0 0 1 0 9.489z" fill="#fff" data-name="Path 215" transform="translate(-706.973 -852.115)"></path> |
| </g> |
| </g> |
| </svg> |
| |
| </a> |
| |
| <a class="footer-section__media-section--link" target="_blank" href="https://stackoverflow.com/questions/tagged/airflow"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="37.647" height="44.6" viewBox="0 0 37.647 44.6"> |
| <g id="Group_209" data-name="Group 209" transform="translate(-645.2 -975.455)"> |
| <path id="Path_216" d="M677.028 1043.1v-11.948h3.966v15.914H645.2v-15.914h3.966v11.948z" fill="#fff" data-name="Path 216" transform="translate(0 -27.014)"></path> |
| <path id="Path_217" d="M661.012 1003.008l19.467 4.069.824-3.914-19.467-4.069zm2.575-9.27l18.025 8.395 1.648-3.605-18.025-8.446zm5-8.858l15.3 12.721 2.524-3.039-15.3-12.721zm9.888-9.425l-3.193 2.369 11.845 15.965 3.193-2.369zm-17.875 36.617h19.879v-3.966H660.6z" fill="#fff" data-name="Path 217" transform="translate(-7.469)"></path> |
| </g> |
| </svg> |
| |
| </a> |
| |
| <a class="footer-section__media-section--link" target="_blank" href="https://twitter.com/ApacheAirflow"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="44.355" height="44.355" viewBox="0 0 44.355 44.355"> |
| <g id="Group_211" data-name="Group 211" transform="translate(-503 -1382.2)"> |
| <path id="Path_221" d="M541.811 1382.2h-33.267a5.546 5.546 0 0 0-5.544 5.544v33.266a5.547 5.547 0 0 0 5.544 5.545h33.267a5.546 5.546 0 0 0 5.544-5.545v-33.266a5.546 5.546 0 0 0-5.544-5.544zm-5.611 16.533c.011.244.011.488.011.732 0 7.507-5.722 16.178-16.179 16.178a16.173 16.173 0 0 1-8.7-2.539 10.92 10.92 0 0 0 1.353.078 11.4 11.4 0 0 0 7.064-2.44 5.685 5.685 0 0 1-5.311-3.947 5.725 5.725 0 0 0 2.561-.1 5.7 5.7 0 0 1-4.557-5.578v-.078a5.752 5.752 0 0 0 2.572.71 5.709 5.709 0 0 1-1.763-7.6 16.146 16.146 0 0 0 11.721 5.944 6.282 6.282 0 0 1-.144-1.3 5.688 5.688 0 0 1 9.836-3.892 11.3 11.3 0 0 0 3.615-1.375 5.721 5.721 0 0 1-2.506 3.149 11.538 11.538 0 0 0 3.271-.9 11.585 11.585 0 0 1-2.844 2.958z" fill="#fff" data-name="Path 221"></path> |
| </g> |
| </svg> |
| |
| </a> |
| |
| <a class="footer-section__media-section--link" target="_blank" href="https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="49.594" height="34.941" viewBox="0 0 49.594 34.941"> |
| <path id="Path_223" d="M1124.557 1230a6.232 6.232 0 0 0-4.385-4.413c-3.867-1.043-19.376-1.043-19.376-1.043s-15.508 0-19.376 1.043a6.232 6.232 0 0 0-4.385 4.413c-1.036 3.893-1.036 12.014-1.036 12.014s0 8.122 1.036 12.015a6.232 6.232 0 0 0 4.385 4.413c3.867 1.043 19.376 1.043 19.376 1.043s15.509 0 19.376-1.043a6.232 6.232 0 0 0 4.385-4.413c1.036-3.893 1.036-12.015 1.036-12.015s.001-8.123-1.036-12.014zm-28.833 19.388v-14.748l12.962 7.374z" fill="#fff" data-name="Path 223" transform="translate(-1076 -1224.542)"></path> |
| </svg> |
| |
| </a> |
| |
| </div> |
| |
| |
| <div class="footer-section__media-section--button-with-text"> |
| <span class="footer-section__media-section--text">Want to be a part of Apache Airflow?</span> |
| <a href="/community"> |
| |
| <button id="" class="btn-filled bodytext__medium--white ">Join community</button> |
| |
| </a> |
| </div> |
| |
| |
| </div> |
| <div class="footer-section footer-section__policies-section"> |
| <div class="footer-section"> |
| <span>© The Apache Software Foundation <script>document.write(new Date().getFullYear())</script></span> |
| <div class="footer-section__policies-section--policies"> |
| |
| <a href="https://www.apache.org/licenses/" class="footer-section__policies-section--policy-item"> |
| <span>License</span> |
| </a> |
| |
| <a href="https://www.apache.org/foundation/sponsorship.html" class="footer-section__policies-section--policy-item"> |
| <span>Donate</span> |
| </a> |
| |
| <a href="https://www.apache.org/foundation/thanks.html" class="footer-section__policies-section--policy-item"> |
| <span>Thanks</span> |
| </a> |
| |
| <a href="https://www.apache.org/security/" class="footer-section__policies-section--policy-item"> |
| <span>Security</span> |
| </a> |
| |
| |
| </div> |
| </div> |
| <span class="footer-section__policies-section--disclaimer"> |
| Apache Airflow, Apache, Airflow, the Airflow logo, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation. |
| All other products or name brands are trademarks of their respective holders, including The Apache Software Foundation. |
| </span> |
| </div> |
| </footer> |
| <script type="text/javascript" src="../../../_static/_gen/js/docs.js"></script> |
| <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script> |
| <script src="https://code.jquery.com/jquery-3.4.1.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script> |
| <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script> |
| <script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script> |
| <script src="../../../_static/jquery.js"></script> |
| <script src="../../../_static/underscore.js"></script> |
| <script src="../../../_static/doctools.js"></script> |
| <script src="../../../_static/clipboard.min.js"></script> |
| <script src="../../../_static/copybutton.js"></script> |
| <script src="../../../_static/gh-jira-links.js"></script> |
| </body> |
| </html> |