| |
| |
| <!doctype html> |
| <html lang="en" class="no-js"> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> |
| <meta name="generator" content="Hugo 0.91.2" /> |
| |
| <META NAME="ROBOTS" CONTENT="INDEX, FOLLOW"> |
| |
| |
| |
| <link rel="apple-touch-icon" sizes="57x57" href="/favicons/apple-icon-57x57.png"> |
| <link rel="apple-touch-icon" sizes="60x60" href="/favicons/apple-icon-60x60.png"> |
| <link rel="apple-touch-icon" sizes="72x72" href="/favicons/apple-icon-72x72.png"> |
| <link rel="apple-touch-icon" sizes="76x76" href="/favicons/apple-icon-76x76.png"> |
| <link rel="apple-touch-icon" sizes="114x114" href="/favicons/apple-icon-114x114.png"> |
| <link rel="apple-touch-icon" sizes="120x120" href="/favicons/apple-icon-120x120.png"> |
| <link rel="apple-touch-icon" sizes="144x144" href="/favicons/apple-icon-144x144.png"> |
| <link rel="apple-touch-icon" sizes="152x152" href="/favicons/apple-icon-152x152.png"> |
| <link rel="apple-touch-icon" sizes="180x180" href="/favicons/apple-icon-180x180.png"> |
| <link rel="icon" type="image/png" sizes="192x192" href="/favicons/android-icon-192x192.png"> |
| <link rel="icon" type="image/png" sizes="32x32" href="/favicons/favicon-32x32.png"> |
| <link rel="icon" type="image/png" sizes="96x96" href="/favicons/favicon-96x96.png"> |
| <link rel="icon" type="image/png" sizes="16x16" href="/favicons/favicon-16x16.png"> |
| <link rel="manifest" href="/manifest.json"> |
| <meta name="msapplication-TileColor" content="#ffffff"> |
| <meta name="msapplication-TileImage" content="/favicons/ms-icon-144x144.png"> |
| <meta name="theme-color" content="#ffffff"> |
| |
| <title>Sift | Apache Airflow</title><meta property="og:title" content="Sift" /> |
| <meta property="og:description" content="What was the problem? At Sift, we’re constantly training machine learning models that feed into the core of Sift’s Digital Trust & Safety platform. The platform gives our customers a way to discern suspicious online behavior from trustworthy behavior, allowing our customers to protect their online transactions, maintain the integrity of their content platforms, and keep their users’ accounts secure. To make this possible, we’ve built model training pipelines that consist of hundreds of steps in MapReduce and Spark, with complex requirements between them." /> |
| <meta property="og:type" content="article" /> |
| <meta property="og:url" content="/use-cases/sift/" /><meta property="og:image" content="/images/feature-image.png"/><meta property="article:section" content="use-cases" /> |
| |
| <meta property="article:modified_time" content="2024-04-21T10:55:02+03:00" /><meta property="og:site_name" content="Apache Airflow" /> |
| |
| <meta itemprop="name" content="Sift"> |
| <meta itemprop="description" content="What was the problem? At Sift, we’re constantly training machine learning models that feed into the core of Sift’s Digital Trust & Safety platform. The platform gives our customers a way to discern suspicious online behavior from trustworthy behavior, allowing our customers to protect their online transactions, maintain the integrity of their content platforms, and keep their users’ accounts secure. To make this possible, we’ve built model training pipelines that consist of hundreds of steps in MapReduce and Spark, with complex requirements between them."> |
| <meta itemprop="dateModified" content="2024-04-21T10:55:02+03:00" /> |
| <meta itemprop="wordCount" content="641"><meta itemprop="image" content="/images/feature-image.png"/> |
| <meta itemprop="keywords" content="" /><meta name="twitter:card" content="summary_large_image"/> |
| <meta name="twitter:image" content="/images/feature-image.png"/> |
| |
| <meta name="twitter:title" content="Sift"/> |
| <meta name="twitter:description" content="What was the problem? At Sift, we’re constantly training machine learning models that feed into the core of Sift’s Digital Trust & Safety platform. The platform gives our customers a way to discern suspicious online behavior from trustworthy behavior, allowing our customers to protect their online transactions, maintain the integrity of their content platforms, and keep their users’ accounts secure. To make this possible, we’ve built model training pipelines that consist of hundreds of steps in MapReduce and Spark, with complex requirements between them."/> |
| |
| |
| |
| |
| |
| |
| |
| <link rel="preload" href="/scss/main.min.6668c09ec5024c20cbf3e5c6d1f29463e1b7a1797526dc9cade6bda004c6da9d.css" as="style"> |
| <link href="/scss/main.min.6668c09ec5024c20cbf3e5c6d1f29463e1b7a1797526dc9cade6bda004c6da9d.css" rel="stylesheet" integrity=""> |
| |
| <script |
| src="https://code.jquery.com/jquery-3.3.1.min.js" |
| integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" |
| crossorigin="anonymous"></script> |
| |
| |
| <meta name="description" content="Platform created by the community to programmatically author, schedule and monitor workflows." /> |
| |
| |
| |
| |
| <link rel="preload" href="/scss/main-custom.min.81a544302502ad91a303486a1565b735d95b991a21b74ab73619fd0e40431272.css" as="style"> |
| <link href="/scss/main-custom.min.81a544302502ad91a303486a1565b735d95b991a21b74ab73619fd0e40431272.css" rel="stylesheet" integrity=""> |
| |
| |
| |
| <script> |
| var _paq = window._paq = window._paq || []; |
| |
| |
| _paq.push(['disableCookies']); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '13']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| |
| |
| |
| <link rel="preload" href="/main.22619.js" as="script"> |
| <link rel="preload" href="/chunk-1.22619.js" as="script"> |
| |
| <link rel="preload" href="/chunk-4.22619.js" as="script"> |
| |
| |
| |
| |
| </head> |
| <body class="td-page"> |
| <header> |
| |
| |
| |
| <a href="https://airflowsummit.org" target="_blank" class="d-block fixed-top px-3 py-2 bg-success text-center text-bold text-white"> |
| Airflow Summit 2024 is coming September 10-12. Register now! |
| </a> |
| <nav class="js-navbar-scroll navbar" style="top: 40px;"> |
| <div class="navbar__icon-container"> |
| <a href="/"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="155.314" height="60" viewBox="0 0 155.314 60"> |
| <defs> |
| <clipPath id="clip-path"> |
| <path id="Rectangle_1" d="M0 0h155.314v60H0z" fill="none" data-name="Rectangle 1"/> |
| </clipPath> |
| </defs> |
| <g id="logo" transform="translate(-1305 -780.355)"> |
| <g id="Group_2" clip-path="url(#clip-path)" data-name="Group 2" transform="translate(1305 780.355)"> |
| <g id="Group_1" data-name="Group 1" transform="translate(.486 .486)"> |
| <path id="Path_1" d="M1307.562 880.867l28.187-28.893a.521.521 0 0 0 .063-.666c-1.714-2.393-4.877-2.808-6.049-4.416-3.472-4.763-4.353-7.459-5.845-7.292a.456.456 0 0 0-.271.143l-10.182 10.438c-5.858 6-6.7 19.225-6.852 30.3a.552.552 0 0 0 .949.386z" fill="#017cee" data-name="Path 1" transform="translate(-1306.613 -822.232)"/> |
| <path id="Path_2" d="M1405.512 908.489l-28.893-28.189a.521.521 0 0 0-.667-.063c-2.393 1.715-2.808 4.877-4.416 6.049-4.763 3.472-7.459 4.353-7.292 5.845a.456.456 0 0 0 .143.27l10.438 10.182c6 5.858 19.225 6.7 30.3 6.852a.552.552 0 0 0 .387-.946z" fill="#00ad46" data-name="Path 2" transform="translate(-1346.876 -850.567)"/> |
| <path id="Path_3" d="M1373.909 902.252c-3.28-3.2-4.8-9.53 1.486-22.583-10.219 4.567-13.8 10.57-12.039 12.289z" fill="#04d659" data-name="Path 3" transform="translate(-1345.96 -850.233)"/> |
| <path id="Path_4" d="M1433.132 782.359l-28.186 28.893a.52.52 0 0 0-.063.666c1.715 2.393 4.876 2.808 6.049 4.416 3.472 4.763 4.354 7.459 5.845 7.292a.454.454 0 0 0 .271-.143l10.182-10.438c5.858-6 6.7-19.225 6.852-30.3a.553.553 0 0 0-.95-.386z" fill="#00c7d4" data-name="Path 4" transform="translate(-1375.21 -782.123)"/> |
| <path id="Path_5" d="M1426.9 881.155c-3.2 3.28-9.53 4.8-22.584-1.486 4.567 10.219 10.57 13.8 12.289 12.039z" fill="#11e1ee" data-name="Path 5" transform="translate(-1374.875 -850.233)"/> |
| <path id="Path_6" d="M1307 782.919l28.893 28.186a.521.521 0 0 0 .666.063c2.393-1.715 2.808-4.877 4.416-6.049 4.763-3.472 7.459-4.353 7.292-5.845a.459.459 0 0 0-.143-.271l-10.438-10.182c-6-5.858-19.225-6.7-30.3-6.852a.552.552 0 0 0-.386.95z" fill="#e43921" data-name="Path 6" transform="translate(-1306.766 -781.97)"/> |
| <path id="Path_7" d="M1405.8 804.711c3.28 3.2 4.8 9.53-1.486 22.584 10.219-4.567 13.8-10.571 12.039-12.289z" fill-rule="evenodd" fill="#ff7557" data-name="Path 7" transform="translate(-1374.875 -797.859)"/> |
| <path id="Path_8" d="M1329.355 849.266c3.2-3.28 9.53-4.8 22.584 1.486-4.567-10.219-10.57-13.8-12.289-12.039z" fill="#0cb6ff" data-name="Path 8" transform="translate(-1322.503 -821.316)"/> |
| <circle id="Ellipse_1" cx="1.26" cy="1.26" r="1.26" fill="#4a4848" data-name="Ellipse 1" transform="translate(28.18 28.171)"/> |
| <path id="Path_9" d="M1527.558 827.347a.229.229 0 0 1-.223-.223.458.458 0 0 1 .011-.123l2.766-7.214a.346.346 0 0 1 .357-.245h.758a.348.348 0 0 1 .357.245l2.754 7.214.022.123a.228.228 0 0 1-.223.223h-.568a.288.288 0 0 1-.19-.056.352.352 0 0 1-.089-.134l-.613-1.583h-3.657l-.613 1.583a.317.317 0 0 1-.1.134.269.269 0 0 1-.178.056zm4.795-2.732l-1.505-3.958-1.505 3.958zm3.322 4.85a.258.258 0 0 1-.189-.078.241.241 0 0 1-.067-.178v-7.4a.241.241 0 0 1 .067-.178.258.258 0 0 1 .189-.078h.513a.268.268 0 0 1 .256.256v.49a2.118 2.118 0 0 1 1.828-.858 2.092 2.092 0 0 1 1.751.736 3.135 3.135 0 0 1 .636 1.9q.011.122.011.379t-.011.379a3.168 3.168 0 0 1-.636 1.9 2.111 2.111 0 0 1-1.751.736 2.154 2.154 0 0 1-1.806-.836v2.587a.241.241 0 0 1-.067.178.223.223 0 0 1-.179.078zm2.364-2.91a1.324 1.324 0 0 0 1.149-.491 2.266 2.266 0 0 0 .4-1.293q.011-.111.011-.323 0-2.107-1.562-2.107a1.365 1.365 0 0 0-1.159.513 2.111 2.111 0 0 0-.412 1.2l-.012.424.012.435a1.862 1.862 0 0 0 .424 1.149 1.4 1.4 0 0 0 1.148.493zm5.628.9a2.329 2.329 0 0 1-1.015-.223 1.94 1.94 0 0 1-.747-.6 1.487 1.487 0 0 1-.268-.859 1.459 1.459 0 0 1 .6-1.2 3.4 3.4 0 0 1 1.65-.624l1.661-.234v-.323q0-1.137-1.3-1.137a1.4 1.4 0 0 0-.8.212 1.376 1.376 0 0 0-.468.48.305.305 0 0 1-.089.145.18.18 0 0 1-.134.045h-.48a.23.23 0 0 1-.245-.245 1.17 1.17 0 0 1 .245-.6 1.931 1.931 0 0 1 .747-.591 2.7 2.7 0 0 1 1.238-.256 2.351 2.351 0 0 1 1.8.591 2.032 2.032 0 0 1 .547 1.45v3.613a.257.257 0 0 1-.078.19.24.24 0 0 1-.178.067h-.513a.233.233 0 0 1-.257-.256v-.479a1.923 1.923 0 0 1-.714.6 2.557 2.557 0 0 1-1.203.237zm.234-.836a1.579 1.579 0 0 0 1.182-.469 1.881 1.881 0 0 0 .468-1.371v-.312l-1.293.19a2.918 2.918 0 0 0-1.193.379.761.761 0 0 0-.4.658.784.784 0 0 0 .368.691 1.585 1.585 0 0 0 .867.237zm6.643.836a2.556 2.556 0 0 1-1.873-.669 2.738 2.738 0 0 1-.714-1.9l-.011-.446.011-.446a2.7 2.7 0 0 1 .714-1.885 2.531 2.531 0 0 1 1.873-.68 2.917 2.917 0 0 1 1.36.29 2.077 2.077 0 0 1 .825.714 1.7 1.7 0 0 1 .3.848.2.2 0 0 1-.067.178.281.281 0 0 1-.19.067h-.535a.265.265 0 0 1-.168-.045.458.458 0 0 1-.111-.178 1.428 1.428 0 0 0-.535-.758 1.516 1.516 0 0 0-.87-.234 1.45 1.45 0 0 0-1.1.435 1.952 1.952 0 0 0-.435 1.3l-.011.4.011.379a1.969 1.969 0 0 0 .435 1.316 1.446 1.446 0 0 0 1.1.424 1.577 1.577 0 0 0 .87-.223 1.493 1.493 0 0 0 .535-.769.458.458 0 0 1 .111-.178.228.228 0 0 1 .168-.056h.535a.258.258 0 0 1 .19.078.2.2 0 0 1 .067.178 1.75 1.75 0 0 1-.3.847 2.078 2.078 0 0 1-.825.714 2.876 2.876 0 0 1-1.361.302zm4.078-.112a.233.233 0 0 1-.257-.256v-7.4a.241.241 0 0 1 .067-.178.259.259 0 0 1 .19-.078h.557a.267.267 0 0 1 .257.256v2.6a2.167 2.167 0 0 1 .758-.624 2.353 2.353 0 0 1 1.082-.223 2.067 2.067 0 0 1 1.661.691 2.642 2.642 0 0 1 .6 1.818v3.144a.257.257 0 0 1-.078.19.24.24 0 0 1-.178.067h-.557a.233.233 0 0 1-.256-.256V824a1.775 1.775 0 0 0-.39-1.227 1.387 1.387 0 0 0-1.1-.435 1.481 1.481 0 0 0-1.126.446 1.7 1.7 0 0 0-.412 1.215v3.088a.257.257 0 0 1-.078.19.24.24 0 0 1-.178.067zm8.846.112a2.466 2.466 0 0 1-1.84-.7 2.938 2.938 0 0 1-.747-1.94l-.011-.379.011-.368a2.953 2.953 0 0 1 .758-1.918 2.7 2.7 0 0 1 3.735.078 3.114 3.114 0 0 1 .68 2.119v.19a.257.257 0 0 1-.078.189.241.241 0 0 1-.178.067h-3.858v.1a2.11 2.11 0 0 0 .435 1.238 1.332 1.332 0 0 0 1.081.5 1.563 1.563 0 0 0 .836-.2 1.7 1.7 0 0 0 .491-.435.6.6 0 0 1 .145-.156.391.391 0 0 1 .19-.033h.547a.252.252 0 0 1 .167.056.192.192 0 0 1 .067.156.975.975 0 0 1-.312.591 2.51 2.51 0 0 1-.859.6 3.049 3.049 0 0 1-1.26.248zm1.527-3.434v-.033a1.817 1.817 0 0 0-.424-1.249 1.512 1.512 0 0 0-2.23 0 1.883 1.883 0 0 0-.4 1.249v.033z" fill="#51504f" data-name="Path 9" transform="translate(-1460.834 -808.144)"/> |
| <path id="Path_10" d="M1527.2 827.081l-.061.061zm-.056-.279l-.08-.031zm2.766-7.214l.08.031zm1.472 0l-.081.029zm2.754 7.214l.084-.015a.064.064 0 0 0 0-.015zm.022.123h.086v-.015zm-.067.156l.06.061zm-.914.011l-.061.061.006.005zm-.089-.134l.081-.027zm-.613-1.583l.08-.031a.086.086 0 0 0-.08-.055zm-3.657 0v-.086a.086.086 0 0 0-.08.055zm-.613 1.583l-.08-.031zm-.1.134l.055.066zm4.047-2.676v.086a.086.086 0 0 0 .08-.116zm-1.505-3.958l.08-.03a.086.086 0 0 0-.16 0zm-1.505 3.958l-.08-.03a.086.086 0 0 0 .08.116zm-1.784 2.646a.128.128 0 0 1-.1-.042l-.122.121a.3.3 0 0 0 .217.092zm-.1-.042a.129.129 0 0 1-.042-.1h-.171a.3.3 0 0 0 .092.217zm-.042-.1a.38.38 0 0 1 .007-.1l-.163-.054a.514.514 0 0 0-.016.15zm.005-.092l2.765-7.214-.16-.061-2.765 7.214zm2.766-7.216a.283.283 0 0 1 .1-.143.3.3 0 0 1 .174-.046v-.172a.47.47 0 0 0-.271.076.453.453 0 0 0-.166.226zm.276-.189h.758v-.172h-.758zm.758 0a.3.3 0 0 1 .175.046.283.283 0 0 1 .1.143l.161-.059a.451.451 0 0 0-.166-.226.47.47 0 0 0-.272-.076zm.277.19l2.754 7.214.16-.061-2.754-7.214zm2.75 7.2l.022.123.169-.031-.022-.123zm.021.107a.13.13 0 0 1-.042.1l.121.121a.3.3 0 0 0 .092-.217zm-.042.1a.13.13 0 0 1-.1.042v.171a.3.3 0 0 0 .217-.092zm-.1.042h-.568v.171h.568zm-.568 0a.206.206 0 0 1-.135-.036l-.11.132a.373.373 0 0 0 .245.076zm-.129-.031a.262.262 0 0 1-.069-.1l-.162.054a.431.431 0 0 0 .11.167zm-.07-.1l-.613-1.584-.16.062.613 1.583zm-.693-1.638h-3.657v.171h3.657zm-3.737.055l-.614 1.584.16.062.613-1.583zm-.615 1.587a.235.235 0 0 1-.075.1l.111.13a.4.4 0 0 0 .126-.172zm-.074.1a.185.185 0 0 1-.124.036v.171a.354.354 0 0 0 .233-.076zm-.124.036h-.569v.171h.569zm4.306-2.677l-1.505-3.958-.16.061 1.505 3.958zm-1.666-3.958l-1.505 3.958.16.061 1.505-3.958zm-1.425 4.075h3.01v-.171h-3.01zm6.143 4.687l-.06.061zm0-7.761l.061.061zm.881 0l-.065.056.01.009zm.078.669h-.086a.086.086 0 0 0 .155.051zm3.579-.123l-.067.053zm.636 1.9h-.086zm0 .758l-.085-.007zm-.636 1.9l.067.054zm-3.557-.1l.068-.052a.086.086 0 0 0-.154.052zm-.067 2.765l-.061-.06zm2.787-3.323l-.069-.051zm.4-1.293l-.085-.008v.005zm-2.709-1.918l-.068-.052zm-.413 1.2h-.086zm-.011.423h-.085zm.011.435h-.086zm.424 1.149l.066-.054zm-1.216 3.315a.173.173 0 0 1-.129-.053l-.121.121a.342.342 0 0 0 .25.1zm-.129-.053a.157.157 0 0 1-.042-.118h-.172a.325.325 0 0 0 .092.239zm-.042-.118v-7.4h-.172v7.4zm0-7.4a.157.157 0 0 1 .042-.118l-.121-.121a.324.324 0 0 0-.092.239zm.042-.118a.173.173 0 0 1 .129-.053v-.172a.342.342 0 0 0-.25.1zm.129-.053h.513v-.172h-.513zm.513 0a.137.137 0 0 1 .113.048l.13-.111a.309.309 0 0 0-.244-.108zm.123.058a.137.137 0 0 1 .048.113h.171a.309.309 0 0 0-.108-.243zm.048.113v.49h.171v-.49zm.155.541a2.033 2.033 0 0 1 1.759-.823v-.171a2.2 2.2 0 0 0-1.9.894zm1.759-.823a2.007 2.007 0 0 1 1.683.7l.135-.106a2.177 2.177 0 0 0-1.818-.768zm1.683.7a3.045 3.045 0 0 1 .617 1.845l.171-.007a3.218 3.218 0 0 0-.654-1.946zm.617 1.85c.007.078.011.2.011.372h.171c0-.171 0-.3-.012-.387zm.011.372c0 .171 0 .294-.011.372l.17.015c.008-.086.012-.216.012-.387zm-.011.376a3.08 3.08 0 0 1-.617 1.846l.134.106a3.25 3.25 0 0 0 .654-1.945zm-.617 1.845a2.025 2.025 0 0 1-1.683.7v.171a2.2 2.2 0 0 0 1.817-.768zm-1.683.7a2.068 2.068 0 0 1-1.739-.8l-.136.1a2.239 2.239 0 0 0 1.874.87zm-1.892-.75v2.587h.172v-2.587zm0 2.587a.156.156 0 0 1-.042.118l.121.121a.325.325 0 0 0 .092-.239zm-.046.123a.138.138 0 0 1-.114.048v.172a.308.308 0 0 0 .244-.108zm-.114.048h-.546v.172h.546zm1.817-2.739a1.408 1.408 0 0 0 1.218-.526l-.138-.1a1.24 1.24 0 0 1-1.079.455zm1.217-.525a2.355 2.355 0 0 0 .419-1.341l-.171-.007a2.182 2.182 0 0 1-.385 1.246zm.418-1.336c.008-.079.012-.19.012-.332h-.172c0 .14 0 .245-.011.315zm.012-.332a2.726 2.726 0 0 0-.407-1.632 1.448 1.448 0 0 0-1.24-.562v.171a1.278 1.278 0 0 1 1.1.492 2.565 2.565 0 0 1 .374 1.53zm-1.647-2.193a1.452 1.452 0 0 0-1.228.547l.136.1a1.282 1.282 0 0 1 1.091-.479zm-1.228.547a2.2 2.2 0 0 0-.43 1.252l.172.008a2.028 2.028 0 0 1 .4-1.157zm-.43 1.254l-.011.424h.171l.011-.424zm-.011.428l.011.435h.172l-.011-.435zm.011.436a1.95 1.95 0 0 0 .443 1.2l.133-.109a1.776 1.776 0 0 1-.4-1.1zm.443 1.2a1.484 1.484 0 0 0 1.214.522v-.171a1.314 1.314 0 0 1-1.082-.459zm5.828 1.117l.037-.077zm-.747-.6l-.07.049zm.335-2.063l.052.068zm1.65-.624l.012.085zm1.661-.234l.012.085a.086.086 0 0 0 .074-.085zm-2.107-1.249l.046.072zm-.468.48l-.075-.042a.083.083 0 0 0-.006.015zm-.089.145l-.054-.067-.007.006zm-.792-.022l-.065.056.009.009zm-.067-.178h-.086zm.245-.6l-.07-.049zm.747-.591l.038.077zm3.033.334l-.063.058zm.468 5.252l.06.061zm-.881 0l-.065.056a.043.043 0 0 0 .009.009zm-.067-.669h.086a.086.086 0 0 0-.156-.048zm-.714.6l-.04-.076zm.223-1.059l-.062-.06zm.468-1.684h.086a.086.086 0 0 0-.1-.085zm-1.293.189l.012.085zm-1.193.379l.046.072zm-.033 1.349l-.047.071zm.635.985a2.241 2.241 0 0 1-.978-.215l-.074.155a2.412 2.412 0 0 0 1.051.231zm-.978-.215a1.859 1.859 0 0 1-.715-.576l-.138.1a2.024 2.024 0 0 0 .779.629zm-.713-.573a1.4 1.4 0 0 1-.253-.81h-.172a1.571 1.571 0 0 0 .283.907zm-.253-.81a1.374 1.374 0 0 1 .569-1.136l-.105-.135a1.544 1.544 0 0 0-.635 1.272zm.569-1.137a3.316 3.316 0 0 1 1.609-.607l-.024-.17a3.481 3.481 0 0 0-1.691.642zm1.609-.607l1.661-.234-.024-.17-1.662.234zm1.735-.319v-.323h-.171v.323zm0-.323a1.156 1.156 0 0 0-.355-.917 1.536 1.536 0 0 0-1.035-.306v.172a1.37 1.37 0 0 1 .922.263.986.986 0 0 1 .3.788zm-1.39-1.223a1.486 1.486 0 0 0-.851.227l.1.142a1.316 1.316 0 0 1 .755-.2zm-.849.226a1.452 1.452 0 0 0-.5.51l.15.084a1.286 1.286 0 0 1 .44-.449zm-.5.524a.226.226 0 0 1-.062.105l.107.134a.391.391 0 0 0 .117-.185zm-.068.112a.1.1 0 0 1-.073.019v.171a.266.266 0 0 0 .194-.07zm-.073.019h-.48v.171h.48zm-.48 0a.18.18 0 0 1-.122-.046l-.112.13a.352.352 0 0 0 .234.087zm-.113-.037a.18.18 0 0 1-.047-.123h-.171a.352.352 0 0 0 .087.234zm-.047-.119a1.1 1.1 0 0 1 .23-.557l-.14-.1a1.253 1.253 0 0 0-.261.648zm.23-.556a1.843 1.843 0 0 1 .715-.564l-.075-.154a2.018 2.018 0 0 0-.78.618zm.716-.564a2.611 2.611 0 0 1 1.2-.247v-.171a2.781 2.781 0 0 0-1.277.266zm1.2-.247a2.268 2.268 0 0 1 1.732.563l.126-.116a2.435 2.435 0 0 0-1.858-.618zm1.733.564a1.945 1.945 0 0 1 .523 1.391h.171a2.117 2.117 0 0 0-.57-1.508zm.523 1.391v3.613h.171v-3.613zm0 3.613a.172.172 0 0 1-.053.129l.121.121a.344.344 0 0 0 .1-.25zm-.053.129a.157.157 0 0 1-.118.042v.171a.326.326 0 0 0 .239-.092zm-.118.042h-.513v.171h.513zm-.513 0a.2.2 0 0 1-.134-.046l-.111.13a.367.367 0 0 0 .245.088zm-.124-.037a.194.194 0 0 1-.047-.134h-.171a.366.366 0 0 0 .087.245zm-.047-.134v-.479h-.171v.479zm-.156-.528a1.846 1.846 0 0 1-.683.575l.079.152a2.012 2.012 0 0 0 .745-.629zm-.683.575a2.476 2.476 0 0 1-1.153.236v.171a2.644 2.644 0 0 0 1.233-.255zm-.919-.429a1.666 1.666 0 0 0 1.244-.494l-.123-.12a1.493 1.493 0 0 1-1.121.442zm1.244-.494a1.969 1.969 0 0 0 .492-1.431h-.171a1.8 1.8 0 0 1-.444 1.312zm.492-1.431v-.312h-.171v.312zm-.1-.4l-1.293.189.025.17 1.293-.189zm-1.293.189a3 3 0 0 0-1.228.393l.095.143a2.837 2.837 0 0 1 1.158-.365zm-1.227.392a.845.845 0 0 0-.441.73h.172a.676.676 0 0 1 .362-.586zm-.441.73a.869.869 0 0 0 .406.762l.095-.142a.7.7 0 0 1-.33-.62zm.408.764a1.673 1.673 0 0 0 .916.247v-.171a1.5 1.5 0 0 1-.823-.221zm5.686.329l-.061.06zm-.714-1.9h-.085zm-.011-.446h-.085zm.011-.446h-.085zm.714-1.885l.061.061zm3.234-.39l-.04.076zm.825.713l-.073.046zm.3.848h-.086zm-.067.178l.056.065zm-.892.022l.054-.067zm-.112-.178l-.081.029zm-.535-.758l-.048.071zm-1.974.2l-.062-.059zm-.435 1.3h-.086zm-.011.4h-.086zm.011.379h-.086zm.435 1.316l-.062.059zm1.974.2l.046.072zm.535-.769l-.079-.033zm.112-.178l.054.067.007-.006zm.892.022l-.061.06zm.067.178l-.085-.009zm-.3.847l-.072-.046zm-.825.714l-.04-.076zm-1.36.2a2.471 2.471 0 0 1-1.814-.644l-.12.122a2.64 2.64 0 0 0 1.933.694zm-1.813-.643a2.653 2.653 0 0 1-.689-1.839l-.171.006a2.822 2.822 0 0 0 .738 1.952zm-.689-1.838l-.011-.446h-.171l.011.446zm-.011-.442l.011-.446h-.171l-.011.446zm.011-.445a2.611 2.611 0 0 1 .689-1.827l-.122-.121a2.78 2.78 0 0 0-.738 1.942zm.689-1.827a2.447 2.447 0 0 1 1.813-.655v-.171a2.617 2.617 0 0 0-1.934.705zm1.813-.655a2.836 2.836 0 0 1 1.32.28l.079-.152a3 3 0 0 0-1.4-.3zm1.32.28a1.99 1.99 0 0 1 .792.683l.145-.091a2.158 2.158 0 0 0-.858-.744zm.793.685a1.617 1.617 0 0 1 .287.8l.171-.009a1.789 1.789 0 0 0-.315-.89zm.287.809a.11.11 0 0 1-.037.1l.112.13a.281.281 0 0 0 .1-.252zm-.037.1a.2.2 0 0 1-.134.046v.171a.369.369 0 0 0 .246-.088zm-.134.046h-.535v.171h.535zm-.535 0a.184.184 0 0 1-.114-.026l-.107.134a.345.345 0 0 0 .221.064zm-.114-.026a.389.389 0 0 1-.086-.144l-.158.066a.533.533 0 0 0 .137.212zm-.084-.14a1.514 1.514 0 0 0-.57-.8l-.093.144a1.343 1.343 0 0 1 .5.715zm-.568-.8a1.6 1.6 0 0 0-.918-.249v.171a1.435 1.435 0 0 1 .822.219zm-.918-.249a1.535 1.535 0 0 0-1.166.462l.124.118a1.364 1.364 0 0 1 1.042-.408zm-1.166.462a2.036 2.036 0 0 0-.458 1.36l.171.006a1.872 1.872 0 0 1 .411-1.249zm-.458 1.361l-.011.4h.171l.011-.4zm-.011.406l.011.379.171-.005-.011-.379zm.011.38a2.052 2.052 0 0 0 .458 1.371l.124-.118a1.889 1.889 0 0 1-.411-1.26zm.458 1.371a1.533 1.533 0 0 0 1.166.451v-.172a1.363 1.363 0 0 1-1.042-.4zm1.166.451a1.661 1.661 0 0 0 .916-.237l-.093-.144a1.491 1.491 0 0 1-.823.209zm.918-.238a1.576 1.576 0 0 0 .568-.812l-.162-.057a1.409 1.409 0 0 1-.5.727zm.566-.807a.39.39 0 0 1 .086-.144l-.107-.134a.533.533 0 0 0-.137.213zm.093-.151a.144.144 0 0 1 .107-.031v-.171a.31.31 0 0 0-.228.081zm.107-.031h.535v-.171h-.535zm.535 0a.173.173 0 0 1 .129.053l.121-.121a.344.344 0 0 0-.25-.1zm.134.057a.11.11 0 0 1 .037.1l.17.017a.281.281 0 0 0-.1-.252zm.037.109a1.664 1.664 0 0 1-.288.806l.144.092a1.839 1.839 0 0 0 .315-.889zm-.288.806a1.989 1.989 0 0 1-.792.683l.079.152a2.162 2.162 0 0 0 .858-.744zm-.793.684a2.8 2.8 0 0 1-1.32.28v.171a2.96 2.96 0 0 0 1.4-.3zm2.568.187l-.065.056.01.009zm0-7.772l.061.06zm.926 0l-.065.056.009.009zm.078 2.776h-.085a.086.086 0 0 0 .153.053zm.758-.624l.038.077zm2.743.468l-.065.056zm.524 5.151l-.061-.06zm-.925 0l-.065.056.009.009zm-.457-4.5l-.065.056zm-2.23.011l-.062-.059zm-.49 4.493l-.061-.06zm-.736-.019a.2.2 0 0 1-.134-.046l-.112.13a.367.367 0 0 0 .245.088zm-.124-.037a.2.2 0 0 1-.046-.134h-.172a.367.367 0 0 0 .087.245zm-.046-.134v-7.4h-.172v7.4zm0-7.4a.156.156 0 0 1 .042-.118l-.121-.121a.326.326 0 0 0-.092.239zm.042-.118a.172.172 0 0 1 .129-.053v-.171a.343.343 0 0 0-.25.1zm.129-.053h.557v-.171h-.557zm.557 0a.137.137 0 0 1 .113.048l.13-.112a.308.308 0 0 0-.244-.108zm.122.057a.137.137 0 0 1 .048.113h.172a.309.309 0 0 0-.108-.243zm.048.113v2.6h.172v-2.6zm.153 2.651a2.076 2.076 0 0 1 .728-.6l-.075-.154a2.248 2.248 0 0 0-.788.649zm.73-.6a2.272 2.272 0 0 1 1.043-.214v-.171a2.438 2.438 0 0 0-1.121.232zm1.043-.214a1.982 1.982 0 0 1 1.6.661l.13-.112a2.152 2.152 0 0 0-1.727-.721zm1.6.663a2.557 2.557 0 0 1 .581 1.761h.171a2.727 2.727 0 0 0-.624-1.874zm.581 1.761v3.144h.171v-3.144zm0 3.144a.173.173 0 0 1-.053.129l.121.121a.345.345 0 0 0 .1-.25zm-.053.129a.156.156 0 0 1-.118.042v.171a.327.327 0 0 0 .239-.092zm-.118.042h-.557v.171h.557zm-.557 0a.2.2 0 0 1-.134-.046l-.112.13a.367.367 0 0 0 .245.088zm-.124-.037a.2.2 0 0 1-.046-.134h-.172a.368.368 0 0 0 .087.245zm-.046-.134V823.8h-.172v3.088zm0-3.088a1.859 1.859 0 0 0-.412-1.284l-.128.114a1.69 1.69 0 0 1 .368 1.169zm-.411-1.283a1.471 1.471 0 0 0-1.169-.464v.171a1.3 1.3 0 0 1 1.039.406zm-1.169-.464a1.566 1.566 0 0 0-1.188.473l.124.118a1.4 1.4 0 0 1 1.064-.419zm-1.188.473a1.779 1.779 0 0 0-.436 1.275h.172a1.609 1.609 0 0 1 .389-1.156zm-.436 1.275v3.088h.172V823.8zm0 3.088a.172.172 0 0 1-.053.129l.121.121a.344.344 0 0 0 .1-.25zm-.053.129a.156.156 0 0 1-.118.042v.171a.327.327 0 0 0 .239-.092zm-.118.042h-.557v.171h.557zm6.449-.505l-.062.059zm-.747-1.94h-.086zm-.012-.379h-.085v.005zm.012-.368l-.086-.006zm.758-1.918l-.061-.06zm3.735.078l-.065.056zm.6 2.5l.061.061zm-4.036.067v-.086a.086.086 0 0 0-.086.086zm0 .1h-.086zm.435 1.238l-.068.053zm1.918.3l.045.073zm.491-.435l-.069-.051zm.145-.156l.039.077h.006zm.9.022l-.055.066zm-.245.747l-.064-.057zm-.858.6l.035.078zm.267-3.189v.086a.086.086 0 0 0 .086-.086zm-.424-1.282l-.066.055zm-2.23 0l-.065-.055zm-.4 1.282h-.086a.086.086 0 0 0 .086.086zm1.528 3.349a2.38 2.38 0 0 1-1.779-.677l-.122.12a2.55 2.55 0 0 0 1.9.728zm-1.778-.676a2.86 2.86 0 0 1-.724-1.886l-.171.009a3.027 3.027 0 0 0 .771 1.995zm-.723-1.884l-.011-.379h-.171l.011.379zm-.011-.374l.011-.368-.172-.005-.011.368zm.011-.365a2.871 2.871 0 0 1 .735-1.864l-.124-.118a3.042 3.042 0 0 0-.782 1.971zm.734-1.864a2.331 2.331 0 0 1 1.756-.687v-.171a2.5 2.5 0 0 0-1.879.74zm1.756-.687a2.307 2.307 0 0 1 1.853.762l.13-.112a2.477 2.477 0 0 0-1.983-.821zm1.854.762a3.03 3.03 0 0 1 .659 2.062h.172a3.2 3.2 0 0 0-.7-2.175zm.659 2.062v.19h.172v-.19zm0 .19a.172.172 0 0 1-.053.129l.121.121a.345.345 0 0 0 .1-.25zm-.053.129a.156.156 0 0 1-.118.042v.171a.327.327 0 0 0 .239-.092zm-.118.042h-3.858v.171h3.858zm-3.944.086v.1h.172v-.1zm0 .1a2.2 2.2 0 0 0 .453 1.287l.135-.106a2.027 2.027 0 0 1-.417-1.189zm.454 1.288a1.418 1.418 0 0 0 1.148.533v-.171a1.247 1.247 0 0 1-1.015-.47zm1.148.533a1.647 1.647 0 0 0 .882-.214l-.09-.146a1.481 1.481 0 0 1-.791.188zm.882-.214a1.777 1.777 0 0 0 .515-.458l-.14-.1a1.613 1.613 0 0 1-.466.412zm.513-.456a1.251 1.251 0 0 1 .081-.1.28.28 0 0 1 .026-.025l.008-.006-.077-.153a.326.326 0 0 0-.083.068 1.55 1.55 0 0 0-.092.113zm.12-.134a.328.328 0 0 1 .146-.021v-.171a.468.468 0 0 0-.234.046zm.146-.021h.547v-.171h-.547zm.547 0a.166.166 0 0 1 .112.036l.11-.132a.337.337 0 0 0-.222-.076zm.112.036a.107.107 0 0 1 .036.09h.171a.277.277 0 0 0-.1-.222zm.036.09a.9.9 0 0 1-.291.534l.128.115a1.055 1.055 0 0 0 .334-.649zm-.291.535a2.42 2.42 0 0 1-.83.581l.072.156a2.6 2.6 0 0 0 .888-.624zm-.829.58a2.964 2.964 0 0 1-1.224.238v.171a3.133 3.133 0 0 0 1.295-.253zm.389-3.111v-.033h-.171v.033zm0-.033a1.9 1.9 0 0 0-.445-1.306l-.129.114a1.731 1.731 0 0 1 .4 1.192zm-.444-1.3a1.466 1.466 0 0 0-1.181-.521v.172a1.3 1.3 0 0 1 1.049.46zm-1.181-.521a1.466 1.466 0 0 0-1.18.521l.131.11a1.3 1.3 0 0 1 1.049-.46zm-1.181.521a1.965 1.965 0 0 0-.422 1.3h.172a1.794 1.794 0 0 1 .382-1.194zm-.422 1.3v.033h.172v-.033zm.086.119h3.055v-.171h-3.055z" fill="#51504f" data-name="Path 10" transform="translate(-1460.636 -807.945)"/> |
| <path id="Path_11" d="M1519.066 884.011a.581.581 0 0 1-.567-.567 1.151 1.151 0 0 1 .028-.312l7.026-18.328a.881.881 0 0 1 .906-.623h1.926a.882.882 0 0 1 .907.623l7 18.328.057.312a.583.583 0 0 1-.567.567h-1.445a.735.735 0 0 1-.482-.142.9.9 0 0 1-.226-.34l-1.558-4.023h-9.292l-1.558 4.023a.8.8 0 0 1-.255.34.688.688 0 0 1-.453.142zm12.181-6.94l-3.824-10.056-3.823 10.055zm8.184-10.538a.592.592 0 0 1-.652-.651v-1.53a.714.714 0 0 1 .17-.482.656.656 0 0 1 .482-.2h1.785a.677.677 0 0 1 .68.68v1.53a.655.655 0 0 1-.2.481.713.713 0 0 1-.481.17zm.227 17.479a.593.593 0 0 1-.652-.652v-13.428a.611.611 0 0 1 .17-.453.656.656 0 0 1 .482-.2h1.359a.679.679 0 0 1 .652.651v13.427a.655.655 0 0 1-.2.482.613.613 0 0 1-.453.17zm6.861 0a.592.592 0 0 1-.651-.652v-13.4a.715.715 0 0 1 .17-.481.656.656 0 0 1 .482-.2h1.3a.677.677 0 0 1 .68.68v1.246a4.255 4.255 0 0 1 3.966-1.926h1.1a.679.679 0 0 1 .651.651v1.161a.566.566 0 0 1-.2.453.612.612 0 0 1-.453.17h-1.7a3.2 3.2 0 0 0-2.408.907 3.253 3.253 0 0 0-.879 2.408v8.328a.656.656 0 0 1-.2.482.716.716 0 0 1-.482.17zm12.234 0a.593.593 0 0 1-.651-.652v-11.814h-2.408a.592.592 0 0 1-.651-.651v-.963a.611.611 0 0 1 .17-.453.654.654 0 0 1 .481-.2h2.408v-1.417q0-4.816 4.872-4.815h1.586a.679.679 0 0 1 .652.651v.963a.656.656 0 0 1-.2.481.613.613 0 0 1-.453.17h-1.529a2.1 2.1 0 0 0-1.785.68 3.248 3.248 0 0 0-.51 2.011v1.275h6.062V863.7a.613.613 0 0 1 .17-.453.656.656 0 0 1 .482-.2h1.3a.679.679 0 0 1 .652.651v19.659a.655.655 0 0 1-.2.482.613.613 0 0 1-.454.17h-1.3a.592.592 0 0 1-.652-.652v-11.811h-6.062v11.813a.657.657 0 0 1-.2.482.614.614 0 0 1-.454.17zm20.9.283a6.487 6.487 0 0 1-4.844-1.757 6.837 6.837 0 0 1-1.813-4.674l-.029-1.218.029-1.218a6.732 6.732 0 0 1 1.841-4.646 7.389 7.389 0 0 1 9.631 0 6.736 6.736 0 0 1 1.841 4.646q.028.311.028 1.218t-.028 1.218a6.772 6.772 0 0 1-1.841 4.674 6.391 6.391 0 0 1-4.82 1.756zm0-2.181a3.582 3.582 0 0 0 2.8-1.133 4.931 4.931 0 0 0 1.133-3.258q.028-.283.028-1.076t-.028-1.076a4.931 4.931 0 0 0-1.133-3.258 3.582 3.582 0 0 0-2.8-1.133 3.671 3.671 0 0 0-2.833 1.133 4.83 4.83 0 0 0-1.1 3.258l-.028 1.076.028 1.076a4.83 4.83 0 0 0 1.1 3.258 3.671 3.671 0 0 0 2.828 1.132zm13.755 1.9a.846.846 0 0 1-.566-.17 1.321 1.321 0 0 1-.34-.538l-4.023-13.144-.056-.283a.575.575 0 0 1 .17-.425.641.641 0 0 1 .425-.17h1.246a.612.612 0 0 1 .453.17.646.646 0 0 1 .255.312l3.145 10.679 3.371-10.566a.761.761 0 0 1 .255-.4.726.726 0 0 1 .538-.2h.963a.728.728 0 0 1 .539.2.76.76 0 0 1 .255.4l3.371 10.566 3.144-10.679a.655.655 0 0 1 .2-.312.714.714 0 0 1 .482-.17h1.275a.542.542 0 0 1 .4.17.576.576 0 0 1 .17.425l-.057.283-3.994 13.144a1.323 1.323 0 0 1-.34.538.9.9 0 0 1-.6.17h-1.1a.86.86 0 0 1-.935-.708l-3.286-10.141-3.286 10.141a.928.928 0 0 1-.963.708z" fill="#51504f" data-name="Path 11" transform="translate(-1454.66 -838.62)"/> |
| </g> |
| </g> |
| </g> |
| </svg> |
| |
| </a> |
| </div> |
| <div class="desktop-only navbar__menu-container"> |
| |
| <div class="navbar__menu-content" id="main_navbar"> |
| |
| |
| <div class="navbar__links-container"> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/community/" |
| > |
| Community |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/meetups/" |
| > |
| Meetups |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/docs/" |
| > |
| Documentation |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link active" |
| href="/use-cases/" |
| > |
| Use Cases |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/announcements/" |
| > |
| Announcements |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/blog/" |
| > |
| Blog |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/ecosystem/" |
| > |
| Ecosystem |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/survey/" |
| > |
| Airflow Survey 2023 |
| </a> |
| |
| </div> |
| </div> |
| |
| </div> |
| <div class="no-desktop navbar__drawer-container"> |
| <button class="navbar__toggle-button" id="navbar-toggle-button"> |
| |
| <div id="hamburger-icon" class="navbar__toggle-button--icon visible"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="26" height="20" viewBox="0 0 26 20"> |
| <g id="Group_1294" data-name="Group 1294" transform="translate(-38.791 291)"> |
| <g id="Group_1291" data-name="Group 1291" transform="translate(39 -291)"> |
| <rect id="Rectangle_461" width="26" height="2" fill="#51504f" data-name="Rectangle 461" rx="1" transform="translate(-.209)"/> |
| </g> |
| <g id="Group_1292" data-name="Group 1292" transform="translate(39 -281.822)"> |
| <rect id="Rectangle_462" width="26" height="2" fill="#51504f" data-name="Rectangle 462" rx="1" transform="translate(-.209 -.178)"/> |
| </g> |
| <g id="Group_1293" data-name="Group 1293" transform="translate(39 -272.644)"> |
| <rect id="Rectangle_463" width="26" height="2" fill="#51504f" data-name="Rectangle 463" rx="1" transform="translate(-.209 -.356)"/> |
| </g> |
| </g> |
| </svg> |
| |
| </div> |
| |
| |
| <div id="close-icon" class="navbar__toggle-button--icon"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="19.799" height="19.799" viewBox="0 0 19.799 19.799"> |
| <g id="Group_1574" data-name="Group 1574" transform="translate(-41.892 290.899)"> |
| <g id="Group_1291" data-name="Group 1291" transform="rotate(-45 -308.114 -187.077)"> |
| <rect id="Rectangle_461" width="26" height="2" fill="#51504f" data-name="Rectangle 461" rx="1" transform="translate(-.209)"/> |
| </g> |
| <g id="Group_1292" data-name="Group 1292" transform="rotate(45 372.48 -93.011)"> |
| <rect id="Rectangle_462" width="26" height="2" fill="#51504f" data-name="Rectangle 462" rx="1" transform="translate(-.209 -.178)"/> |
| </g> |
| </g> |
| </svg> |
| |
| </div> |
| |
| </button> |
| <div class="navbar__drawer" id="navbar-drawer"> |
| <div class="navbar__menu-content" id="main_navbar"> |
| |
| |
| <div class="navbar__links-container"> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/community/" |
| > |
| Community |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/meetups/" |
| > |
| Meetups |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/docs/" |
| > |
| Documentation |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link active" |
| href="/use-cases/" |
| > |
| Use Cases |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/announcements/" |
| > |
| Announcements |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/blog/" |
| > |
| Blog |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/ecosystem/" |
| > |
| Ecosystem |
| </a> |
| |
| |
| |
| |
| |
| |
| |
| <a class="navbar__text-link " |
| href="/survey/" |
| > |
| Airflow Survey 2023 |
| </a> |
| |
| </div> |
| </div> |
| </div> |
| </div> |
| </nav> |
| |
| |
| |
| </header> |
| <div class="container-fluid td-default"> |
| <div class="base-layout"> |
| <main role="main" class="td-main container"> |
| |
| <div class="case-study-page no-width-restriction"> |
| |
| |
| <div class="td-content"> |
| |
| |
| |
| |
| <div class="quote"> |
| <p class="quote--text">Airflow helped us to define and organize our ML pipeline dependencies, and empowered us to introduce new, diverse batch processes at increasing scale.</p> |
| <p class="quote--author">Handong Park</p> |
| |
| <img src="/usecase-logos/sift_logo.png" alt="Sift logo" class="quote--logo" /> |
| |
| </div> |
| |
| |
| |
| <div class="markdown-content"> |
| <h5 id="what-was-the-problem">What was the problem?</h5> |
| <p>At Sift, we’re constantly training machine learning models that feed into the core of Sift’s Digital Trust & Safety platform. The platform gives our customers a way to discern suspicious online behavior from trustworthy behavior, allowing our customers to protect their online transactions, maintain the integrity of their content platforms, and keep their users’ accounts secure. To make this possible, we’ve built model training pipelines that consist of hundreds of steps in MapReduce and Spark, with complex requirements between them.</p> |
| <p>When we built these workflows, we found that we needed a centralized way to organize the interactions between the many steps in each workflow. But before Airflow, we didn’t have an easy way to express those dependencies. And as we added steps to the workflows, it became increasingly difficult to coordinate their dependencies and keep ML experiments in sync.</p> |
| <p>It soon became clear that we needed a way to orchestrate both the scheduled execution of our jobs and the dependencies between steps of not only a single workflow, but of multiple workflows. We needed a way to dynamically create several experimental ML workflows at once that could each have their own code, dependencies, and tasks. Additionally, we needed a way to be able to monitor the status of tasks, and re-run or restart tasks from any given point in a workflow with ease.</p> |
| <h5 id="how-did-apache-airflow-help-to-solve-this-problem">How did Apache Airflow help to solve this problem?</h5> |
| <p>Airflow makes it easy to clearly define the interactions between various jobs, expanding the scope of what we can do in our model training pipelines. We now have the ability to schedule and coordinate all jobs while managing the dependencies between them using DAGs. Each of our main workflows, including our model training pipeline and ETL pipelines, has its own DAG code that manages its tasks’ dependencies and the execution schedule for the pipeline. We even define dependencies between separate DAGs by using Airflow’s ExternalTaskSensor. This allows our DAGs to actually depend on each other and keep each one of them focused and compact in its scope.</p> |
| <p>As part of our custom Airflow setup, we’ve built out a separate Airflow ecosystem for short-lived experimental DAGs as well, so that we can test changes to our jobs or run separate model training pipelines in isolation. Using deployment scripts that edit our DAGs when we upload them to Airflow, the same code that powers an existing DAG can be deployed in a separate, isolated environment with experimental edits. This means that each experiment can have its own isolated code, running in parallel with other pipelines, without accidentally touching others’ jobs or dependencies.</p> |
| <p>Finally, Airflow has given us the ability to manage our tasks’ successes and failures through its user interface. Airflow allows us to track our tasks’ failures, duration, history, and logs in one central UI, and that same UI also allows us to easily retry single tasks, branches of a DAG, or entire DAGs.</p> |
| <h5 id="what-are-the-results">What are the results?</h5> |
| <p>Airflow initially gave us a way to solve our existing problems: we used Airflow to replace rigid crons with well-defined DAG dependencies, to build isolated ML experiments using short-lived DAGs, and to track our pipelines’ successes and failures.</p> |
| <p>But even after that, Airflow helped us to grow beyond those initial challenges, and expanded the scope of what we could feasibly tackle. Airflow not only made it easier to manage our ever-expanding ML pipelines, but also allowed us to create entirely new pipelines, ranging from workflows that back up our production data to complex ETL pipelines that transform data into experimentation-ready formats.</p> |
| <p>Airflow also allowed us to support a more diverse toolset. Shell scripts, Java, Python, Jupyter notebooks, and more - all of these can be managed from an Airflow DAG, allowing developers to utilize our data to test new ideas, generate insights, and improve our models with ease.</p> |
| |
| </div> |
| </div> |
| |
| |
| |
| <div class="pager"> |
| <a href="/use-cases/snapp/"> |
| |
| |
| <button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Previous</button> |
| |
| </a> |
| <a href="/use-cases/seniorlink/"> |
| |
| |
| <button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" >Next</button> |
| |
| </a> |
| </div> |
| |
| </div> |
| |
| </main> |
| |
| |
| <div class="base-layout--button"> |
| |
| <a href=https://github.com/apache/airflow-site/edit/main/landing-pages/site/content/en/use-cases/sift.md> |
| |
| |
| |
| <button class="btn-hollow btn-brown btn-with-icon with-box-shadow button-fixed"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="30.76" height="30"> |
| <path d="M15.379 0a15.381 15.381 0 00-4.86 29.974c.769.141 1.05-.334 1.05-.741 0-.365-.013-1.332-.021-2.616-4.278.929-5.181-2.062-5.181-2.062a4.071 4.071 0 00-1.708-2.25c-1.4-.954.106-.935.106-.935a3.231 3.231 0 012.356 1.585 3.274 3.274 0 004.476 1.278 3.287 3.287 0 01.976-2.056c-3.415-.388-7.005-1.707-7.005-7.6a5.947 5.947 0 011.583-4.127 5.53 5.53 0 01.151-4.07s1.291-.413 4.229 1.577a14.577 14.577 0 017.7 0c2.936-1.99 4.225-1.577 4.225-1.577a5.523 5.523 0 01.153 4.07 5.937 5.937 0 011.581 4.127c0 5.909-3.6 7.209-7.022 7.589a3.672 3.672 0 011.044 2.848c0 2.056-.019 3.715-.019 4.219 0 .411.277.89 1.057.74A15.382 15.382 0 0015.378.001z" |
| data-name="Path 224" fill="#51504f" fill-rule="evenodd"/> |
| </svg> |
| |
| <span class="bodytext__medium--brownish-grey">Suggest a change on this page</span> |
| </button> |
| |
| </a> |
| </div> |
| |
| </div> |
| </div> |
| |
| |
| |
| <footer> |
| <div class="footer-section footer-section__media-section"> |
| <div class="d-flex align-items-center"> |
| |
| |
| |
| <a class="footer-section__media-section--link" target="_blank" href="https://github.com/apache/airflow/"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="46.14" height="45" viewBox="0 0 46.14 45"> |
| <path id="Path_207" d="M228.962 1078.578a23.072 23.072 0 0 0-7.29 44.96c1.154.212 1.574-.5 1.574-1.112 0-.548-.02-2-.031-3.924-6.417 1.394-7.771-3.093-7.771-3.093a6.109 6.109 0 0 0-2.562-3.375c-2.095-1.431.159-1.4.159-1.4a4.846 4.846 0 0 1 3.533 2.377c2.058 3.525 5.4 2.507 6.714 1.917a4.926 4.926 0 0 1 1.464-3.084c-5.123-.582-10.508-2.562-10.508-11.4a8.919 8.919 0 0 1 2.374-6.191 8.3 8.3 0 0 1 .226-6.105s1.937-.62 6.344 2.365a21.857 21.857 0 0 1 11.551 0c4.4-2.985 6.338-2.365 6.338-2.365a8.284 8.284 0 0 1 .23 6.105 8.9 8.9 0 0 1 2.371 6.191c0 8.862-5.393 10.812-10.533 11.384a5.506 5.506 0 0 1 1.566 4.272c0 3.084-.028 5.572-.028 6.329 0 .617.415 1.334 1.586 1.109a23.073 23.073 0 0 0-7.308-44.958z" fill="#fff" fill-rule="evenodd" data-name="Path 207" transform="translate(-205.894 -1078.578)"/> |
| </svg> |
| |
| </a> |
| |
| <a class="footer-section__media-section--link" target="_blank" href="https://apache-airflow.slack.com/"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="45.073" height="45.073" viewBox="0 0 45.073 45.073"> |
| <g id="Group_208" data-name="Group 208" transform="translate(-661.145 -806.287)"> |
| <g id="Group_204" data-name="Group 204" transform="translate(661.145 830.01)"> |
| <path id="Path_208" d="M670.634 856.859a4.744 4.744 0 1 1-4.744-4.744h4.744z" fill="#fff" data-name="Path 208" transform="translate(-661.145 -852.115)"/> |
| <path id="Path_209" d="M684.059 856.859a4.744 4.744 0 0 1 9.489 0v11.861a4.744 4.744 0 1 1-9.489 0z" fill="#fff" data-name="Path 209" transform="translate(-672.198 -852.115)"/> |
| </g> |
| <g id="Group_205" data-name="Group 205" transform="translate(661.145 806.287)"> |
| <path id="Path_210" d="M688.8 815.776a4.744 4.744 0 1 1 4.744-4.745v4.745z" fill="#fff" data-name="Path 210" transform="translate(-672.198 -806.287)"/> |
| <path id="Path_211" d="M677.751 829.2a4.744 4.744 0 0 1 0 9.489H665.89a4.744 4.744 0 1 1 0-9.489z" fill="#fff" data-name="Path 211" transform="translate(-661.145 -817.34)"/> |
| </g> |
| <g id="Group_206" data-name="Group 206" transform="translate(684.868 806.287)"> |
| <path id="Path_212" d="M729.887 833.945a4.744 4.744 0 1 1 4.745 4.745h-4.745z" fill="#fff" data-name="Path 212" transform="translate(-718.026 -817.34)"/> |
| <path id="Path_213" d="M716.462 822.893a4.744 4.744 0 1 1-9.489 0v-11.862a4.744 4.744 0 0 1 9.489 0z" fill="#fff" data-name="Path 213" transform="translate(-706.973 -806.287)"/> |
| </g> |
| <g id="Group_207" data-name="Group 207" transform="translate(684.868 830.01)"> |
| <path id="Path_214" d="M711.718 875.029a4.744 4.744 0 1 1-4.745 4.744v-4.744z" fill="#fff" data-name="Path 214" transform="translate(-706.973 -863.168)"/> |
| <path id="Path_215" d="M711.718 861.6a4.744 4.744 0 1 1 0-9.489h11.861a4.744 4.744 0 0 1 0 9.489z" fill="#fff" data-name="Path 215" transform="translate(-706.973 -852.115)"/> |
| </g> |
| </g> |
| </svg> |
| |
| </a> |
| |
| <a class="footer-section__media-section--link" target="_blank" href="https://stackoverflow.com/questions/tagged/airflow/"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="37.647" height="44.6" viewBox="0 0 37.647 44.6"> |
| <g id="Group_209" data-name="Group 209" transform="translate(-645.2 -975.455)"> |
| <path id="Path_216" d="M677.028 1043.1v-11.948h3.966v15.914H645.2v-15.914h3.966v11.948z" fill="#fff" data-name="Path 216" transform="translate(0 -27.014)"/> |
| <path id="Path_217" d="M661.012 1003.008l19.467 4.069.824-3.914-19.467-4.069zm2.575-9.27l18.025 8.395 1.648-3.605-18.025-8.446zm5-8.858l15.3 12.721 2.524-3.039-15.3-12.721zm9.888-9.425l-3.193 2.369 11.845 15.965 3.193-2.369zm-17.875 36.617h19.879v-3.966H660.6z" fill="#fff" data-name="Path 217" transform="translate(-7.469)"/> |
| </g> |
| </svg> |
| |
| </a> |
| |
| <a class="footer-section__media-section--link" target="_blank" href="https://twitter.com/apacheairflow/"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="44.355" height="44.355" viewBox="0 0 44.355 44.355"> |
| <g id="Group_211" data-name="Group 211" transform="translate(-503 -1382.2)"> |
| <path id="Path_221" d="M541.811 1382.2h-33.267a5.546 5.546 0 0 0-5.544 5.544v33.266a5.547 5.547 0 0 0 5.544 5.545h33.267a5.546 5.546 0 0 0 5.544-5.545v-33.266a5.546 5.546 0 0 0-5.544-5.544zm-5.611 16.533c.011.244.011.488.011.732 0 7.507-5.722 16.178-16.179 16.178a16.173 16.173 0 0 1-8.7-2.539 10.92 10.92 0 0 0 1.353.078 11.4 11.4 0 0 0 7.064-2.44 5.685 5.685 0 0 1-5.311-3.947 5.725 5.725 0 0 0 2.561-.1 5.7 5.7 0 0 1-4.557-5.578v-.078a5.752 5.752 0 0 0 2.572.71 5.709 5.709 0 0 1-1.763-7.6 16.146 16.146 0 0 0 11.721 5.944 6.282 6.282 0 0 1-.144-1.3 5.688 5.688 0 0 1 9.836-3.892 11.3 11.3 0 0 0 3.615-1.375 5.721 5.721 0 0 1-2.506 3.149 11.538 11.538 0 0 0 3.271-.9 11.585 11.585 0 0 1-2.844 2.958z" fill="#fff" data-name="Path 221"/> |
| </g> |
| </svg> |
| |
| </a> |
| |
| <a class="footer-section__media-section--link" target="_blank" href="https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"> |
| <svg xmlns="http://www.w3.org/2000/svg" width="49.594" height="34.941" viewBox="0 0 49.594 34.941"> |
| <path id="Path_223" d="M1124.557 1230a6.232 6.232 0 0 0-4.385-4.413c-3.867-1.043-19.376-1.043-19.376-1.043s-15.508 0-19.376 1.043a6.232 6.232 0 0 0-4.385 4.413c-1.036 3.893-1.036 12.014-1.036 12.014s0 8.122 1.036 12.015a6.232 6.232 0 0 0 4.385 4.413c3.867 1.043 19.376 1.043 19.376 1.043s15.509 0 19.376-1.043a6.232 6.232 0 0 0 4.385-4.413c1.036-3.893 1.036-12.015 1.036-12.015s.001-8.123-1.036-12.014zm-28.833 19.388v-14.748l12.962 7.374z" fill="#fff" data-name="Path 223" transform="translate(-1076 -1224.542)"/> |
| </svg> |
| |
| </a> |
| |
| |
| |
| </div> |
| <div class="footer-section__media-section--button-with-text"> |
| <span class="footer-section__media-section--text">Want to be a part of Apache Airflow?</span> |
| <a href="/community/"> |
| |
| |
| <button id="" class="btn-filled bodytext__medium--white ">Join community</button> |
| |
| </a> |
| </div> |
| </div> |
| <div class="footer-section footer-section__policies-section"> |
| <div class="footer-section"> |
| <span>© The Apache Software Foundation 2024</span> |
| <div class="footer-section__policies-section--policies"> |
| |
| |
| |
| <a href="https://www.apache.org/licenses/" class="footer-section__policies-section--policy-item"> |
| <span>License</span> |
| </a> |
| |
| <a href="https://www.apache.org/foundation/sponsorship.html" class="footer-section__policies-section--policy-item"> |
| <span>Donate</span> |
| </a> |
| |
| <a href="https://www.apache.org/foundation/thanks.html" class="footer-section__policies-section--policy-item"> |
| <span>Thanks</span> |
| </a> |
| |
| <a href="https://www.apache.org/security/" class="footer-section__policies-section--policy-item"> |
| <span>Security</span> |
| </a> |
| |
| <a href="/privacy-notice/" class="footer-section__policies-section--policy-item"> |
| <span>Privacy notice</span> |
| </a> |
| |
| <a href="/code-of-conduct/" class="footer-section__policies-section--policy-item"> |
| <span>Code of Conduct</span> |
| </a> |
| |
| |
| |
| <div class="footer-section__policies-section--language-toggle"> |
| |
| |
| |
| <a class="dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> |
| <span>English</span> |
| </a> |
| <div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink"> |
| |
| </div> |
| |
| </div> |
| </div> |
| </div> |
| <span class="footer-section__policies-section--disclaimer"> |
| Apache Airflow, Apache, Airflow, the Airflow logo, and the Apache feather logo are either registered trademarks or trademarks of |
| <a href="https://apache.org/">The Apache Software Foundation</a>. |
| All other products or name brands are trademarks of their respective holders, including The Apache Software Foundation. |
| </span> |
| </div> |
| </footer> |
| |
| |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script> |
| <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script> |
| |
| |
| |
| |
| |
| |
| <script src="/js/main.min.5c74b870c6953931a705f390a49c7e4c0a842ec5c83b24354758dd674343ed0d.js" integrity="sha256-XHS4cMaVOTGnBfOQpJx+TAqELsXIOyQ1R1jdZ0ND7Q0=" crossorigin="anonymous"></script> |
| |
| |
| |
| </body> |
| |
| |
| <script src="/main.22619.js"></script> |
| |
| </html> |
