| |
| |
| <!-- |
| Javascript to render AIRFLOW-XXX and PR references in text |
| as HTML links. |
| |
| Overrides extrahead block from sphinx_rtd_theme |
| https://www.sphinx-doc.org/en/master/templating.html |
| --> |
| |
| |
| <!DOCTYPE html> |
| <!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> |
| <!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> |
| <head> |
| <meta charset="utf-8"> |
| |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| |
| <title>Configuration Reference — Airflow Documentation</title> |
| |
| |
| |
| |
| <link rel="shortcut icon" href="_static/pin_32.png"/> |
| |
| |
| |
| |
| |
| <script type="text/javascript" src="_static/js/modernizr.min.js"></script> |
| |
| |
| <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script> |
| <script type="text/javascript" src="_static/jquery.js"></script> |
| <script type="text/javascript" src="_static/underscore.js"></script> |
| <script type="text/javascript" src="_static/doctools.js"></script> |
| <script type="text/javascript" src="_static/language_data.js"></script> |
| <script type="text/javascript" src="_static/jira-links.js"></script> |
| |
| <script type="text/javascript" src="_static/js/theme.js"></script> |
| |
| |
| |
| |
| <link rel="stylesheet" href="_static/css/theme.css" type="text/css" /> |
| <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> |
| <link rel="stylesheet" href="_static/graphviz.css" type="text/css" /> |
| <link rel="stylesheet" href="_static/exampleinclude.css" type="text/css" /> |
| <link rel="index" title="Index" href="genindex.html" /> |
| <link rel="search" title="Search" href="search.html" /> |
| <link rel="prev" title="airflow.models.xcom" href="_api/airflow/models/xcom/index.html" /> |
| |
| <script> |
| </script> |
| <style> |
| |
| </style> |
| |
| </head> |
| |
| <body class="wy-body-for-nav"> |
| |
| |
| <div class="wy-grid-for-nav"> |
| |
| <nav data-toggle="wy-nav-shift" class="wy-nav-side"> |
| <div class="wy-side-scroll"> |
| <div class="wy-side-nav-search" > |
| |
| |
| |
| <a href="index.html" class="icon icon-home"> Airflow |
| |
| |
| |
| </a> |
| |
| |
| |
| |
| <div class="version"> |
| 1.10.8 |
| </div> |
| |
| |
| |
| |
| <div role="search"> |
| <form id="rtd-search-form" class="wy-form" action="search.html" method="get"> |
| <input type="text" name="q" placeholder="Search docs" /> |
| <input type="hidden" name="check_keywords" value="yes" /> |
| <input type="hidden" name="area" value="default" /> |
| </form> |
| </div> |
| |
| |
| </div> |
| |
| <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> |
| |
| |
| |
| |
| |
| |
| <ul> |
| <li class="toctree-l1"><a class="reference internal" href="project.html">Project</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="license.html">License</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="start.html">Quick Start</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="tutorial.html">Tutorial</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="howto/index.html">How-to Guides</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="ui.html">UI / Screenshots</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="concepts.html">Concepts</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="profiling.html">Data Profiling</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="cli.html">Command Line Interface Reference</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling & Triggers</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="executor/index.html">Executor</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="timezone.html">Time zones</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="api.html">REST API Reference</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="metrics.html">Metrics</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="errors.html">Error Tracking</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="kubernetes.html">Kubernetes</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="lineage.html">Lineage</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="dag-serialization.html">DAG Serialization</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="changelog.html">Changelog</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="best-practices.html">Best Practices</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="macros.html">Macros reference</a></li> |
| <li class="toctree-l1"><a class="reference internal" href="privacy_notice.html">Privacy Notice</a></li> |
| </ul> |
| <p class="caption"><span class="caption-text">References</span></p> |
| <ul class="current"> |
| <li class="toctree-l1"><a class="reference internal" href="_api/index.html">Python API</a></li> |
| <li class="toctree-l1 current"><a class="current reference internal" href="#">Configurations</a><ul> |
| <li class="toctree-l2"><a class="reference internal" href="#core">core</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#dags-folder">dags_folder</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#base-log-folder">base_log_folder</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#remote-logging">remote_logging</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#remote-log-conn-id">remote_log_conn_id</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#remote-base-log-folder">remote_base_log_folder</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#encrypt-s3-logs">encrypt_s3_logs</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#logging-level">logging_level</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#fab-logging-level">fab_logging_level</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#logging-config-class">logging_config_class</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#colored-console-log">colored_console_log</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#colored-log-format">colored_log_format</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#colored-formatter-class">colored_formatter_class</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#log-format">log_format</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#simple-log-format">simple_log_format</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#log-filename-template">log_filename_template</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#log-processor-filename-template">log_processor_filename_template</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dag-processor-manager-log-location">dag_processor_manager_log_location</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#task-log-reader">task_log_reader</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#hostname-callable">hostname_callable</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#default-timezone">default_timezone</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#executor">executor</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#sql-alchemy-conn">sql_alchemy_conn</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#sql-engine-encoding">sql_engine_encoding</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#sql-alchemy-pool-enabled">sql_alchemy_pool_enabled</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#sql-alchemy-pool-size">sql_alchemy_pool_size</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#sql-alchemy-max-overflow">sql_alchemy_max_overflow</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#sql-alchemy-pool-recycle">sql_alchemy_pool_recycle</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#sql-alchemy-pool-pre-ping">sql_alchemy_pool_pre_ping</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#sql-alchemy-schema">sql_alchemy_schema</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#parallelism">parallelism</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dag-concurrency">dag_concurrency</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dags-are-paused-at-creation">dags_are_paused_at_creation</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#max-active-runs-per-dag">max_active_runs_per_dag</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#load-examples">load_examples</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#plugins-folder">plugins_folder</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#fernet-key">fernet_key</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#donot-pickle">donot_pickle</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dagbag-import-timeout">dagbag_import_timeout</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dag-file-processor-timeout">dag_file_processor_timeout</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#task-runner">task_runner</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#default-impersonation">default_impersonation</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#security">security</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#secure-mode">secure_mode</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#unit-test-mode">unit_test_mode</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#enable-xcom-pickling">enable_xcom_pickling</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#killed-task-cleanup-time">killed_task_cleanup_time</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dag-run-conf-overrides-params">dag_run_conf_overrides_params</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#worker-precheck">worker_precheck</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dag-discovery-safe-mode">dag_discovery_safe_mode</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#default-task-retries">default_task_retries</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#store-serialized-dags">store_serialized_dags</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#min-serialized-dag-update-interval">min_serialized_dag_update_interval</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#check-slas">check_slas</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#cli">cli</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#api-client">api_client</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#endpoint-url">endpoint_url</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#debug">debug</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#fail-fast">fail_fast</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#api">api</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#auth-backend">auth_backend</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#lineage">lineage</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#backend">backend</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#atlas">atlas</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#sasl-enabled">sasl_enabled</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#host">host</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#port">port</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#username">username</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#password">password</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#operators">operators</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#default-owner">default_owner</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#default-cpus">default_cpus</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#default-ram">default_ram</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#default-disk">default_disk</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#default-gpus">default_gpus</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#hive">hive</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#default-hive-mapred-queue">default_hive_mapred_queue</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#webserver">webserver</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#base-url">base_url</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#web-server-host">web_server_host</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#web-server-port">web_server_port</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#web-server-ssl-cert">web_server_ssl_cert</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#web-server-ssl-key">web_server_ssl_key</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#web-server-master-timeout">web_server_master_timeout</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#web-server-worker-timeout">web_server_worker_timeout</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#worker-refresh-batch-size">worker_refresh_batch_size</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#worker-refresh-interval">worker_refresh_interval</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#secret-key">secret_key</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#workers">workers</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#worker-class">worker_class</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#access-logfile">access_logfile</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#error-logfile">error_logfile</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#expose-config">expose_config</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#expose-hostname">expose_hostname</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#expose-stacktrace">expose_stacktrace</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#authenticate">authenticate</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#filter-by-owner">filter_by_owner</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#owner-mode">owner_mode</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dag-default-view">dag_default_view</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dag-orientation">dag_orientation</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#demo-mode">demo_mode</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#log-fetch-timeout-sec">log_fetch_timeout_sec</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#log-fetch-delay-sec">log_fetch_delay_sec</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#log-auto-tailing-offset">log_auto_tailing_offset</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#log-animation-speed">log_animation_speed</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#hide-paused-dags-by-default">hide_paused_dags_by_default</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#page-size">page_size</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#rbac">rbac</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#navbar-color">navbar_color</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#default-dag-run-display-number">default_dag_run_display_number</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#enable-proxy-fix">enable_proxy_fix</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#proxy-fix-x-for">proxy_fix_x_for</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#proxy-fix-x-proto">proxy_fix_x_proto</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#proxy-fix-x-host">proxy_fix_x_host</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#proxy-fix-x-port">proxy_fix_x_port</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#proxy-fix-x-prefix">proxy_fix_x_prefix</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#cookie-secure">cookie_secure</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#cookie-samesite">cookie_samesite</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#default-wrap">default_wrap</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#x-frame-enabled">x_frame_enabled</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#analytics-tool">analytics_tool</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#analytics-id">analytics_id</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#update-fab-perms">update_fab_perms</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#force-log-out-after">force_log_out_after</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#session-lifetime-days">session_lifetime_days</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#email">email</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#email-backend">email_backend</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#smtp">smtp</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#smtp-host">smtp_host</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#smtp-starttls">smtp_starttls</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#smtp-ssl">smtp_ssl</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#smtp-user">smtp_user</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#smtp-password">smtp_password</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#smtp-port">smtp_port</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#smtp-mail-from">smtp_mail_from</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#sentry">sentry</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#sentry-dsn">sentry_dsn</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#celery">celery</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#celery-app-name">celery_app_name</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#worker-concurrency">worker_concurrency</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#worker-autoscale">worker_autoscale</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#worker-log-server-port">worker_log_server_port</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#broker-url">broker_url</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#result-backend">result_backend</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#flower-host">flower_host</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#flower-url-prefix">flower_url_prefix</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#flower-port">flower_port</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#flower-basic-auth">flower_basic_auth</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#default-queue">default_queue</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#sync-parallelism">sync_parallelism</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#celery-config-options">celery_config_options</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#ssl-active">ssl_active</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#ssl-key">ssl_key</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#ssl-cert">ssl_cert</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#ssl-cacert">ssl_cacert</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#pool">pool</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#operation-timeout">operation_timeout</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#celery-broker-transport-options">celery_broker_transport_options</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#visibility-timeout">visibility_timeout</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#dask">dask</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#cluster-address">cluster_address</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#tls-ca">tls_ca</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#tls-cert">tls_cert</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#tls-key">tls_key</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#scheduler">scheduler</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#job-heartbeat-sec">job_heartbeat_sec</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#scheduler-heartbeat-sec">scheduler_heartbeat_sec</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#run-duration">run_duration</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#num-runs">num_runs</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#processor-poll-interval">processor_poll_interval</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#min-file-process-interval">min_file_process_interval</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dag-dir-list-interval">dag_dir_list_interval</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#print-stats-interval">print_stats_interval</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#scheduler-health-check-threshold">scheduler_health_check_threshold</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#child-process-log-directory">child_process_log_directory</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#scheduler-zombie-task-threshold">scheduler_zombie_task_threshold</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#catchup-by-default">catchup_by_default</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#max-tis-per-query">max_tis_per_query</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#statsd-on">statsd_on</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#statsd-host">statsd_host</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#statsd-port">statsd_port</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#statsd-prefix">statsd_prefix</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#statsd-allow-list">statsd_allow_list</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#max-threads">max_threads</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#id1">authenticate</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#use-job-schedule">use_job_schedule</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#allow-trigger-in-future">allow_trigger_in_future</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#ldap">ldap</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#uri">uri</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#user-filter">user_filter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#user-name-attr">user_name_attr</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#group-member-attr">group_member_attr</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#superuser-filter">superuser_filter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#data-profiler-filter">data_profiler_filter</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#bind-user">bind_user</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#bind-password">bind_password</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#basedn">basedn</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#cacert">cacert</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#search-scope">search_scope</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#ignore-malformed-schema">ignore_malformed_schema</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#mesos">mesos</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#master">master</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#framework-name">framework_name</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#task-cpu">task_cpu</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#task-memory">task_memory</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#checkpoint">checkpoint</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#failover-timeout">failover_timeout</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#id2">authenticate</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#default-principal">default_principal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#default-secret">default_secret</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#docker-image-slave">docker_image_slave</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#kerberos">kerberos</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#ccache">ccache</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#principal">principal</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#reinit-frequency">reinit_frequency</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#kinit-path">kinit_path</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#keytab">keytab</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#github-enterprise">github_enterprise</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#api-rev">api_rev</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#admin">admin</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#hide-sensitive-variable-fields">hide_sensitive_variable_fields</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#elasticsearch">elasticsearch</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#id3">host</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#log-id-template">log_id_template</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#end-of-log-mark">end_of_log_mark</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#frontend">frontend</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#write-stdout">write_stdout</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#json-format">json_format</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#json-fields">json_fields</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#elasticsearch-configs">elasticsearch_configs</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#use-ssl">use_ssl</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#verify-certs">verify_certs</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#kubernetes">kubernetes</a><ul> |
| <li class="toctree-l3"><a class="reference internal" href="#worker-container-repository">worker_container_repository</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#worker-container-tag">worker_container_tag</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#worker-container-image-pull-policy">worker_container_image_pull_policy</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#delete-worker-pods">delete_worker_pods</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#worker-pods-creation-batch-size">worker_pods_creation_batch_size</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#namespace">namespace</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#airflow-configmap">airflow_configmap</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#airflow-local-settings-configmap">airflow_local_settings_configmap</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dags-in-image">dags_in_image</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dags-volume-subpath">dags_volume_subpath</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dags-volume-claim">dags_volume_claim</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#logs-volume-subpath">logs_volume_subpath</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#logs-volume-claim">logs_volume_claim</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#dags-volume-host">dags_volume_host</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#logs-volume-host">logs_volume_host</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#env-from-configmap-ref">env_from_configmap_ref</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#env-from-secret-ref">env_from_secret_ref</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-repo">git_repo</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-branch">git_branch</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-subpath">git_subpath</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-sync-rev">git_sync_rev</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-user">git_user</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-password">git_password</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-sync-root">git_sync_root</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-sync-dest">git_sync_dest</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-dags-folder-mount-point">git_dags_folder_mount_point</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-ssh-key-secret-name">git_ssh_key_secret_name</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-ssh-known-hosts-configmap-name">git_ssh_known_hosts_configmap_name</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-sync-credentials-secret">git_sync_credentials_secret</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-sync-container-repository">git_sync_container_repository</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-sync-container-tag">git_sync_container_tag</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-sync-init-container-name">git_sync_init_container_name</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#git-sync-run-as-user">git_sync_run_as_user</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#worker-service-account-name">worker_service_account_name</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#image-pull-secrets">image_pull_secrets</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#gcp-service-account-keys">gcp_service_account_keys</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#in-cluster">in_cluster</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#cluster-context">cluster_context</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#config-file">config_file</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#affinity">affinity</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#tolerations">tolerations</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#kube-client-request-args">kube_client_request_args</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#run-as-user">run_as_user</a></li> |
| <li class="toctree-l3"><a class="reference internal" href="#fs-group">fs_group</a></li> |
| </ul> |
| </li> |
| <li class="toctree-l2"><a class="reference internal" href="#kubernetes-node-selectors">kubernetes_node_selectors</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="#kubernetes-annotations">kubernetes_annotations</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="#kubernetes-environment-variables">kubernetes_environment_variables</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="#kubernetes-secrets">kubernetes_secrets</a></li> |
| <li class="toctree-l2"><a class="reference internal" href="#kubernetes-labels">kubernetes_labels</a></li> |
| </ul> |
| </li> |
| </ul> |
| |
| |
| |
| </div> |
| </div> |
| </nav> |
| |
| <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> |
| |
| |
| <nav class="wy-nav-top" aria-label="top navigation"> |
| |
| <i data-toggle="wy-nav-top" class="fa fa-bars"></i> |
| <a href="index.html">Airflow</a> |
| |
| </nav> |
| |
| |
| <div class="wy-nav-content"> |
| |
| <div class="rst-content"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <div role="navigation" aria-label="breadcrumbs navigation"> |
| |
| <ul class="wy-breadcrumbs"> |
| |
| <li><a href="index.html">Docs</a> »</li> |
| |
| <li>Configuration Reference</li> |
| |
| |
| <li class="wy-breadcrumbs-aside"> |
| |
| |
| <a href="_sources/configurations-ref.rst.txt" rel="nofollow"> View page source</a> |
| |
| |
| </li> |
| |
| </ul> |
| |
| |
| <hr/> |
| </div> |
| <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> |
| <div itemprop="articleBody"> |
| |
| <blockquote> |
| <div></div></blockquote> |
| <div class="section" id="configuration-reference"> |
| <h1>Configuration Reference<a class="headerlink" href="#configuration-reference" title="Permalink to this headline">¶</a></h1> |
| <p>This page contains the list of all the available Airflow configurations that you |
| can set in <code class="docutils literal notranslate"><span class="pre">airflow.cfg</span></code> file or using environment variables.</p> |
| <div class="contents local topic" id="sections"> |
| <p class="topic-title">Sections:</p> |
| <ul class="simple"> |
| <li><p><a class="reference internal" href="#core" id="id4">core</a></p></li> |
| <li><p><a class="reference internal" href="#cli" id="id5">cli</a></p></li> |
| <li><p><a class="reference internal" href="#debug" id="id6">debug</a></p></li> |
| <li><p><a class="reference internal" href="#api" id="id7">api</a></p></li> |
| <li><p><a class="reference internal" href="#lineage" id="id8">lineage</a></p></li> |
| <li><p><a class="reference internal" href="#atlas" id="id9">atlas</a></p></li> |
| <li><p><a class="reference internal" href="#operators" id="id10">operators</a></p></li> |
| <li><p><a class="reference internal" href="#hive" id="id11">hive</a></p></li> |
| <li><p><a class="reference internal" href="#webserver" id="id12">webserver</a></p></li> |
| <li><p><a class="reference internal" href="#email" id="id13">email</a></p></li> |
| <li><p><a class="reference internal" href="#smtp" id="id14">smtp</a></p></li> |
| <li><p><a class="reference internal" href="#sentry" id="id15">sentry</a></p></li> |
| <li><p><a class="reference internal" href="#celery" id="id16">celery</a></p></li> |
| <li><p><a class="reference internal" href="#celery-broker-transport-options" id="id17">celery_broker_transport_options</a></p></li> |
| <li><p><a class="reference internal" href="#dask" id="id18">dask</a></p></li> |
| <li><p><a class="reference internal" href="#scheduler" id="id19">scheduler</a></p></li> |
| <li><p><a class="reference internal" href="#ldap" id="id20">ldap</a></p></li> |
| <li><p><a class="reference internal" href="#mesos" id="id21">mesos</a></p></li> |
| <li><p><a class="reference internal" href="#kerberos" id="id22">kerberos</a></p></li> |
| <li><p><a class="reference internal" href="#github-enterprise" id="id23">github_enterprise</a></p></li> |
| <li><p><a class="reference internal" href="#admin" id="id24">admin</a></p></li> |
| <li><p><a class="reference internal" href="#elasticsearch" id="id25">elasticsearch</a></p></li> |
| <li><p><a class="reference internal" href="#elasticsearch-configs" id="id26">elasticsearch_configs</a></p></li> |
| <li><p><a class="reference internal" href="#kubernetes" id="id27">kubernetes</a></p></li> |
| <li><p><a class="reference internal" href="#kubernetes-node-selectors" id="id28">kubernetes_node_selectors</a></p></li> |
| <li><p><a class="reference internal" href="#kubernetes-annotations" id="id29">kubernetes_annotations</a></p></li> |
| <li><p><a class="reference internal" href="#kubernetes-environment-variables" id="id30">kubernetes_environment_variables</a></p></li> |
| <li><p><a class="reference internal" href="#kubernetes-secrets" id="id31">kubernetes_secrets</a></p></li> |
| <li><p><a class="reference internal" href="#kubernetes-labels" id="id32">kubernetes_labels</a></p></li> |
| </ul> |
| </div> |
| <div class="section" id="core"> |
| <h2><a class="toc-backref" href="#id4">core</a><a class="headerlink" href="#core" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="dags-folder"> |
| <h3>dags_folder<a class="headerlink" href="#dags-folder" title="Permalink to this headline">¶</a></h3> |
| <p>The folder where your airflow pipelines live, most likely a |
| subfolder in a code repository. This path must be absolute.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">{AIRFLOW_HOME}/dags</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__DAGS_FOLDER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="base-log-folder"> |
| <h3>base_log_folder<a class="headerlink" href="#base-log-folder" title="Permalink to this headline">¶</a></h3> |
| <p>The folder where airflow should store its log files |
| This path must be absolute</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">{AIRFLOW_HOME}/logs</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__BASE_LOG_FOLDER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="remote-logging"> |
| <h3>remote_logging<a class="headerlink" href="#remote-logging" title="Permalink to this headline">¶</a></h3> |
| <p>Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. |
| Set this to True if you want to enable remote logging.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__REMOTE_LOGGING</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="remote-log-conn-id"> |
| <h3>remote_log_conn_id<a class="headerlink" href="#remote-log-conn-id" title="Permalink to this headline">¶</a></h3> |
| <p>Users must supply an Airflow connection id that provides access to the storage |
| location.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__REMOTE_LOG_CONN_ID</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="remote-base-log-folder"> |
| <h3>remote_base_log_folder<a class="headerlink" href="#remote-base-log-folder" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="encrypt-s3-logs"> |
| <h3>encrypt_s3_logs<a class="headerlink" href="#encrypt-s3-logs" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__ENCRYPT_S3_LOGS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="logging-level"> |
| <h3>logging_level<a class="headerlink" href="#logging-level" title="Permalink to this headline">¶</a></h3> |
| <p>Logging level</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">INFO</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__LOGGING_LEVEL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="fab-logging-level"> |
| <h3>fab_logging_level<a class="headerlink" href="#fab-logging-level" title="Permalink to this headline">¶</a></h3> |
| <p>Logging level for Flask-appbuilder UI</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">WARN</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__FAB_LOGGING_LEVEL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="logging-config-class"> |
| <h3>logging_config_class<a class="headerlink" href="#logging-config-class" title="Permalink to this headline">¶</a></h3> |
| <p>Logging class |
| Specify the class that will specify the logging configuration |
| This class has to be on the python classpath</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__LOGGING_CONFIG_CLASS</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">my.path.default_local_settings.LOGGING_CONFIG</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="colored-console-log"> |
| <h3>colored_console_log<a class="headerlink" href="#colored-console-log" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>Flag to enable/disable Colored logs in Console |
| Colour the logs when the controlling terminal is a TTY.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__COLORED_CONSOLE_LOG</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="colored-log-format"> |
| <h3>colored_log_format<a class="headerlink" href="#colored-log-format" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>Log format for when Colored logs is enabled</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">[%%(blue)s%%(asctime)s%%(reset)s]</span> <span class="pre">{{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}}</span> <span class="pre">%%(log_color)s%%(levelname)s%%(reset)s</span> <span class="pre">-</span> <span class="pre">%%(log_color)s%%(message)s%%(reset)s</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__COLORED_LOG_FORMAT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="colored-formatter-class"> |
| <h3>colored_formatter_class<a class="headerlink" href="#colored-formatter-class" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow.utils.log.colored_log.CustomTTYColoredFormatter</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__COLORED_FORMATTER_CLASS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="log-format"> |
| <h3>log_format<a class="headerlink" href="#log-format" title="Permalink to this headline">¶</a></h3> |
| <p>Format of Log line</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">[%%(asctime)s]</span> <span class="pre">{{%%(filename)s:%%(lineno)d}}</span> <span class="pre">%%(levelname)s</span> <span class="pre">-</span> <span class="pre">%%(message)s</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__LOG_FORMAT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="simple-log-format"> |
| <h3>simple_log_format<a class="headerlink" href="#simple-log-format" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">%%(asctime)s</span> <span class="pre">%%(levelname)s</span> <span class="pre">-</span> <span class="pre">%%(message)s</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SIMPLE_LOG_FORMAT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="log-filename-template"> |
| <h3>log_filename_template<a class="headerlink" href="#log-filename-template" title="Permalink to this headline">¶</a></h3> |
| <p>Log filename format</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">{{{{</span> <span class="pre">ti.dag_id</span> <span class="pre">}}}}/{{{{</span> <span class="pre">ti.task_id</span> <span class="pre">}}}}/{{{{</span> <span class="pre">ts</span> <span class="pre">}}}}/{{{{</span> <span class="pre">try_number</span> <span class="pre">}}}}.log</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__LOG_FILENAME_TEMPLATE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="log-processor-filename-template"> |
| <h3>log_processor_filename_template<a class="headerlink" href="#log-processor-filename-template" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">{{{{</span> <span class="pre">filename</span> <span class="pre">}}}}.log</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__LOG_PROCESSOR_FILENAME_TEMPLATE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dag-processor-manager-log-location"> |
| <h3>dag_processor_manager_log_location<a class="headerlink" href="#dag-processor-manager-log-location" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">{AIRFLOW_HOME}/logs/dag_processor_manager/dag_processor_manager.log</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="task-log-reader"> |
| <h3>task_log_reader<a class="headerlink" href="#task-log-reader" title="Permalink to this headline">¶</a></h3> |
| <p>Name of handler to read task instance logs. |
| Default to use task handler.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">task</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__TASK_LOG_READER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="hostname-callable"> |
| <h3>hostname_callable<a class="headerlink" href="#hostname-callable" title="Permalink to this headline">¶</a></h3> |
| <p>Hostname by providing a path to a callable, which will resolve the hostname. |
| The format is “package:function”.</p> |
| <p>For example, default value “socket:getfqdn” means that result from getfqdn() of “socket” |
| package will be used as hostname.</p> |
| <p>No argument should be required in the function specified. |
| If using IP address as hostname is preferred, use value <code class="docutils literal notranslate"><span class="pre">airflow.utils.net:get_host_ip_address</span></code></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">socket:getfqdn</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__HOSTNAME_CALLABLE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="default-timezone"> |
| <h3>default_timezone<a class="headerlink" href="#default-timezone" title="Permalink to this headline">¶</a></h3> |
| <p>Default timezone in case supplied date times are naive |
| can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam)</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">utc</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__DEFAULT_TIMEZONE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="executor"> |
| <h3>executor<a class="headerlink" href="#executor" title="Permalink to this headline">¶</a></h3> |
| <p>The executor class that airflow should use. Choices include |
| SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor, KubernetesExecutor</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">SequentialExecutor</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__EXECUTOR</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="sql-alchemy-conn"> |
| <h3>sql_alchemy_conn<a class="headerlink" href="#sql-alchemy-conn" title="Permalink to this headline">¶</a></h3> |
| <p>The SqlAlchemy connection string to the metadata database. |
| SqlAlchemy supports many different database engine, more information |
| their website</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">sqlite:///{AIRFLOW_HOME}/airflow.db</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_CONN</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="sql-engine-encoding"> |
| <h3>sql_engine_encoding<a class="headerlink" href="#sql-engine-encoding" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.1.</span></p> |
| </div> |
| <p>The encoding for the databases</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">utf-8</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ENGINE_ENCODING</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="sql-alchemy-pool-enabled"> |
| <h3>sql_alchemy_pool_enabled<a class="headerlink" href="#sql-alchemy-pool-enabled" title="Permalink to this headline">¶</a></h3> |
| <p>If SqlAlchemy should pool database connections.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_POOL_ENABLED</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="sql-alchemy-pool-size"> |
| <h3>sql_alchemy_pool_size<a class="headerlink" href="#sql-alchemy-pool-size" title="Permalink to this headline">¶</a></h3> |
| <p>The SqlAlchemy pool size is the maximum number of database connections |
| in the pool. 0 indicates no limit.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">5</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_POOL_SIZE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="sql-alchemy-max-overflow"> |
| <h3>sql_alchemy_max_overflow<a class="headerlink" href="#sql-alchemy-max-overflow" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>The maximum overflow size of the pool. |
| When the number of checked-out connections reaches the size set in pool_size, |
| additional connections will be returned up to this limit. |
| When those additional connections are returned to the pool, they are disconnected and discarded. |
| It follows then that the total number of simultaneous connections the pool will allow |
| is pool_size + max_overflow, |
| and the total number of “sleeping” connections the pool will allow is pool_size. |
| max_overflow can be set to -1 to indicate no overflow limit; |
| no limit will be placed on the total number of concurrent connections. Defaults to 10.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">10</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_MAX_OVERFLOW</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="sql-alchemy-pool-recycle"> |
| <h3>sql_alchemy_pool_recycle<a class="headerlink" href="#sql-alchemy-pool-recycle" title="Permalink to this headline">¶</a></h3> |
| <p>The SqlAlchemy pool recycle is the number of seconds a connection |
| can be idle in the pool before it is invalidated. This config does |
| not apply to sqlite. If the number of DB connections is ever exceeded, |
| a lower config value will allow the system to recover faster.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">1800</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_POOL_RECYCLE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="sql-alchemy-pool-pre-ping"> |
| <h3>sql_alchemy_pool_pre_ping<a class="headerlink" href="#sql-alchemy-pool-pre-ping" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.6.</span></p> |
| </div> |
| <p>Check connection at the start of each connection pool checkout. |
| Typically, this is a simple statement like “SELECT 1”. |
| More information here: |
| <a class="reference external" href="https://docs.sqlalchemy.org/en/13/core/pooling.html#disconnect-handling-pessimistic">https://docs.sqlalchemy.org/en/13/core/pooling.html#disconnect-handling-pessimistic</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_POOL_PRE_PING</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="sql-alchemy-schema"> |
| <h3>sql_alchemy_schema<a class="headerlink" href="#sql-alchemy-schema" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>The schema to use for the metadata database. |
| SqlAlchemy supports databases with the concept of multiple schemas.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_SCHEMA</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="parallelism"> |
| <h3>parallelism<a class="headerlink" href="#parallelism" title="Permalink to this headline">¶</a></h3> |
| <p>The amount of parallelism as a setting to the executor. This defines |
| the max number of task instances that should run simultaneously |
| on this airflow installation</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">32</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__PARALLELISM</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dag-concurrency"> |
| <h3>dag_concurrency<a class="headerlink" href="#dag-concurrency" title="Permalink to this headline">¶</a></h3> |
| <p>The number of task instances allowed to run concurrently by the scheduler</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">16</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__DAG_CONCURRENCY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dags-are-paused-at-creation"> |
| <h3>dags_are_paused_at_creation<a class="headerlink" href="#dags-are-paused-at-creation" title="Permalink to this headline">¶</a></h3> |
| <p>Are DAGs paused by default at creation</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="max-active-runs-per-dag"> |
| <h3>max_active_runs_per_dag<a class="headerlink" href="#max-active-runs-per-dag" title="Permalink to this headline">¶</a></h3> |
| <p>The maximum number of active DAG runs per DAG</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">16</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="load-examples"> |
| <h3>load_examples<a class="headerlink" href="#load-examples" title="Permalink to this headline">¶</a></h3> |
| <p>Whether to load the examples that ship with Airflow. It’s good to |
| get started, but you probably want to set this to False in a production |
| environment</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__LOAD_EXAMPLES</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="plugins-folder"> |
| <h3>plugins_folder<a class="headerlink" href="#plugins-folder" title="Permalink to this headline">¶</a></h3> |
| <p>Where your Airflow plugins are stored</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">{AIRFLOW_HOME}/plugins</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__PLUGINS_FOLDER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="fernet-key"> |
| <h3>fernet_key<a class="headerlink" href="#fernet-key" title="Permalink to this headline">¶</a></h3> |
| <p>Secret key to save connection passwords in the db</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">{FERNET_KEY}</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__FERNET_KEY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="donot-pickle"> |
| <h3>donot_pickle<a class="headerlink" href="#donot-pickle" title="Permalink to this headline">¶</a></h3> |
| <p>Whether to disable pickling dags</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__DONOT_PICKLE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dagbag-import-timeout"> |
| <h3>dagbag_import_timeout<a class="headerlink" href="#dagbag-import-timeout" title="Permalink to this headline">¶</a></h3> |
| <p>How long before timing out a python file import</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">30</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dag-file-processor-timeout"> |
| <h3>dag_file_processor_timeout<a class="headerlink" href="#dag-file-processor-timeout" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.6.</span></p> |
| </div> |
| <p>How long before timing out a DagFileProcessor, which processes a dag file</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">50</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__DAG_FILE_PROCESSOR_TIMEOUT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="task-runner"> |
| <h3>task_runner<a class="headerlink" href="#task-runner" title="Permalink to this headline">¶</a></h3> |
| <p>The class to use for running task instances in a subprocess</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">StandardTaskRunner</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__TASK_RUNNER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="default-impersonation"> |
| <h3>default_impersonation<a class="headerlink" href="#default-impersonation" title="Permalink to this headline">¶</a></h3> |
| <p>If set, tasks without a <code class="docutils literal notranslate"><span class="pre">run_as_user</span></code> argument will be run with this user |
| Can be used to de-elevate a sudo user running Airflow when executing tasks</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__DEFAULT_IMPERSONATION</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="security"> |
| <h3>security<a class="headerlink" href="#security" title="Permalink to this headline">¶</a></h3> |
| <p>What security module to use (for example kerberos)</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SECURITY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="secure-mode"> |
| <h3>secure_mode<a class="headerlink" href="#secure-mode" title="Permalink to this headline">¶</a></h3> |
| <p>If set to False enables some unsecure features like Charts and Ad Hoc Queries. |
| In 2.0 will default to True.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SECURE_MODE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="unit-test-mode"> |
| <h3>unit_test_mode<a class="headerlink" href="#unit-test-mode" title="Permalink to this headline">¶</a></h3> |
| <p>Turn unit test mode on (overwrites many configuration options with test |
| values at runtime)</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__UNIT_TEST_MODE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="enable-xcom-pickling"> |
| <h3>enable_xcom_pickling<a class="headerlink" href="#enable-xcom-pickling" title="Permalink to this headline">¶</a></h3> |
| <p>Whether to enable pickling for xcom (note that this is insecure and allows for |
| RCE exploits). This will be deprecated in Airflow 2.0 (be forced to False).</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__ENABLE_XCOM_PICKLING</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="killed-task-cleanup-time"> |
| <h3>killed_task_cleanup_time<a class="headerlink" href="#killed-task-cleanup-time" title="Permalink to this headline">¶</a></h3> |
| <p>When a task is killed forcefully, this is the amount of time in seconds that |
| it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">60</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__KILLED_TASK_CLEANUP_TIME</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dag-run-conf-overrides-params"> |
| <h3>dag_run_conf_overrides_params<a class="headerlink" href="#dag-run-conf-overrides-params" title="Permalink to this headline">¶</a></h3> |
| <p>Whether to override params with dag_run.conf. If you pass some key-value pairs |
| through <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">dags</span> <span class="pre">backfill</span> <span class="pre">-c</span></code> or |
| <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">dags</span> <span class="pre">trigger</span> <span class="pre">-c</span></code>, the key-value pairs will override the existing ones in params.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__DAG_RUN_CONF_OVERRIDES_PARAMS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="worker-precheck"> |
| <h3>worker_precheck<a class="headerlink" href="#worker-precheck" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.1.</span></p> |
| </div> |
| <p>Worker initialisation check to validate Metadata Database connection</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__WORKER_PRECHECK</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dag-discovery-safe-mode"> |
| <h3>dag_discovery_safe_mode<a class="headerlink" href="#dag-discovery-safe-mode" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>When discovering DAGs, ignore any files that don’t contain the strings <code class="docutils literal notranslate"><span class="pre">DAG</span></code> and <code class="docutils literal notranslate"><span class="pre">airflow</span></code>.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__DAG_DISCOVERY_SAFE_MODE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="default-task-retries"> |
| <h3>default_task_retries<a class="headerlink" href="#default-task-retries" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.6.</span></p> |
| </div> |
| <p>The number of retries each task is going to have by default. Can be overridden at dag or task level.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">0</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__DEFAULT_TASK_RETRIES</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="store-serialized-dags"> |
| <h3>store_serialized_dags<a class="headerlink" href="#store-serialized-dags" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.7.</span></p> |
| </div> |
| <p>Whether to serialises DAGs and persist them in DB. |
| If set to True, Webserver reads from DB instead of parsing DAG files |
| More details: <a class="reference external" href="https://airflow.apache.org/docs/stable/dag-serialization.html">https://airflow.apache.org/docs/stable/dag-serialization.html</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__STORE_SERIALIZED_DAGS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="min-serialized-dag-update-interval"> |
| <h3>min_serialized_dag_update_interval<a class="headerlink" href="#min-serialized-dag-update-interval" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.7.</span></p> |
| </div> |
| <p>Updating serialized DAG can not be faster than a minimum interval to reduce database write rate.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">30</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__MIN_SERIALIZED_DAG_UPDATE_INTERVAL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="check-slas"> |
| <h3>check_slas<a class="headerlink" href="#check-slas" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>On each dagrun check against defined SLAs</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__CHECK_SLAS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="cli"> |
| <h2><a class="toc-backref" href="#id5">cli</a><a class="headerlink" href="#cli" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="api-client"> |
| <h3>api_client<a class="headerlink" href="#api-client" title="Permalink to this headline">¶</a></h3> |
| <p>In what way should the cli access the API. The LocalClient will use the |
| database directly, while the json_client will use the api running on the |
| webserver</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow.api.client.local_client</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CLI__API_CLIENT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="endpoint-url"> |
| <h3>endpoint_url<a class="headerlink" href="#endpoint-url" title="Permalink to this headline">¶</a></h3> |
| <p>If you set web_server_url_prefix, do NOT forget to append it here, ex: |
| <code class="docutils literal notranslate"><span class="pre">endpoint_url</span> <span class="pre">=</span> <span class="pre">http://localhost:8080/myroot</span></code> |
| So api will look like: <code class="docutils literal notranslate"><span class="pre">http://localhost:8080/myroot/api/experimental/...</span></code></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">http://localhost:8080</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CLI__ENDPOINT_URL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="debug"> |
| <h2><a class="toc-backref" href="#id6">debug</a><a class="headerlink" href="#debug" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="fail-fast"> |
| <h3>fail_fast<a class="headerlink" href="#fail-fast" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>Used only with DebugExecutor. If set to True DAG will fail with first |
| failed task. Helpful for debugging purposes.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__DEBUG__FAIL_FAST</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="api"> |
| <h2><a class="toc-backref" href="#id7">api</a><a class="headerlink" href="#api" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="auth-backend"> |
| <h3>auth_backend<a class="headerlink" href="#auth-backend" title="Permalink to this headline">¶</a></h3> |
| <p>How to authenticate users of the API</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow.api.auth.backend.default</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__API__AUTH_BACKEND</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="lineage"> |
| <h2><a class="toc-backref" href="#id8">lineage</a><a class="headerlink" href="#lineage" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="backend"> |
| <h3>backend<a class="headerlink" href="#backend" title="Permalink to this headline">¶</a></h3> |
| <p>what lineage backend to use</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LINEAGE__BACKEND</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="atlas"> |
| <h2><a class="toc-backref" href="#id9">atlas</a><a class="headerlink" href="#atlas" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="sasl-enabled"> |
| <h3>sasl_enabled<a class="headerlink" href="#sasl-enabled" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ATLAS__SASL_ENABLED</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="host"> |
| <h3>host<a class="headerlink" href="#host" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ATLAS__HOST</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="port"> |
| <h3>port<a class="headerlink" href="#port" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">21000</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ATLAS__PORT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="username"> |
| <h3>username<a class="headerlink" href="#username" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ATLAS__USERNAME</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="password"> |
| <h3>password<a class="headerlink" href="#password" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ATLAS__PASSWORD</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="operators"> |
| <h2><a class="toc-backref" href="#id10">operators</a><a class="headerlink" href="#operators" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="default-owner"> |
| <h3>default_owner<a class="headerlink" href="#default-owner" title="Permalink to this headline">¶</a></h3> |
| <p>The default owner assigned to each new operator, unless |
| provided explicitly or passed via <code class="docutils literal notranslate"><span class="pre">default_args</span></code></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__OPERATORS__DEFAULT_OWNER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="default-cpus"> |
| <h3>default_cpus<a class="headerlink" href="#default-cpus" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__OPERATORS__DEFAULT_CPUS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="default-ram"> |
| <h3>default_ram<a class="headerlink" href="#default-ram" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">512</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__OPERATORS__DEFAULT_RAM</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="default-disk"> |
| <h3>default_disk<a class="headerlink" href="#default-disk" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">512</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__OPERATORS__DEFAULT_DISK</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="default-gpus"> |
| <h3>default_gpus<a class="headerlink" href="#default-gpus" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">0</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__OPERATORS__DEFAULT_GPUS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="hive"> |
| <h2><a class="toc-backref" href="#id11">hive</a><a class="headerlink" href="#hive" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="default-hive-mapred-queue"> |
| <h3>default_hive_mapred_queue<a class="headerlink" href="#default-hive-mapred-queue" title="Permalink to this headline">¶</a></h3> |
| <p>Default mapreduce queue for HiveOperator tasks</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__HIVE__DEFAULT_HIVE_MAPRED_QUEUE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="webserver"> |
| <h2><a class="toc-backref" href="#id12">webserver</a><a class="headerlink" href="#webserver" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="base-url"> |
| <h3>base_url<a class="headerlink" href="#base-url" title="Permalink to this headline">¶</a></h3> |
| <p>The base url of your website as airflow cannot guess what domain or |
| cname you are using. This is used in automated emails that |
| airflow sends to point links to the right web server</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">http://localhost:8080</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__BASE_URL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="web-server-host"> |
| <h3>web_server_host<a class="headerlink" href="#web-server-host" title="Permalink to this headline">¶</a></h3> |
| <p>The ip specified when starting the web server</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">0.0.0.0</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__WEB_SERVER_HOST</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="web-server-port"> |
| <h3>web_server_port<a class="headerlink" href="#web-server-port" title="Permalink to this headline">¶</a></h3> |
| <p>The port on which to run the web server</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">8080</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__WEB_SERVER_PORT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="web-server-ssl-cert"> |
| <h3>web_server_ssl_cert<a class="headerlink" href="#web-server-ssl-cert" title="Permalink to this headline">¶</a></h3> |
| <p>Paths to the SSL certificate and key for the web server. When both are |
| provided SSL will be enabled. This does not change the web server port.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__WEB_SERVER_SSL_CERT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="web-server-ssl-key"> |
| <h3>web_server_ssl_key<a class="headerlink" href="#web-server-ssl-key" title="Permalink to this headline">¶</a></h3> |
| <p>Paths to the SSL certificate and key for the web server. When both are |
| provided SSL will be enabled. This does not change the web server port.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__WEB_SERVER_SSL_KEY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="web-server-master-timeout"> |
| <h3>web_server_master_timeout<a class="headerlink" href="#web-server-master-timeout" title="Permalink to this headline">¶</a></h3> |
| <p>Number of seconds the webserver waits before killing gunicorn master that doesn’t respond</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">120</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__WEB_SERVER_MASTER_TIMEOUT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="web-server-worker-timeout"> |
| <h3>web_server_worker_timeout<a class="headerlink" href="#web-server-worker-timeout" title="Permalink to this headline">¶</a></h3> |
| <p>Number of seconds the gunicorn webserver waits before timing out on a worker</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">120</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__WEB_SERVER_WORKER_TIMEOUT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="worker-refresh-batch-size"> |
| <h3>worker_refresh_batch_size<a class="headerlink" href="#worker-refresh-batch-size" title="Permalink to this headline">¶</a></h3> |
| <p>Number of workers to refresh at a time. When set to 0, worker refresh is |
| disabled. When nonzero, airflow periodically refreshes webserver workers by |
| bringing up new ones and killing old ones.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="worker-refresh-interval"> |
| <h3>worker_refresh_interval<a class="headerlink" href="#worker-refresh-interval" title="Permalink to this headline">¶</a></h3> |
| <p>Number of seconds to wait before refreshing a batch of workers.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">30</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__WORKER_REFRESH_INTERVAL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="secret-key"> |
| <h3>secret_key<a class="headerlink" href="#secret-key" title="Permalink to this headline">¶</a></h3> |
| <p>Secret key used to run your flask app |
| It should be as random as possible</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">temporary_key</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__SECRET_KEY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="workers"> |
| <h3>workers<a class="headerlink" href="#workers" title="Permalink to this headline">¶</a></h3> |
| <p>Number of workers to run the Gunicorn web server</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">4</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__WORKERS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="worker-class"> |
| <h3>worker_class<a class="headerlink" href="#worker-class" title="Permalink to this headline">¶</a></h3> |
| <p>The worker class gunicorn should use. Choices include |
| sync (default), eventlet, gevent</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">sync</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__WORKER_CLASS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="access-logfile"> |
| <h3>access_logfile<a class="headerlink" href="#access-logfile" title="Permalink to this headline">¶</a></h3> |
| <p>Log files for the gunicorn webserver. ‘-‘ means log to stderr.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">-</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__ACCESS_LOGFILE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="error-logfile"> |
| <h3>error_logfile<a class="headerlink" href="#error-logfile" title="Permalink to this headline">¶</a></h3> |
| <p>Log files for the gunicorn webserver. ‘-‘ means log to stderr.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">-</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__ERROR_LOGFILE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="expose-config"> |
| <h3>expose_config<a class="headerlink" href="#expose-config" title="Permalink to this headline">¶</a></h3> |
| <p>Expose the configuration file in the web server</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__EXPOSE_CONFIG</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="expose-hostname"> |
| <h3>expose_hostname<a class="headerlink" href="#expose-hostname" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>Expose hostname in the web server</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__EXPOSE_HOSTNAME</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="expose-stacktrace"> |
| <h3>expose_stacktrace<a class="headerlink" href="#expose-stacktrace" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>Expose stacktrace in the web server</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__EXPOSE_STACKTRACE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="authenticate"> |
| <h3>authenticate<a class="headerlink" href="#authenticate" title="Permalink to this headline">¶</a></h3> |
| <p>Set to true to turn on authentication: |
| <a class="reference external" href="https://airflow.apache.org/security.html#web-authentication">https://airflow.apache.org/security.html#web-authentication</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>boolean</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__AUTHENTICATE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="filter-by-owner"> |
| <h3>filter_by_owner<a class="headerlink" href="#filter-by-owner" title="Permalink to this headline">¶</a></h3> |
| <p>Filter the list of dags by owner name (requires authentication to be enabled)</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>boolean</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__FILTER_BY_OWNER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="owner-mode"> |
| <h3>owner_mode<a class="headerlink" href="#owner-mode" title="Permalink to this headline">¶</a></h3> |
| <p>Filtering mode. Choices include user (default) and ldapgroup. |
| Ldap group filtering requires using the ldap backend</p> |
| <p>Note that the ldap server needs the “memberOf” overlay to be set up |
| in order to user the ldapgroup mode.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">user</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__OWNER_MODE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dag-default-view"> |
| <h3>dag_default_view<a class="headerlink" href="#dag-default-view" title="Permalink to this headline">¶</a></h3> |
| <p>Default DAG view. Valid values are: |
| tree, graph, duration, gantt, landing_times</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">tree</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__DAG_DEFAULT_VIEW</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dag-orientation"> |
| <h3>dag_orientation<a class="headerlink" href="#dag-orientation" title="Permalink to this headline">¶</a></h3> |
| <p>“Default DAG orientation. Valid values are:” |
| LR (Left->Right), TB (Top->Bottom), RL (Right->Left), BT (Bottom->Top)</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">LR</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__DAG_ORIENTATION</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="demo-mode"> |
| <h3>demo_mode<a class="headerlink" href="#demo-mode" title="Permalink to this headline">¶</a></h3> |
| <p>Puts the webserver in demonstration mode; blurs the names of Operators for |
| privacy.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__DEMO_MODE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="log-fetch-timeout-sec"> |
| <h3>log_fetch_timeout_sec<a class="headerlink" href="#log-fetch-timeout-sec" title="Permalink to this headline">¶</a></h3> |
| <p>The amount of time (in secs) webserver will wait for initial handshake |
| while fetching logs from other worker machine</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">5</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__LOG_FETCH_TIMEOUT_SEC</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="log-fetch-delay-sec"> |
| <h3>log_fetch_delay_sec<a class="headerlink" href="#log-fetch-delay-sec" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>Time interval (in secs) to wait before next log fetching.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>int</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">2</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__LOG_FETCH_DELAY_SEC</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="log-auto-tailing-offset"> |
| <h3>log_auto_tailing_offset<a class="headerlink" href="#log-auto-tailing-offset" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>Distance away from page bottom to enable auto tailing.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>int</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">30</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__LOG_AUTO_TAILING_OFFSET</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="log-animation-speed"> |
| <h3>log_animation_speed<a class="headerlink" href="#log-animation-speed" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>Animation speed for auto tailing log display.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>int</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">1000</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__LOG_ANIMATION_SPEED</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="hide-paused-dags-by-default"> |
| <h3>hide_paused_dags_by_default<a class="headerlink" href="#hide-paused-dags-by-default" title="Permalink to this headline">¶</a></h3> |
| <p>By default, the webserver shows paused DAGs. Flip this to hide paused |
| DAGs by default</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__HIDE_PAUSED_DAGS_BY_DEFAULT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="page-size"> |
| <h3>page_size<a class="headerlink" href="#page-size" title="Permalink to this headline">¶</a></h3> |
| <p>Consistent page size across all listing views in the UI</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">100</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__PAGE_SIZE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="rbac"> |
| <h3>rbac<a class="headerlink" href="#rbac" title="Permalink to this headline">¶</a></h3> |
| <p>Use FAB-based webserver with RBAC feature</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__RBAC</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="navbar-color"> |
| <h3>navbar_color<a class="headerlink" href="#navbar-color" title="Permalink to this headline">¶</a></h3> |
| <p>Define the color of navigation bar</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">#007A87</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__NAVBAR_COLOR</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="default-dag-run-display-number"> |
| <h3>default_dag_run_display_number<a class="headerlink" href="#default-dag-run-display-number" title="Permalink to this headline">¶</a></h3> |
| <p>Default dagrun to show in UI</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">25</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__DEFAULT_DAG_RUN_DISPLAY_NUMBER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="enable-proxy-fix"> |
| <h3>enable_proxy_fix<a class="headerlink" href="#enable-proxy-fix" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.1.</span></p> |
| </div> |
| <p>Enable werkzeug <code class="docutils literal notranslate"><span class="pre">ProxyFix</span></code> middleware for reverse proxy</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>boolean</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="proxy-fix-x-for"> |
| <h3>proxy_fix_x_for<a class="headerlink" href="#proxy-fix-x-for" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.7.</span></p> |
| </div> |
| <p>Number of values to trust for <code class="docutils literal notranslate"><span class="pre">X-Forwarded-For</span></code>. |
| More info: <a class="reference external" href="https://werkzeug.palletsprojects.com/en/0.16.x/middleware/proxy_fix/">https://werkzeug.palletsprojects.com/en/0.16.x/middleware/proxy_fix/</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>integer</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__PROXY_FIX_X_FOR</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="proxy-fix-x-proto"> |
| <h3>proxy_fix_x_proto<a class="headerlink" href="#proxy-fix-x-proto" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.7.</span></p> |
| </div> |
| <p>Number of values to trust for <code class="docutils literal notranslate"><span class="pre">X-Forwarded-Proto</span></code></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>integer</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__PROXY_FIX_X_PROTO</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="proxy-fix-x-host"> |
| <h3>proxy_fix_x_host<a class="headerlink" href="#proxy-fix-x-host" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.7.</span></p> |
| </div> |
| <p>Number of values to trust for <code class="docutils literal notranslate"><span class="pre">X-Forwarded-Host</span></code></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>integer</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__PROXY_FIX_X_HOST</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="proxy-fix-x-port"> |
| <h3>proxy_fix_x_port<a class="headerlink" href="#proxy-fix-x-port" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.7.</span></p> |
| </div> |
| <p>Number of values to trust for <code class="docutils literal notranslate"><span class="pre">X-Forwarded-Port</span></code></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>integer</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__PROXY_FIX_X_PORT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="proxy-fix-x-prefix"> |
| <h3>proxy_fix_x_prefix<a class="headerlink" href="#proxy-fix-x-prefix" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.7.</span></p> |
| </div> |
| <p>Number of values to trust for <code class="docutils literal notranslate"><span class="pre">X-Forwarded-Prefix</span></code></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>integer</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__PROXY_FIX_X_PREFIX</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="cookie-secure"> |
| <h3>cookie_secure<a class="headerlink" href="#cookie-secure" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>Set secure flag on session cookie</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__COOKIE_SECURE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="cookie-samesite"> |
| <h3>cookie_samesite<a class="headerlink" href="#cookie-samesite" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>Set samesite policy on session cookie</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__COOKIE_SAMESITE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="default-wrap"> |
| <h3>default_wrap<a class="headerlink" href="#default-wrap" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>Default setting for wrap toggle on DAG code and TI log views.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>boolean</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__DEFAULT_WRAP</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="x-frame-enabled"> |
| <h3>x_frame_enabled<a class="headerlink" href="#x-frame-enabled" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>Allow the UI to be rendered in a frame</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>boolean</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__X_FRAME_ENABLED</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="analytics-tool"> |
| <h3>analytics_tool<a class="headerlink" href="#analytics-tool" title="Permalink to this headline">¶</a></h3> |
| <p>Send anonymous user activity to your analytics tool |
| choose from google_analytics, segment, or metarouter</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__ANALYTICS_TOOL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="analytics-id"> |
| <h3>analytics_id<a class="headerlink" href="#analytics-id" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.5.</span></p> |
| </div> |
| <p>Unique ID of your account in the analytics tool</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__ANALYTICS_ID</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="update-fab-perms"> |
| <h3>update_fab_perms<a class="headerlink" href="#update-fab-perms" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.7.</span></p> |
| </div> |
| <p>Update FAB permissions and sync security manager roles |
| on webserver startup</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__UPDATE_FAB_PERMS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="force-log-out-after"> |
| <h3>force_log_out_after<a class="headerlink" href="#force-log-out-after" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>Minutes of non-activity before logged out from UI |
| 0 means never get forcibly logged out</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">0</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__FORCE_LOG_OUT_AFTER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="session-lifetime-days"> |
| <h3>session_lifetime_days<a class="headerlink" href="#session-lifetime-days" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>The UI cookie lifetime in days</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">30</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__WEBSERVER__SESSION_LIFETIME_DAYS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="email"> |
| <h2><a class="toc-backref" href="#id13">email</a><a class="headerlink" href="#email" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="email-backend"> |
| <h3>email_backend<a class="headerlink" href="#email-backend" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow.utils.email.send_email_smtp</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__EMAIL__EMAIL_BACKEND</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="smtp"> |
| <h2><a class="toc-backref" href="#id14">smtp</a><a class="headerlink" href="#smtp" title="Permalink to this headline">¶</a></h2> |
| <p>If you want airflow to send emails on retries, failure, and you want to use |
| the airflow.utils.email.send_email_smtp function, you have to configure an |
| smtp server here</p> |
| <div class="section" id="smtp-host"> |
| <h3>smtp_host<a class="headerlink" href="#smtp-host" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">localhost</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SMTP__SMTP_HOST</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="smtp-starttls"> |
| <h3>smtp_starttls<a class="headerlink" href="#smtp-starttls" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SMTP__SMTP_STARTTLS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="smtp-ssl"> |
| <h3>smtp_ssl<a class="headerlink" href="#smtp-ssl" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SMTP__SMTP_SSL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="smtp-user"> |
| <h3>smtp_user<a class="headerlink" href="#smtp-user" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SMTP__SMTP_USER</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="smtp-password"> |
| <h3>smtp_password<a class="headerlink" href="#smtp-password" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SMTP__SMTP_PASSWORD</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="smtp-port"> |
| <h3>smtp_port<a class="headerlink" href="#smtp-port" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">25</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SMTP__SMTP_PORT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="smtp-mail-from"> |
| <h3>smtp_mail_from<a class="headerlink" href="#smtp-mail-from" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow@example.com</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SMTP__SMTP_MAIL_FROM</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="sentry"> |
| <h2><a class="toc-backref" href="#id15">sentry</a><a class="headerlink" href="#sentry" title="Permalink to this headline">¶</a></h2> |
| <p>Sentry (<a class="reference external" href="https://docs.sentry.io">https://docs.sentry.io</a>) integration</p> |
| <div class="section" id="sentry-dsn"> |
| <h3>sentry_dsn<a class="headerlink" href="#sentry-dsn" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.6.</span></p> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SENTRY__SENTRY_DSN</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="celery"> |
| <h2><a class="toc-backref" href="#id16">celery</a><a class="headerlink" href="#celery" title="Permalink to this headline">¶</a></h2> |
| <p>This section only applies if you are using the CeleryExecutor in |
| <code class="docutils literal notranslate"><span class="pre">[core]</span></code> section above</p> |
| <div class="section" id="celery-app-name"> |
| <h3>celery_app_name<a class="headerlink" href="#celery-app-name" title="Permalink to this headline">¶</a></h3> |
| <p>The app name that will be used by celery</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__CELERY_APP_NAME</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="worker-concurrency"> |
| <h3>worker_concurrency<a class="headerlink" href="#worker-concurrency" title="Permalink to this headline">¶</a></h3> |
| <p>The concurrency that will be used when starting workers with the |
| <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">celery</span> <span class="pre">worker</span></code> command. This defines the number of task instances that |
| a worker will take, so size up your workers based on the resources on |
| your worker box and the nature of your tasks</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">16</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__WORKER_CONCURRENCY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="worker-autoscale"> |
| <h3>worker_autoscale<a class="headerlink" href="#worker-autoscale" title="Permalink to this headline">¶</a></h3> |
| <p>The maximum and minimum concurrency that will be used when starting workers with the |
| <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">celery</span> <span class="pre">worker</span></code> command (always keep minimum processes, but grow |
| to maximum if necessary). Note the value should be max_concurrency,min_concurrency |
| Pick these numbers based on resources on worker box and the nature of the task. |
| If autoscale option is available, worker_concurrency will be ignored. |
| <a class="reference external" href="http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale">http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">16,12</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__WORKER_AUTOSCALE</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">16,12</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="worker-log-server-port"> |
| <h3>worker_log_server_port<a class="headerlink" href="#worker-log-server-port" title="Permalink to this headline">¶</a></h3> |
| <p>When you start an airflow worker, airflow starts a tiny web server |
| subprocess to serve the workers local log files to the airflow main |
| web server, who then builds pages and sends them to users. This defines |
| the port on which the logs are served. It needs to be unused, and open |
| visible from the main web server to connect into the workers.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">8793</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__WORKER_LOG_SERVER_PORT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="broker-url"> |
| <h3>broker_url<a class="headerlink" href="#broker-url" title="Permalink to this headline">¶</a></h3> |
| <p>The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally |
| a sqlalchemy database. Refer to the Celery documentation for more |
| information. |
| <a class="reference external" href="http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings">http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">sqla+mysql://airflow:airflow@localhost:3306/airflow</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__BROKER_URL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="result-backend"> |
| <h3>result_backend<a class="headerlink" href="#result-backend" title="Permalink to this headline">¶</a></h3> |
| <p>The Celery result_backend. When a job finishes, it needs to update the |
| metadata of the job. Therefore it will post a message on a message bus, |
| or insert it into a database (depending of the backend) |
| This status is used by the scheduler to update the state of the task |
| The use of a database is highly recommended |
| <a class="reference external" href="http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings">http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">db+mysql://airflow:airflow@localhost:3306/airflow</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__RESULT_BACKEND</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="flower-host"> |
| <h3>flower_host<a class="headerlink" href="#flower-host" title="Permalink to this headline">¶</a></h3> |
| <p>Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start |
| it <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">flower</span></code>. This defines the IP that Celery Flower runs on</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">0.0.0.0</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__FLOWER_HOST</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="flower-url-prefix"> |
| <h3>flower_url_prefix<a class="headerlink" href="#flower-url-prefix" title="Permalink to this headline">¶</a></h3> |
| <p>The root URL for Flower</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__FLOWER_URL_PREFIX</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">/flower</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="flower-port"> |
| <h3>flower_port<a class="headerlink" href="#flower-port" title="Permalink to this headline">¶</a></h3> |
| <p>This defines the port that Celery Flower runs on</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">5555</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__FLOWER_PORT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="flower-basic-auth"> |
| <h3>flower_basic_auth<a class="headerlink" href="#flower-basic-auth" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <p>Securing Flower with Basic Authentication |
| Accepts user:password pairs separated by a comma</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__FLOWER_BASIC_AUTH</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">user1:password1,user2:password2</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="default-queue"> |
| <h3>default_queue<a class="headerlink" href="#default-queue" title="Permalink to this headline">¶</a></h3> |
| <p>Default queue that tasks get assigned to and that worker listen on.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">default</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__DEFAULT_QUEUE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="sync-parallelism"> |
| <h3>sync_parallelism<a class="headerlink" href="#sync-parallelism" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>How many processes CeleryExecutor uses to sync task state. |
| 0 means to use max(1, number of cores - 1) processes.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">0</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__SYNC_PARALLELISM</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="celery-config-options"> |
| <h3>celery_config_options<a class="headerlink" href="#celery-config-options" title="Permalink to this headline">¶</a></h3> |
| <p>Import path for celery configuration options</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__CELERY_CONFIG_OPTIONS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="ssl-active"> |
| <h3>ssl_active<a class="headerlink" href="#ssl-active" title="Permalink to this headline">¶</a></h3> |
| <p>In case of using SSL</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__SSL_ACTIVE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="ssl-key"> |
| <h3>ssl_key<a class="headerlink" href="#ssl-key" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__SSL_KEY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="ssl-cert"> |
| <h3>ssl_cert<a class="headerlink" href="#ssl-cert" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__SSL_CERT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="ssl-cacert"> |
| <h3>ssl_cacert<a class="headerlink" href="#ssl-cacert" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__SSL_CACERT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="pool"> |
| <h3>pool<a class="headerlink" href="#pool" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>Celery Pool implementation. |
| Choices include: prefork (default), eventlet, gevent or solo. |
| See: |
| <a class="reference external" href="https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency">https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency</a> |
| <a class="reference external" href="https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html">https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">prefork</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__POOL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="operation-timeout"> |
| <h3>operation_timeout<a class="headerlink" href="#operation-timeout" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>The number of seconds to wait before timing out <code class="docutils literal notranslate"><span class="pre">send_task_to_executor</span></code> or |
| <code class="docutils literal notranslate"><span class="pre">fetch_celery_task_state</span></code> operations.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>int</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">2</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__OPERATION_TIMEOUT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="celery-broker-transport-options"> |
| <h2><a class="toc-backref" href="#id17">celery_broker_transport_options</a><a class="headerlink" href="#celery-broker-transport-options" title="Permalink to this headline">¶</a></h2> |
| <p>This section is for specifying options which can be passed to the |
| underlying celery broker transport. See: |
| <a class="reference external" href="http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options">http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options</a></p> |
| <div class="section" id="visibility-timeout"> |
| <h3>visibility_timeout<a class="headerlink" href="#visibility-timeout" title="Permalink to this headline">¶</a></h3> |
| <p>The visibility timeout defines the number of seconds to wait for the worker |
| to acknowledge the task before the message is redelivered to another worker. |
| Make sure to increase the visibility timeout to match the time of the longest |
| ETA you’re planning to use. |
| visibility_timeout is only supported for Redis and SQS celery brokers. |
| See: |
| <a class="reference external" href="http://docs.celeryproject.org/en/master/userguide/configuration.html#std:setting-broker_transport_options">http://docs.celeryproject.org/en/master/userguide/configuration.html#std:setting-broker_transport_options</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY_BROKER_TRANSPORT_OPTIONS__VISIBILITY_TIMEOUT</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">21600</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="dask"> |
| <h2><a class="toc-backref" href="#id18">dask</a><a class="headerlink" href="#dask" title="Permalink to this headline">¶</a></h2> |
| <p>This section only applies if you are using the DaskExecutor in |
| [core] section above</p> |
| <div class="section" id="cluster-address"> |
| <h3>cluster_address<a class="headerlink" href="#cluster-address" title="Permalink to this headline">¶</a></h3> |
| <p>The IP address and port of the Dask cluster’s scheduler.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">127.0.0.1:8786</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__DASK__CLUSTER_ADDRESS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="tls-ca"> |
| <h3>tls_ca<a class="headerlink" href="#tls-ca" title="Permalink to this headline">¶</a></h3> |
| <p>TLS/ SSL settings to access a secured Dask scheduler.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__DASK__TLS_CA</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="tls-cert"> |
| <h3>tls_cert<a class="headerlink" href="#tls-cert" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__DASK__TLS_CERT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="tls-key"> |
| <h3>tls_key<a class="headerlink" href="#tls-key" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__DASK__TLS_KEY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="scheduler"> |
| <h2><a class="toc-backref" href="#id19">scheduler</a><a class="headerlink" href="#scheduler" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="job-heartbeat-sec"> |
| <h3>job_heartbeat_sec<a class="headerlink" href="#job-heartbeat-sec" title="Permalink to this headline">¶</a></h3> |
| <p>Task instances listen for external kill signal (when you clear tasks |
| from the CLI or the UI), this defines the frequency at which they should |
| listen (in seconds).</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">5</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__JOB_HEARTBEAT_SEC</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="scheduler-heartbeat-sec"> |
| <h3>scheduler_heartbeat_sec<a class="headerlink" href="#scheduler-heartbeat-sec" title="Permalink to this headline">¶</a></h3> |
| <p>The scheduler constantly tries to trigger new tasks (look at the |
| scheduler section in the docs for more information). This defines |
| how often the scheduler should run (in seconds).</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">5</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__SCHEDULER_HEARTBEAT_SEC</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="run-duration"> |
| <h3>run_duration<a class="headerlink" href="#run-duration" title="Permalink to this headline">¶</a></h3> |
| <p>After how much time should the scheduler terminate in seconds |
| -1 indicates to run continuously (see also num_runs)</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">-1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__RUN_DURATION</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="num-runs"> |
| <h3>num_runs<a class="headerlink" href="#num-runs" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.6.</span></p> |
| </div> |
| <p>The number of times to try to schedule each DAG file |
| -1 indicates unlimited number</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">-1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__NUM_RUNS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="processor-poll-interval"> |
| <h3>processor_poll_interval<a class="headerlink" href="#processor-poll-interval" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.6.</span></p> |
| </div> |
| <p>The number of seconds to wait between consecutive DAG file processing</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__PROCESSOR_POLL_INTERVAL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="min-file-process-interval"> |
| <h3>min_file_process_interval<a class="headerlink" href="#min-file-process-interval" title="Permalink to this headline">¶</a></h3> |
| <p>after how much time (seconds) a new DAGs should be picked up from the filesystem</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">0</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__MIN_FILE_PROCESS_INTERVAL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dag-dir-list-interval"> |
| <h3>dag_dir_list_interval<a class="headerlink" href="#dag-dir-list-interval" title="Permalink to this headline">¶</a></h3> |
| <p>How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">300</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="print-stats-interval"> |
| <h3>print_stats_interval<a class="headerlink" href="#print-stats-interval" title="Permalink to this headline">¶</a></h3> |
| <p>How often should stats be printed to the logs. Setting to 0 will disable printing stats</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">30</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__PRINT_STATS_INTERVAL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="scheduler-health-check-threshold"> |
| <h3>scheduler_health_check_threshold<a class="headerlink" href="#scheduler-health-check-threshold" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <p>If the last scheduler heartbeat happened more than scheduler_health_check_threshold |
| ago (in seconds), scheduler is considered unhealthy. |
| This is used by the health check in the “/health” endpoint</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">30</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__SCHEDULER_HEALTH_CHECK_THRESHOLD</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="child-process-log-directory"> |
| <h3>child_process_log_directory<a class="headerlink" href="#child-process-log-directory" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">{AIRFLOW_HOME}/logs/scheduler</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="scheduler-zombie-task-threshold"> |
| <h3>scheduler_zombie_task_threshold<a class="headerlink" href="#scheduler-zombie-task-threshold" title="Permalink to this headline">¶</a></h3> |
| <p>Local task jobs periodically heartbeat to the DB. If the job has |
| not heartbeat in this many seconds, the scheduler will mark the |
| associated task instance as failed and will re-schedule the task.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">300</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__SCHEDULER_ZOMBIE_TASK_THRESHOLD</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="catchup-by-default"> |
| <h3>catchup_by_default<a class="headerlink" href="#catchup-by-default" title="Permalink to this headline">¶</a></h3> |
| <p>Turn off scheduler catchup by setting this to False. |
| Default behavior is unchanged and |
| Command Line Backfills still work, but the scheduler |
| will not do scheduler catchup if this is False, |
| however it can be set on a per DAG basis in the |
| DAG definition (catchup)</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__CATCHUP_BY_DEFAULT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="max-tis-per-query"> |
| <h3>max_tis_per_query<a class="headerlink" href="#max-tis-per-query" title="Permalink to this headline">¶</a></h3> |
| <p>This changes the batch size of queries in the scheduling main loop. |
| If this is too high, SQL query performance may be impacted by one |
| or more of the following: |
| - reversion to full table scan |
| - complexity of query predicate |
| - excessive locking |
| Additionally, you may hit the maximum allowable query length for your db. |
| Set this to 0 for no limit (not advised)</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">512</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="statsd-on"> |
| <h3>statsd_on<a class="headerlink" href="#statsd-on" title="Permalink to this headline">¶</a></h3> |
| <p>Statsd (<a class="reference external" href="https://github.com/etsy/statsd">https://github.com/etsy/statsd</a>) integration settings</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__STATSD_ON</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="statsd-host"> |
| <h3>statsd_host<a class="headerlink" href="#statsd-host" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">localhost</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__STATSD_HOST</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="statsd-port"> |
| <h3>statsd_port<a class="headerlink" href="#statsd-port" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">8125</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__STATSD_PORT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="statsd-prefix"> |
| <h3>statsd_prefix<a class="headerlink" href="#statsd-prefix" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__STATSD_PREFIX</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="statsd-allow-list"> |
| <h3>statsd_allow_list<a class="headerlink" href="#statsd-allow-list" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.6.</span></p> |
| </div> |
| <p>If you want to avoid send all the available metrics to StatsD, |
| you can configure an allow list of prefixes to send only the metrics that |
| start with the elements of the list (e.g: scheduler,executor,dagrun)</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__STATSD_ALLOW_LIST</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="max-threads"> |
| <h3>max_threads<a class="headerlink" href="#max-threads" title="Permalink to this headline">¶</a></h3> |
| <p>The scheduler can run multiple threads in parallel to schedule dags. |
| This defines how many threads will run.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">2</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__MAX_THREADS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="id1"> |
| <h3>authenticate<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__AUTHENTICATE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="use-job-schedule"> |
| <h3>use_job_schedule<a class="headerlink" href="#use-job-schedule" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <p>Turn off scheduler use of cron intervals by setting this to False. |
| DAGs submitted manually in the web UI or with trigger_dag will still run.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__USE_JOB_SCHEDULE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="allow-trigger-in-future"> |
| <h3>allow_trigger_in_future<a class="headerlink" href="#allow-trigger-in-future" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>Allow externally triggered DagRuns for Execution Dates in the future |
| Only has effect if schedule_interval is set to None in DAG</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__SCHEDULER__ALLOW_TRIGGER_IN_FUTURE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="ldap"> |
| <h2><a class="toc-backref" href="#id20">ldap</a><a class="headerlink" href="#ldap" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="uri"> |
| <h3>uri<a class="headerlink" href="#uri" title="Permalink to this headline">¶</a></h3> |
| <p>set this to ldaps://<your.ldap.server>:<port></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LDAP__URI</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="user-filter"> |
| <h3>user_filter<a class="headerlink" href="#user-filter" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">objectClass=*</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LDAP__USER_FILTER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="user-name-attr"> |
| <h3>user_name_attr<a class="headerlink" href="#user-name-attr" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">uid</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LDAP__USER_NAME_ATTR</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="group-member-attr"> |
| <h3>group_member_attr<a class="headerlink" href="#group-member-attr" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">memberOf</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LDAP__GROUP_MEMBER_ATTR</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="superuser-filter"> |
| <h3>superuser_filter<a class="headerlink" href="#superuser-filter" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LDAP__SUPERUSER_FILTER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="data-profiler-filter"> |
| <h3>data_profiler_filter<a class="headerlink" href="#data-profiler-filter" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LDAP__DATA_PROFILER_FILTER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="bind-user"> |
| <h3>bind_user<a class="headerlink" href="#bind-user" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">cn=Manager,dc=example,dc=com</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LDAP__BIND_USER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="bind-password"> |
| <h3>bind_password<a class="headerlink" href="#bind-password" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">insecure</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LDAP__BIND_PASSWORD</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="basedn"> |
| <h3>basedn<a class="headerlink" href="#basedn" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">dc=example,dc=com</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LDAP__BASEDN</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="cacert"> |
| <h3>cacert<a class="headerlink" href="#cacert" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">/etc/ca/ldap_ca.crt</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LDAP__CACERT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="search-scope"> |
| <h3>search_scope<a class="headerlink" href="#search-scope" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">LEVEL</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LDAP__SEARCH_SCOPE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="ignore-malformed-schema"> |
| <h3>ignore_malformed_schema<a class="headerlink" href="#ignore-malformed-schema" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>This setting allows the use of LDAP servers that either return a |
| broken schema, or do not return a schema.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__LDAP__IGNORE_MALFORMED_SCHEMA</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="mesos"> |
| <h2><a class="toc-backref" href="#id21">mesos</a><a class="headerlink" href="#mesos" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="master"> |
| <h3>master<a class="headerlink" href="#master" title="Permalink to this headline">¶</a></h3> |
| <p>Mesos master address which MesosExecutor will connect to.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">localhost:5050</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__MESOS__MASTER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="framework-name"> |
| <h3>framework_name<a class="headerlink" href="#framework-name" title="Permalink to this headline">¶</a></h3> |
| <p>The framework name which Airflow scheduler will register itself as on mesos</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">Airflow</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__MESOS__FRAMEWORK_NAME</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="task-cpu"> |
| <h3>task_cpu<a class="headerlink" href="#task-cpu" title="Permalink to this headline">¶</a></h3> |
| <p>Number of cpu cores required for running one task instance using |
| ‘airflow run <dag_id> <task_id> <execution_date> –local -p <pickle_id>’ |
| command on a mesos slave</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>int</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__MESOS__TASK_CPU</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="task-memory"> |
| <h3>task_memory<a class="headerlink" href="#task-memory" title="Permalink to this headline">¶</a></h3> |
| <p>Memory in MB required for running one task instance using |
| ‘airflow run <dag_id> <task_id> <execution_date> –local -p <pickle_id>’ |
| command on a mesos slave</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">256</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__MESOS__TASK_MEMORY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="checkpoint"> |
| <h3>checkpoint<a class="headerlink" href="#checkpoint" title="Permalink to this headline">¶</a></h3> |
| <p>Enable framework checkpointing for mesos |
| See <a class="reference external" href="http://mesos.apache.org/documentation/latest/slave-recovery/">http://mesos.apache.org/documentation/latest/slave-recovery/</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>boolean</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__MESOS__CHECKPOINT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="failover-timeout"> |
| <h3>failover_timeout<a class="headerlink" href="#failover-timeout" title="Permalink to this headline">¶</a></h3> |
| <p>Failover timeout in milliseconds. |
| When checkpointing is enabled and this option is set, Mesos waits |
| until the configured timeout for |
| the MesosExecutor framework to re-register after a failover. Mesos |
| shuts down running tasks if the |
| MesosExecutor framework fails to re-register within this timeframe.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>int</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__MESOS__FAILOVER_TIMEOUT</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">604800</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="id2"> |
| <h3>authenticate<a class="headerlink" href="#id2" title="Permalink to this headline">¶</a></h3> |
| <p>Enable framework authentication for mesos |
| See <a class="reference external" href="http://mesos.apache.org/documentation/latest/configuration/">http://mesos.apache.org/documentation/latest/configuration/</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>boolean</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__MESOS__AUTHENTICATE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="default-principal"> |
| <h3>default_principal<a class="headerlink" href="#default-principal" title="Permalink to this headline">¶</a></h3> |
| <p>Mesos credentials, if authentication is enabled</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>boolean</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__MESOS__DEFAULT_PRINCIPAL</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">admin</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="default-secret"> |
| <h3>default_secret<a class="headerlink" href="#default-secret" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>boolean</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__MESOS__DEFAULT_SECRET</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">admin</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="docker-image-slave"> |
| <h3>docker_image_slave<a class="headerlink" href="#docker-image-slave" title="Permalink to this headline">¶</a></h3> |
| <p>Optional Docker Image to run on slave before running the command |
| This image should be accessible from mesos slave i.e mesos slave |
| should be able to pull this docker image before executing the command.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>boolean</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__MESOS__DOCKER_IMAGE_SLAVE</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">puckel/docker-airflow</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="kerberos"> |
| <h2><a class="toc-backref" href="#id22">kerberos</a><a class="headerlink" href="#kerberos" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="ccache"> |
| <h3>ccache<a class="headerlink" href="#ccache" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">/tmp/airflow_krb5_ccache</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KERBEROS__CCACHE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="principal"> |
| <h3>principal<a class="headerlink" href="#principal" title="Permalink to this headline">¶</a></h3> |
| <p>gets augmented with fqdn</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KERBEROS__PRINCIPAL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="reinit-frequency"> |
| <h3>reinit_frequency<a class="headerlink" href="#reinit-frequency" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">3600</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KERBEROS__REINIT_FREQUENCY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="kinit-path"> |
| <h3>kinit_path<a class="headerlink" href="#kinit-path" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">kinit</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KERBEROS__KINIT_PATH</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="keytab"> |
| <h3>keytab<a class="headerlink" href="#keytab" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow.keytab</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KERBEROS__KEYTAB</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="github-enterprise"> |
| <h2><a class="toc-backref" href="#id23">github_enterprise</a><a class="headerlink" href="#github-enterprise" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="api-rev"> |
| <h3>api_rev<a class="headerlink" href="#api-rev" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">v3</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__GITHUB_ENTERPRISE__API_REV</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="admin"> |
| <h2><a class="toc-backref" href="#id24">admin</a><a class="headerlink" href="#admin" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="hide-sensitive-variable-fields"> |
| <h3>hide_sensitive_variable_fields<a class="headerlink" href="#hide-sensitive-variable-fields" title="Permalink to this headline">¶</a></h3> |
| <p>UI to hide sensitive variable fields when set to True</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ADMIN__HIDE_SENSITIVE_VARIABLE_FIELDS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="elasticsearch"> |
| <h2><a class="toc-backref" href="#id25">elasticsearch</a><a class="headerlink" href="#elasticsearch" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="id3"> |
| <h3>host<a class="headerlink" href="#id3" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>Elasticsearch host</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ELASTICSEARCH__HOST</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="log-id-template"> |
| <h3>log_id_template<a class="headerlink" href="#log-id-template" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>Format of the log_id, which is used to query for a given tasks logs</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">{{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}}</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ELASTICSEARCH__LOG_ID_TEMPLATE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="end-of-log-mark"> |
| <h3>end_of_log_mark<a class="headerlink" href="#end-of-log-mark" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>Used to mark the end of a log stream for a task</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">end_of_log</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ELASTICSEARCH__END_OF_LOG_MARK</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="frontend"> |
| <h3>frontend<a class="headerlink" href="#frontend" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>Qualified URL for an elasticsearch frontend (like Kibana) with a template argument for log_id |
| Code will construct log_id using the log_id template from the argument above. |
| NOTE: The code will prefix the <a class="reference external" href="https://">https://</a> automatically, don’t include that here.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ELASTICSEARCH__FRONTEND</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="write-stdout"> |
| <h3>write_stdout<a class="headerlink" href="#write-stdout" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>Write the task logs to the stdout of the worker, rather than the default files</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ELASTICSEARCH__WRITE_STDOUT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="json-format"> |
| <h3>json_format<a class="headerlink" href="#json-format" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>Instead of the default log formatter, write the log lines as JSON</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ELASTICSEARCH__JSON_FORMAT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="json-fields"> |
| <h3>json_fields<a class="headerlink" href="#json-fields" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>Log fields to also attach to the json output, if enabled</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">asctime,</span> <span class="pre">filename,</span> <span class="pre">lineno,</span> <span class="pre">levelname,</span> <span class="pre">message</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ELASTICSEARCH__JSON_FIELDS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="elasticsearch-configs"> |
| <h2><a class="toc-backref" href="#id26">elasticsearch_configs</a><a class="headerlink" href="#elasticsearch-configs" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="use-ssl"> |
| <h3>use_ssl<a class="headerlink" href="#use-ssl" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.5.</span></p> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ELASTICSEARCH_CONFIGS__USE_SSL</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="verify-certs"> |
| <h3>verify_certs<a class="headerlink" href="#verify-certs" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.5.</span></p> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__ELASTICSEARCH_CONFIGS__VERIFY_CERTS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="kubernetes"> |
| <h2><a class="toc-backref" href="#id27">kubernetes</a><a class="headerlink" href="#kubernetes" title="Permalink to this headline">¶</a></h2> |
| <div class="section" id="worker-container-repository"> |
| <h3>worker_container_repository<a class="headerlink" href="#worker-container-repository" title="Permalink to this headline">¶</a></h3> |
| <p>The repository, tag and imagePullPolicy of the Kubernetes Image for the Worker to Run</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__WORKER_CONTAINER_REPOSITORY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="worker-container-tag"> |
| <h3>worker_container_tag<a class="headerlink" href="#worker-container-tag" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__WORKER_CONTAINER_TAG</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="worker-container-image-pull-policy"> |
| <h3>worker_container_image_pull_policy<a class="headerlink" href="#worker-container-image-pull-policy" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">IfNotPresent</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__WORKER_CONTAINER_IMAGE_PULL_POLICY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="delete-worker-pods"> |
| <h3>delete_worker_pods<a class="headerlink" href="#delete-worker-pods" title="Permalink to this headline">¶</a></h3> |
| <p>If True (default), worker pods will be deleted upon termination</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__DELETE_WORKER_PODS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="worker-pods-creation-batch-size"> |
| <h3>worker_pods_creation_batch_size<a class="headerlink" href="#worker-pods-creation-batch-size" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>Number of Kubernetes Worker Pod creation calls per scheduler loop</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__WORKER_PODS_CREATION_BATCH_SIZE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="namespace"> |
| <h3>namespace<a class="headerlink" href="#namespace" title="Permalink to this headline">¶</a></h3> |
| <p>The Kubernetes namespace where airflow workers should be created. Defaults to <code class="docutils literal notranslate"><span class="pre">default</span></code></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">default</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__NAMESPACE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="airflow-configmap"> |
| <h3>airflow_configmap<a class="headerlink" href="#airflow-configmap" title="Permalink to this headline">¶</a></h3> |
| <p>The name of the Kubernetes ConfigMap containing the Airflow Configuration (this file)</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__AIRFLOW_CONFIGMAP</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow-configmap</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="airflow-local-settings-configmap"> |
| <h3>airflow_local_settings_configmap<a class="headerlink" href="#airflow-local-settings-configmap" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.8.</span></p> |
| </div> |
| <p>The name of the Kubernetes ConfigMap containing <code class="docutils literal notranslate"><span class="pre">airflow_local_settings.py</span></code> file.</p> |
| <p>For example:</p> |
| <p><code class="docutils literal notranslate"><span class="pre">airflow_local_settings_configmap</span> <span class="pre">=</span> <span class="pre">"airflow-configmap"</span></code> if you have the following ConfigMap.</p> |
| <p><code class="docutils literal notranslate"><span class="pre">airflow-configmap.yaml</span></code>:</p> |
| <div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nn">---</span> |
| <span class="nt">apiVersion</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">v1</span> |
| <span class="nt">kind</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">ConfigMap</span> |
| <span class="nt">metadata</span><span class="p">:</span> |
| <span class="nt">name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">airflow-configmap</span> |
| <span class="nt">data</span><span class="p">:</span> |
| <span class="nt">airflow_local_settings.py</span><span class="p">:</span> <span class="p p-Indicator">|</span> |
| <span class="no">def pod_mutation_hook(pod):</span> |
| <span class="no">...</span> |
| <span class="nt">airflow.cfg</span><span class="p">:</span> <span class="p p-Indicator">|</span> |
| <span class="no">...</span> |
| </pre></div> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__AIRFLOW_LOCAL_SETTINGS_CONFIGMAP</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow-configmap</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dags-in-image"> |
| <h3>dags_in_image<a class="headerlink" href="#dags-in-image" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <p>For docker image already contains DAGs, this is set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, and the worker will |
| search for dags in dags_folder, |
| otherwise use git sync or dags volume claim to mount DAGs</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">False</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__DAGS_IN_IMAGE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dags-volume-subpath"> |
| <h3>dags_volume_subpath<a class="headerlink" href="#dags-volume-subpath" title="Permalink to this headline">¶</a></h3> |
| <p>For either git sync or volume mounted DAGs, the worker will look in this subpath for DAGs</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__DAGS_VOLUME_SUBPATH</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dags-volume-claim"> |
| <h3>dags_volume_claim<a class="headerlink" href="#dags-volume-claim" title="Permalink to this headline">¶</a></h3> |
| <p>For DAGs mounted via a volume claim (mutually exclusive with git-sync and host path)</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__DAGS_VOLUME_CLAIM</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="logs-volume-subpath"> |
| <h3>logs_volume_subpath<a class="headerlink" href="#logs-volume-subpath" title="Permalink to this headline">¶</a></h3> |
| <p>For volume mounted logs, the worker will look in this subpath for logs</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__LOGS_VOLUME_SUBPATH</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="logs-volume-claim"> |
| <h3>logs_volume_claim<a class="headerlink" href="#logs-volume-claim" title="Permalink to this headline">¶</a></h3> |
| <p>A shared volume claim for the logs</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__LOGS_VOLUME_CLAIM</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="dags-volume-host"> |
| <h3>dags_volume_host<a class="headerlink" href="#dags-volume-host" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <p>For DAGs mounted via a hostPath volume (mutually exclusive with volume claim and git-sync) |
| Useful in local environment, discouraged in production</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__DAGS_VOLUME_HOST</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="logs-volume-host"> |
| <h3>logs_volume_host<a class="headerlink" href="#logs-volume-host" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <p>A hostPath volume for the logs |
| Useful in local environment, discouraged in production</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__LOGS_VOLUME_HOST</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="env-from-configmap-ref"> |
| <h3>env_from_configmap_ref<a class="headerlink" href="#env-from-configmap-ref" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>A list of configMapsRefs to envFrom. If more than one configMap is |
| specified, provide a comma separated list: configmap_a,configmap_b</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="env-from-secret-ref"> |
| <h3>env_from_secret_ref<a class="headerlink" href="#env-from-secret-ref" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>A list of secretRefs to envFrom. If more than one secret is |
| specified, provide a comma separated list: secret_a,secret_b</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__ENV_FROM_SECRET_REF</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-repo"> |
| <h3>git_repo<a class="headerlink" href="#git-repo" title="Permalink to this headline">¶</a></h3> |
| <p>Git credentials and repository for DAGs mounted via Git (mutually exclusive with volume claim)</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_REPO</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-branch"> |
| <h3>git_branch<a class="headerlink" href="#git-branch" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_BRANCH</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-subpath"> |
| <h3>git_subpath<a class="headerlink" href="#git-subpath" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_SUBPATH</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-sync-rev"> |
| <h3>git_sync_rev<a class="headerlink" href="#git-sync-rev" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.7.</span></p> |
| </div> |
| <p>The specific rev or hash the git_sync init container will checkout |
| This becomes GIT_SYNC_REV environment variable in the git_sync init container for worker pods</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_SYNC_REV</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-user"> |
| <h3>git_user<a class="headerlink" href="#git-user" title="Permalink to this headline">¶</a></h3> |
| <p>Use git_user and git_password for user authentication or git_ssh_key_secret_name |
| and git_ssh_key_secret_key for SSH authentication</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_USER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-password"> |
| <h3>git_password<a class="headerlink" href="#git-password" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_PASSWORD</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-sync-root"> |
| <h3>git_sync_root<a class="headerlink" href="#git-sync-root" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">/git</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_SYNC_ROOT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-sync-dest"> |
| <h3>git_sync_dest<a class="headerlink" href="#git-sync-dest" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">repo</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_SYNC_DEST</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-dags-folder-mount-point"> |
| <h3>git_dags_folder_mount_point<a class="headerlink" href="#git-dags-folder-mount-point" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <p>Mount point of the volume if git-sync is being used. |
| i.e. {AIRFLOW_HOME}/dags</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_DAGS_FOLDER_MOUNT_POINT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-ssh-key-secret-name"> |
| <h3>git_ssh_key_secret_name<a class="headerlink" href="#git-ssh-key-secret-name" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>To get Git-sync SSH authentication set up follow this format</p> |
| <p><code class="docutils literal notranslate"><span class="pre">airflow-secrets.yaml</span></code>:</p> |
| <div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nn">---</span> |
| <span class="nt">apiVersion</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">v1</span> |
| <span class="nt">kind</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">Secret</span> |
| <span class="nt">metadata</span><span class="p">:</span> |
| <span class="nt">name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">airflow-secrets</span> |
| <span class="nt">data</span><span class="p">:</span> |
| <span class="c1"># key needs to be gitSshKey</span> |
| <span class="nt">gitSshKey</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain"><base64_encoded_data></span> |
| </pre></div> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_SSH_KEY_SECRET_NAME</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow-secrets</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-ssh-known-hosts-configmap-name"> |
| <h3>git_ssh_known_hosts_configmap_name<a class="headerlink" href="#git-ssh-known-hosts-configmap-name" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>To get Git-sync SSH authentication set up follow this format</p> |
| <p><code class="docutils literal notranslate"><span class="pre">airflow-configmap.yaml</span></code>:</p> |
| <div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nn">---</span> |
| <span class="nt">apiVersion</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">v1</span> |
| <span class="nt">kind</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">ConfigMap</span> |
| <span class="nt">metadata</span><span class="p">:</span> |
| <span class="nt">name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">airflow-configmap</span> |
| <span class="nt">data</span><span class="p">:</span> |
| <span class="nt">known_hosts</span><span class="p">:</span> <span class="p p-Indicator">|</span> |
| <span class="no">github.com ssh-rsa <...></span> |
| <span class="nt">airflow.cfg</span><span class="p">:</span> <span class="p p-Indicator">|</span> |
| <span class="no">...</span> |
| </pre></div> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_SSH_KNOWN_HOSTS_CONFIGMAP_NAME</span></code></p> |
| </dd> |
| <dt class="field-even">Example</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">airflow-configmap</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-sync-credentials-secret"> |
| <h3>git_sync_credentials_secret<a class="headerlink" href="#git-sync-credentials-secret" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.5.</span></p> |
| </div> |
| <p>To give the git_sync init container credentials via a secret, create a secret |
| with two fields: GIT_SYNC_USERNAME and GIT_SYNC_PASSWORD (example below) and |
| add <code class="docutils literal notranslate"><span class="pre">git_sync_credentials_secret</span> <span class="pre">=</span> <span class="pre"><secret_name></span></code> to your airflow config under the |
| <code class="docutils literal notranslate"><span class="pre">kubernetes</span></code> section</p> |
| <p>Secret Example:</p> |
| <div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nn">---</span> |
| <span class="nt">apiVersion</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">v1</span> |
| <span class="nt">kind</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">Secret</span> |
| <span class="nt">metadata</span><span class="p">:</span> |
| <span class="nt">name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">git-credentials</span> |
| <span class="nt">data</span><span class="p">:</span> |
| <span class="nt">GIT_SYNC_USERNAME</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain"><base64_encoded_git_username></span> |
| <span class="nt">GIT_SYNC_PASSWORD</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain"><base64_encoded_git_password></span> |
| </pre></div> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_SYNC_CREDENTIALS_SECRET</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-sync-container-repository"> |
| <h3>git_sync_container_repository<a class="headerlink" href="#git-sync-container-repository" title="Permalink to this headline">¶</a></h3> |
| <p>For cloning DAGs from git repositories into volumes: <a class="reference external" href="https://github.com/kubernetes/git-sync">https://github.com/kubernetes/git-sync</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">k8s.gcr.io/git-sync</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_SYNC_CONTAINER_REPOSITORY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-sync-container-tag"> |
| <h3>git_sync_container_tag<a class="headerlink" href="#git-sync-container-tag" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">v3.1.1</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_SYNC_CONTAINER_TAG</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-sync-init-container-name"> |
| <h3>git_sync_init_container_name<a class="headerlink" href="#git-sync-init-container-name" title="Permalink to this headline">¶</a></h3> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">git-sync-clone</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_SYNC_INIT_CONTAINER_NAME</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="git-sync-run-as-user"> |
| <h3>git_sync_run_as_user<a class="headerlink" href="#git-sync-run-as-user" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.5.</span></p> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">65533</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GIT_SYNC_RUN_AS_USER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="worker-service-account-name"> |
| <h3>worker_service_account_name<a class="headerlink" href="#worker-service-account-name" title="Permalink to this headline">¶</a></h3> |
| <p>The name of the Kubernetes service account to be associated with airflow workers, if any. |
| Service accounts are required for workers that require access to secrets or cluster resources. |
| See the Kubernetes RBAC documentation for more: |
| <a class="reference external" href="https://kubernetes.io/docs/admin/authorization/rbac/">https://kubernetes.io/docs/admin/authorization/rbac/</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__WORKER_SERVICE_ACCOUNT_NAME</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="image-pull-secrets"> |
| <h3>image_pull_secrets<a class="headerlink" href="#image-pull-secrets" title="Permalink to this headline">¶</a></h3> |
| <p>Any image pull secrets to be given to worker pods, If more than one secret is |
| required, provide a comma separated list: secret_a,secret_b</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__IMAGE_PULL_SECRETS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="gcp-service-account-keys"> |
| <h3>gcp_service_account_keys<a class="headerlink" href="#gcp-service-account-keys" title="Permalink to this headline">¶</a></h3> |
| <p>GCP Service Account Keys to be provided to tasks run on Kubernetes Executors |
| Should be supplied in the format: key-name-1:key-path-1,key-name-2:key-path-2</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__GCP_SERVICE_ACCOUNT_KEYS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="in-cluster"> |
| <h3>in_cluster<a class="headerlink" href="#in-cluster" title="Permalink to this headline">¶</a></h3> |
| <p>Use the service account kubernetes gives to pods to connect to kubernetes cluster. |
| It’s intended for clients that expect to be running inside a pod running on kubernetes. |
| It will raise an exception if called from a process not running in a kubernetes environment.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">True</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__IN_CLUSTER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="cluster-context"> |
| <h3>cluster_context<a class="headerlink" href="#cluster-context" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>When running with in_cluster=False change the default cluster_context or config_file |
| options to Kubernetes client. Leave blank these to use default behaviour like <code class="docutils literal notranslate"><span class="pre">kubectl</span></code> has.</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__CLUSTER_CONTEXT</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="config-file"> |
| <h3>config_file<a class="headerlink" href="#config-file" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__CONFIG_FILE</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="affinity"> |
| <h3>affinity<a class="headerlink" href="#affinity" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <p>Affinity configuration as a single line formatted JSON object. |
| See the affinity model for top-level key names (e.g. <code class="docutils literal notranslate"><span class="pre">nodeAffinity</span></code>, etc.): |
| <a class="reference external" href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#affinity-v1-core">https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#affinity-v1-core</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__AFFINITY</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="tolerations"> |
| <h3>tolerations<a class="headerlink" href="#tolerations" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.2.</span></p> |
| </div> |
| <p>A list of toleration objects as a single line formatted JSON array |
| See: |
| <a class="reference external" href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#toleration-v1-core">https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#toleration-v1-core</a></p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__TOLERATIONS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="kube-client-request-args"> |
| <h3>kube_client_request_args<a class="headerlink" href="#kube-client-request-args" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.4.</span></p> |
| </div> |
| <p>Keyword parameters to pass while calling a kubernetes client core_v1_api methods |
| from Kubernetes Executor provided as a single line formatted JSON dictionary string. |
| List of supported params are similar for all core_v1_apis, hence a single config |
| variable for all apis. |
| See: |
| <a class="reference external" href="https://raw.githubusercontent.com/kubernetes-client/python/master/kubernetes/client/apis/core_v1_api.py">https://raw.githubusercontent.com/kubernetes-client/python/master/kubernetes/client/apis/core_v1_api.py</a> |
| Note that if no _request_timeout is specified, the kubernetes client will wait indefinitely |
| for kubernetes api responses, which will cause the scheduler to hang. |
| The timeout is specified as [connect timeout, read timeout]</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">{{"_request_timeout"</span> <span class="pre">:</span> <span class="pre">[60,60]</span> <span class="pre">}}</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__KUBE_CLIENT_REQUEST_ARGS</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="run-as-user"> |
| <h3>run_as_user<a class="headerlink" href="#run-as-user" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>Specifies the uid to run the first process of the worker pods containers as</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__RUN_AS_USER</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| <div class="section" id="fs-group"> |
| <h3>fs_group<a class="headerlink" href="#fs-group" title="Permalink to this headline">¶</a></h3> |
| <div class="versionadded"> |
| <p><span class="versionmodified added">New in version 1.10.3.</span></p> |
| </div> |
| <p>Specifies a gid to associate with all containers in the worker pods |
| if using a git_ssh_key_secret_name use an fs_group |
| that allows for the key to be read, e.g. 65533</p> |
| <dl class="field-list simple"> |
| <dt class="field-odd">Type</dt> |
| <dd class="field-odd"><p>string</p> |
| </dd> |
| <dt class="field-even">Default</dt> |
| <dd class="field-even"><p><code class="docutils literal notranslate"><span class="pre">''</span></code></p> |
| </dd> |
| <dt class="field-odd">Environment Variable</dt> |
| <dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW__KUBERNETES__FS_GROUP</span></code></p> |
| </dd> |
| </dl> |
| </div> |
| </div> |
| <div class="section" id="kubernetes-node-selectors"> |
| <h2><a class="toc-backref" href="#id28">kubernetes_node_selectors</a><a class="headerlink" href="#kubernetes-node-selectors" title="Permalink to this headline">¶</a></h2> |
| <p>The Key-value pairs to be given to worker pods. |
| The worker pods will be scheduled to the nodes of the specified key-value pairs. |
| Should be supplied in the format: key = value</p> |
| </div> |
| <div class="section" id="kubernetes-annotations"> |
| <h2><a class="toc-backref" href="#id29">kubernetes_annotations</a><a class="headerlink" href="#kubernetes-annotations" title="Permalink to this headline">¶</a></h2> |
| <p>The Key-value annotations pairs to be given to worker pods. |
| Should be supplied in the format: key = value</p> |
| </div> |
| <div class="section" id="kubernetes-environment-variables"> |
| <h2><a class="toc-backref" href="#id30">kubernetes_environment_variables</a><a class="headerlink" href="#kubernetes-environment-variables" title="Permalink to this headline">¶</a></h2> |
| <p>The scheduler sets the following environment variables into your workers. You may define as |
| many environment variables as needed and the kubernetes launcher will set them in the launched workers. |
| Environment variables in this section are defined as follows |
| <code class="docutils literal notranslate"><span class="pre"><environment_variable_key></span> <span class="pre">=</span> <span class="pre"><environment_variable_value></span></code></p> |
| <p>For example if you wanted to set an environment variable with value <cite>prod</cite> and key |
| <code class="docutils literal notranslate"><span class="pre">ENVIRONMENT</span></code> you would follow the following format: |
| ENVIRONMENT = prod</p> |
| <p>Additionally you may override worker airflow settings with the <code class="docutils literal notranslate"><span class="pre">AIRFLOW__<SECTION>__<KEY></span></code> |
| formatting as supported by airflow normally.</p> |
| </div> |
| <div class="section" id="kubernetes-secrets"> |
| <h2><a class="toc-backref" href="#id31">kubernetes_secrets</a><a class="headerlink" href="#kubernetes-secrets" title="Permalink to this headline">¶</a></h2> |
| <p>The scheduler mounts the following secrets into your workers as they are launched by the |
| scheduler. You may define as many secrets as needed and the kubernetes launcher will parse the |
| defined secrets and mount them as secret environment variables in the launched workers. |
| Secrets in this section are defined as follows |
| <code class="docutils literal notranslate"><span class="pre"><environment_variable_mount></span> <span class="pre">=</span> <span class="pre"><kubernetes_secret_object>=<kubernetes_secret_key></span></code></p> |
| <p>For example if you wanted to mount a kubernetes secret key named <code class="docutils literal notranslate"><span class="pre">postgres_password</span></code> from the |
| kubernetes secret object <code class="docutils literal notranslate"><span class="pre">airflow-secret</span></code> as the environment variable <code class="docutils literal notranslate"><span class="pre">POSTGRES_PASSWORD</span></code> into |
| your workers you would follow the following format: |
| <code class="docutils literal notranslate"><span class="pre">POSTGRES_PASSWORD</span> <span class="pre">=</span> <span class="pre">airflow-secret=postgres_credentials</span></code></p> |
| <p>Additionally you may override worker airflow settings with the <code class="docutils literal notranslate"><span class="pre">AIRFLOW__<SECTION>__<KEY></span></code> |
| formatting as supported by airflow normally.</p> |
| </div> |
| <div class="section" id="kubernetes-labels"> |
| <h2><a class="toc-backref" href="#id32">kubernetes_labels</a><a class="headerlink" href="#kubernetes-labels" title="Permalink to this headline">¶</a></h2> |
| <p>The Key-value pairs to be given to worker pods. |
| The worker pods will be given these static labels, as well as some additional dynamic labels |
| to identify the task. |
| Should be supplied in the format: <code class="docutils literal notranslate"><span class="pre">key</span> <span class="pre">=</span> <span class="pre">value</span></code></p> |
| </div> |
| </div> |
| |
| |
| </div> |
| |
| </div> |
| |
| |
| <footer> |
| |
| <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> |
| |
| |
| <a href="_api/airflow/models/xcom/index.html" class="btn btn-neutral float-left" title="airflow.models.xcom" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> |
| |
| </div> |
| |
| |
| <hr/> |
| |
| <div role="contentinfo"> |
| <p> |
| |
| </p> |
| </div> |
| Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. |
| <div class="footer">This page uses <a href="https://analytics.google.com/"> |
| Google Analytics</a> to collect statistics. You can disable it by blocking |
| the JavaScript coming from www.google-analytics.com. Check our |
| <a href="privacy_notice.html">Privacy Policy</a> |
| for more details. |
| </div> |
| |
| |
| </footer> |
| |
| </div> |
| </div> |
| |
| </section> |
| |
| </div> |
| |
| |
| |
| <script type="text/javascript"> |
| jQuery(function () { |
| SphinxRtdTheme.Navigation.enable(true); |
| }); |
| </script> |
| |
| |
| |
| |
| <!-- Theme Analytics --> |
| <script> |
| (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ |
| (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), |
| m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) |
| })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); |
| |
| ga('create', 'UA-140539454-1', 'auto'); |
| ga('send', 'pageview'); |
| </script> |
| |
| |
| |
| |
| </body> |
| </html> |