| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| --- |
| # Default values for airflow. |
| # This is a YAML-formatted file. |
| # Declare variables to be passed into your templates. |
| |
| # User and group of airflow user |
| uid: 50000 |
| gid: 50000 |
| |
| # Airflow home directory |
| # Used for mount paths |
| airflowHome: "/opt/airflow" |
| |
| # Default airflow repository -- overrides all the specific images below |
| defaultAirflowRepository: apache/airflow |
| |
| # Default airflow tag to deploy |
| defaultAirflowTag: 1.10.12 |
| |
| |
| # Select certain nodes for airflow pods. |
| nodeSelector: {} |
| affinity: {} |
| tolerations: [] |
| |
| # Add common labels to all objects and pods defined in this chart. |
| labels: {} |
| |
| # Ingress configuration |
| ingress: |
| # Enable ingress resource |
| enabled: false |
| |
| # Configs for the Ingress of the web Service |
| web: |
| # Annotations for the web Ingress |
| annotations: {} |
| |
| # The path for the web Ingress |
| path: "" |
| |
| # The hostname for the web Ingress |
| host: "" |
| |
| # configs for web Ingress TLS |
| tls: |
| # Enable TLS termination for the web Ingress |
| enabled: false |
| # the name of a pre-created Secret containing a TLS private key and certificate |
| secretName: "" |
| |
| # HTTP paths to add to the web Ingress before the default path |
| precedingPaths: [] |
| |
| # Http paths to add to the web Ingress after the default path |
| succeedingPaths: [] |
| |
| # Configs for the Ingress of the flower Service |
| flower: |
| # Annotations for the flower Ingress |
| annotations: {} |
| |
| # The path for the flower Ingress |
| path: "" |
| |
| # The hostname for the flower Ingress |
| host: "" |
| |
| # configs for web Ingress TLS |
| tls: |
| # Enable TLS termination for the flower Ingress |
| enabled: false |
| # the name of a pre-created Secret containing a TLS private key and certificate |
| secretName: "" |
| |
| # HTTP paths to add to the flower Ingress before the default path |
| precedingPaths: [] |
| |
| # Http paths to add to the flower Ingress after the default path |
| succeedingPaths: [] |
| |
| # Network policy configuration |
| networkPolicies: |
| # Enabled network policies |
| enabled: false |
| |
| |
| # Extra annotations to apply to all |
| # Airflow pods |
| airflowPodAnnotations: {} |
| |
| # Enable RBAC (default on most clusters these days) |
| rbacEnabled: true |
| |
| # Airflow executor |
| # Options: SequentialExecutor, LocalExecutor, CeleryExecutor, KubernetesExecutor |
| executor: "KubernetesExecutor" |
| |
| # If this is true and using LocalExecutor/SequentialExecutor/KubernetesExecutor, the scheudler's |
| # service account will have access to communicate with the api-server and launch pods. |
| # If this is true and using the CeleryExecutor, the workers will be able to launch pods. |
| allowPodLaunching: true |
| |
| # Images |
| images: |
| airflow: |
| repository: ~ |
| tag: ~ |
| pullPolicy: IfNotPresent |
| flower: |
| repository: ~ |
| tag: ~ |
| pullPolicy: IfNotPresent |
| statsd: |
| repository: apache/airflow |
| tag: airflow-statsd-exporter-2020.09.05-v0.17.0 |
| pullPolicy: IfNotPresent |
| redis: |
| repository: redis |
| tag: 6-buster |
| pullPolicy: IfNotPresent |
| pgbouncer: |
| repository: apache/airflow |
| tag: airflow-pgbouncer-2020.09.05-1.14.0 |
| pullPolicy: IfNotPresent |
| pgbouncerExporter: |
| repository: apache/airflow |
| tag: airflow-pgbouncer-exporter-2020.09.25-0.5.0 |
| pullPolicy: IfNotPresent |
| |
| # Environment variables for all airflow containers |
| env: [] |
| # - name: "" |
| # value: "" |
| |
| # Secrets for all airflow containers |
| secret: [] |
| # - envName: "" |
| # secretName: "" |
| # secretKey: "" |
| |
| # Airflow database config |
| data: |
| # If secret names are provided, use those secrets |
| metadataSecretName: ~ |
| resultBackendSecretName: ~ |
| |
| # Otherwise pass connection values in |
| metadataConnection: |
| user: postgres |
| pass: postgres |
| host: ~ |
| port: 5432 |
| db: postgres |
| sslmode: disable |
| resultBackendConnection: |
| user: postgres |
| pass: postgres |
| host: ~ |
| port: 5432 |
| db: postgres |
| sslmode: disable |
| |
| # Fernet key settings |
| fernetKey: ~ |
| fernetKeySecretName: ~ |
| |
| # Airflow Worker Config |
| workers: |
| # Number of airflow celery workers in StatefulSet |
| replicas: 1 |
| |
| # Allow KEDA autoscaling. |
| # Persistence.enabled must be set to false to use KEDA. |
| keda: |
| enabled: false |
| namespaceLabels: {} |
| |
| # How often KEDA polls the airflow DB to report new scale requests to the HPA |
| pollingInterval: 5 |
| |
| # How many seconds KEDA will wait before scaling to zero. |
| # Note that HPA has a separate cooldown period for scale-downs |
| cooldownPeriod: 30 |
| |
| # Maximum number of workers created by keda |
| maxReplicaCount: 10 |
| |
| persistence: |
| # Enable persistent volumes |
| enabled: true |
| # Volume size for worker StatefulSet |
| size: 100Gi |
| # If using a custom storageClass, pass name ref to all statefulSets here |
| storageClassName: |
| # Execute init container to chown log directory. |
| # This is currently only needed in KinD, due to usage |
| # of local-path provisioner. |
| fixPermissions: false |
| |
| resources: {} |
| # limits: |
| # cpu: 100m |
| # memory: 128Mi |
| # requests: |
| # cpu: 100m |
| # memory: 128Mi |
| |
| # Grace period for tasks to finish after SIGTERM is sent from kubernetes |
| terminationGracePeriodSeconds: 600 |
| |
| # This setting tells kubernetes that its ok to evict |
| # when it wants to scale a node down. |
| safeToEvict: true |
| |
| # Airflow scheduler settings |
| scheduler: |
| # Scheduler pod disruption budget |
| podDisruptionBudget: |
| enabled: false |
| |
| # PDB configuration |
| config: |
| maxUnavailable: 1 |
| |
| resources: {} |
| # limits: |
| # cpu: 100m |
| # memory: 128Mi |
| # requests: |
| # cpu: 100m |
| # memory: 128Mi |
| |
| # This setting can overwrite |
| # podMutation settings. |
| airflowLocalSettings: ~ |
| |
| # This setting tells kubernetes that its ok to evict |
| # when it wants to scale a node down. |
| safeToEvict: true |
| |
| # Airflow webserver settings |
| webserver: |
| livenessProbe: |
| initialDelaySeconds: 15 |
| timeoutSeconds: 30 |
| failureThreshold: 20 |
| periodSeconds: 5 |
| |
| readinessProbe: |
| initialDelaySeconds: 15 |
| timeoutSeconds: 30 |
| failureThreshold: 20 |
| periodSeconds: 5 |
| |
| # Number of webservers |
| replicas: 1 |
| |
| # Additional network policies as needed |
| extraNetworkPolicies: [] |
| |
| resources: {} |
| # limits: |
| # cpu: 100m |
| # memory: 128Mi |
| # requests: |
| # cpu: 100m |
| # memory: 128Mi |
| |
| # Create initial user. |
| defaultUser: |
| enabled: true |
| role: Admin |
| username: admin |
| email: admin@example.com |
| firstName: admin |
| lastName: user |
| password: admin |
| |
| # Mount additional volumes into webserver. |
| extraVolumes: [] |
| extraVolumeMounts: [] |
| |
| # This will be mounted into the Airflow Webserver as a custom |
| # webserver_config.py. You can bake a webserver_config.py in to your image |
| # instead |
| webserverConfig: ~ |
| # webserverConfig: | |
| # from airflow import configuration as conf |
| |
| # # The SQLAlchemy connection string. |
| # SQLALCHEMY_DATABASE_URI = conf.get('core', 'SQL_ALCHEMY_CONN') |
| |
| # # Flask-WTF flag for CSRF |
| # CSRF_ENABLED = True |
| |
| service: |
| type: ClusterIP |
| ## service annotations |
| annotations: {} |
| |
| # Flower settings |
| flower: |
| # Additional network policies as needed |
| extraNetworkPolicies: [] |
| resources: {} |
| # limits: |
| # cpu: 100m |
| # memory: 128Mi |
| # requests: |
| # cpu: 100m |
| # memory: 128Mi |
| |
| service: |
| type: ClusterIP |
| |
| # Statsd settings |
| statsd: |
| enabled: true |
| # Additional network policies as needed |
| extraNetworkPolicies: [] |
| resources: {} |
| # limits: |
| # cpu: 100m |
| # memory: 128Mi |
| # requests: |
| # cpu: 100m |
| # memory: 128Mi |
| |
| service: |
| extraAnnotations: {} |
| |
| # Pgbouncer settings |
| pgbouncer: |
| # Enable pgbouncer |
| enabled: false |
| # Additional network policies as needed |
| extraNetworkPolicies: [] |
| |
| # Pool sizes |
| metadataPoolSize: 10 |
| resultBackendPoolSize: 5 |
| |
| # Maximum clients that can connect to pgbouncer (higher = more file descriptors) |
| maxClientConn: 100 |
| |
| # Pgbouner pod disruption budget |
| podDisruptionBudget: |
| enabled: false |
| |
| # PDB configuration |
| config: |
| maxUnavailable: 1 |
| |
| resources: {} |
| # limits: |
| # cpu: 100m |
| # memory: 128Mi |
| # requests: |
| # cpu: 100m |
| # memory: 128Mi |
| |
| service: |
| extraAnnotations: {} |
| |
| # https://www.pgbouncer.org/config.html |
| verbose: 0 |
| logDisconnections: 0 |
| logConnections: 0 |
| |
| redis: |
| terminationGracePeriodSeconds: 600 |
| |
| persistence: |
| # Enable persistent volumes |
| enabled: true |
| # Volume size for worker StatefulSet |
| size: 1Gi |
| # If using a custom storageClass, pass name ref to all statefulSets here |
| storageClassName: |
| |
| resources: {} |
| # limits: |
| # cpu: 100m |
| # memory: 128Mi |
| # requests: |
| # cpu: 100m |
| # memory: 128Mi |
| |
| # If set use as redis secret |
| passwordSecretName: ~ |
| brokerURLSecretName: ~ |
| |
| # Else, if password is set, create secret with it, |
| # else generate a new one on install |
| password: ~ |
| |
| # This setting tells kubernetes that its ok to evict |
| # when it wants to scale a node down. |
| safeToEvict: true |
| |
| # Auth secret for a private registry |
| # This is used if pulling airflow images from a private registry |
| registry: |
| secretName: ~ |
| |
| connection: {} |
| # user: ~ |
| # pass: ~ |
| # host: ~ |
| # email: ~ |
| |
| # Elasticsearch logging configuration |
| elasticsearch: |
| # Enable elasticsearch task logging |
| enabled: false |
| # A secret containing the connection |
| secretName: ~ |
| # Or an object representing the connection |
| connection: {} |
| # user: ~ |
| # pass: ~ |
| # host: ~ |
| # port: ~ |
| |
| # All ports used by chart |
| ports: |
| flowerUI: 5555 |
| airflowUI: 8080 |
| workerLogs: 8793 |
| redisDB: 6379 |
| statsdIngest: 9125 |
| statsdScrape: 9102 |
| pgbouncer: 6543 |
| pgbouncerScrape: 9127 |
| |
| # Define any ResourceQuotas for namespace |
| quotas: {} |
| |
| # Define default/max/min values for pods and containers in namespace |
| limits: [] |
| |
| # Config Settings for pod_mutation_hook |
| podMutation: |
| # Tolerations provided here would be applied using pod_mutation_hook |
| # So any pods spun up using KubernetesExecutor or KubernetesPodOperator will contain these tolerations. |
| tolerations: [] |
| # - key: "dynamic-pods" |
| # operator: "Equal" |
| # value: "true" |
| # effect: "NoSchedule" |
| |
| # Pods spun up would land in the node that matches the affinity |
| affinity: {} |
| # nodeAffinity: |
| # requiredDuringSchedulingIgnoredDuringExecution: |
| # nodeSelectorTerms: |
| # - matchExpressions: |
| # - key: "company.io/dynamic-pods" |
| # operator: In |
| # values: |
| # - "true" |
| |
| # This runs as a CronJob to cleanup old pods. |
| cleanup: |
| enabled: false |
| # Run every 15 minutes |
| schedule: "*/15 * * * *" |
| |
| # Configuration for postgresql subchart |
| # Not recommended for production |
| postgresql: |
| enabled: true |
| postgresqlPassword: postgres |
| postgresqlUsername: postgres |
| |
| # Config settings to go into the mounted airflow.cfg |
| # |
| # Please note that these values are passed through the `tpl` function, so are |
| # all subject to being rendered as go templates. If you need to include a |
| # litera `{{` in a value, it must be expessed like this: |
| # |
| # a: '{{ "{{ not a template }}" }}' |
| # |
| # yamllint disable rule:line-length |
| config: |
| core: |
| dags_folder: '{{ include "airflow_dags" . }}' |
| load_examples: 'False' |
| colored_console_log: 'False' |
| executor: '{{ .Values.executor }}' |
| remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}' |
| # Authentication backend used for the experimental API |
| api: |
| auth_backend: airflow.api.auth.backend.deny_all |
| logging: |
| logging_level: DEBUG |
| webserver: |
| enable_proxy_fix: 'True' |
| expose_config: 'True' |
| rbac: 'True' |
| |
| celery: |
| default_queue: celery |
| |
| scheduler: |
| scheduler_heartbeat_sec: 5 |
| statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}' |
| statsd_port: 9125 |
| statsd_prefix: airflow |
| statsd_host: '{{ printf "%s-statsd" .Release.Name }}' |
| |
| # Restart Scheduler every 41460 seconds (11 hours 31 minutes) |
| # The odd time is chosen so it is not always restarting on the same "hour" boundary |
| run_duration: 41460 |
| elasticsearch: |
| json_format: 'True' |
| log_id_template: "{dag_id}_{task_id}_{execution_date}_{try_number}" |
| elasticsearch_configs: |
| max_retries: 3 |
| timeout: 30 |
| retry_timeout: 'True' |
| |
| kubernetes: |
| namespace: '{{ .Release.Namespace }}' |
| airflow_configmap: '{{ include "airflow_config" . }}' |
| airflow_local_settings_configmap: '{{ include "airflow_config" . }}' |
| pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml' |
| worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}' |
| worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}' |
| delete_worker_pods: 'True' |
| # yamllint enable rule:line-length |
| |
| podTemplate: ~ |
| |
| # Git sync |
| dags: |
| persistence: |
| # Enable persistent volume for storing dags |
| enabled: false |
| # Volume size for dags |
| size: 1Gi |
| # If using a custom storageClass, pass name here |
| storageClassName: |
| # access mode of the persistent volume |
| accessMode: ReadWriteMany |
| ## the name of an existing PVC to use |
| existingClaim: ~ |
| gitSync: |
| enabled: false |
| # git repo clone url |
| # ssh examples ssh://git@github.com/apache/airflow.git |
| # git@github.com:apache/airflow.git |
| # https example: https://github.com/apache/airflow.git |
| repo: https://github.com/apache/airflow.git |
| branch: v1-10-stable |
| rev: HEAD |
| root: "/git" |
| dest: "repo" |
| depth: 1 |
| # the number of consecutive failures allowed before aborting |
| maxFailures: 0 |
| # subpath within the repo where dags are located |
| # should be "" if dags are at repo root |
| subPath: "tests/dags" |
| # if your repo needs a user name password |
| # you can load them to a k8s secret like the one below |
| # --- |
| # apiVersion: v1 |
| # kind: Secret |
| # metadata: |
| # name: git-credentials |
| # data: |
| # GIT_SYNC_USERNAME: <base64_encoded_git_username> |
| # GIT_SYNC_PASSWORD: <base64_encoded_git_password> |
| # and specify the name of the secret below |
| #credentialsSecret: git-credentials |
| # |
| # |
| # If you are using an ssh clone url, you can load |
| # the ssh private key to a k8s secret like the one below |
| # --- |
| # apiVersion: v1 |
| # kind: Secret |
| # metadata: |
| # name: airflow-ssh-secret |
| # data: |
| # # key needs to be gitSshKey |
| # gitSshKey: <base64_encoded_data> |
| # and specify the name of the secret below |
| #sshKeySecret: airflow-ssh-secret |
| # If you are using an ssh private key, you can additionally |
| # specify the content of your known_hosts file, example: |
| #knownHosts: | |
| # <host1>,<ip1> <key1> |
| # <host2>,<ip2> <key2> |
| # interval between git sync attempts in seconds |
| wait: 60 |
| # git sync image details |
| containerRepository: k8s.gcr.io/git-sync |
| containerTag: v3.1.6 |
| containerName: git-sync |
| uid: 65533 |