Remove pull_request_target workflow
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
deleted file mode 100644
index de0675b..0000000
--- a/.github/workflows/ci.yml
+++ /dev/null
@@ -1,1448 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
----
-name: Tests
-on: # yamllint disable-line rule:truthy
- push:
- branches: ['v[0-9]+-[0-9]+-test']
-permissions:
- contents: read
-env:
- MOUNT_SELECTED_LOCAL_SOURCES: "false"
- FORCE_ANSWER_TO_QUESTIONS: "yes"
- CHECK_IMAGE_FOR_REBUILD: "true"
- SKIP_CHECK_REMOTE_IMAGE: "true"
- DB_RESET: "true"
- VERBOSE: "true"
- DOCKER_CACHE: "pulled"
- GITHUB_REPOSITORY: ${{ github.repository }}
- GITHUB_USERNAME: ${{ github.actor }}
- # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
- # Airflow one is going to be used
- CONSTRAINTS_GITHUB_REPOSITORY: >-
- ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
- secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
- # In builds from forks, this token is read-only. For scheduler/direct push it is WRITE one
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- # In builds from forks, this token is empty, and this is good because such builds do not even try
- # to push images to the registry.
- GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
- GITHUB_REGISTRY_PUSH_IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
- INSTALL_PROVIDERS_FROM_SOURCES: "true"
- AIRFLOW_LOGIN_TO_GITHUB_REGISTRY: "true"
- ENABLE_TEST_COVERAGE: "${{ github.event_name == 'push' }}"
-
- # You can switch between building the image in "Build Images" workflow or building them in CI workflow
- # Separately for each job.
- #
- # a) Using images build in the separate "Build Images" workflow:
- #
- # GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true"
- #
- # b) Building images in CI workflow - separately for each job:
- #
- # GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
- #
- # You can also switch back to building images locally and disabling the "Build Images" workflow
- # by defining AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE secret with value set to "false"
- GITHUB_REGISTRY_WAIT_FOR_IMAGE: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }}
-
-concurrency:
- group: ci-${{ github.event.pull_request.number || github.ref }}
- cancel-in-progress: true
-
-jobs:
-
- build-info:
- name: "Build info"
- # The runs-on cannot refer to env. or secrets. context, so we have no
- # option but to specify a hard-coded list here. This is "safe", as the list
- # is checked again by the runner using it's own list, so a PR author cannot
- # change this and get access to our self-hosted runners
- #
- # When changing this list, ensure that it is kept in sync with the
- # /runners/apache/airflow/configOverlay
- # parameter in AWS SSM ParameterStore (which is what the runner uses)
- # and restart the self-hosted runners.
- #
- # This list of committers can be generated with:
- # https://github.com/apache/airflow-ci-infra/blob/main/scripts/list_committers
- runs-on: >-
- ${{ (
- (
- github.event_name == 'push' ||
- github.event_name == 'schedule' ||
- contains(fromJSON('[
- "BasPH",
- "Fokko",
- "KevinYang21",
- "XD-DENG",
- "aijamalnk",
- "alexvanboxel",
- "aoen",
- "artwr",
- "ashb",
- "bbovenzi",
- "bolkedebruin",
- "criccomini",
- "dimberman",
- "dstandish",
- "eladkal",
- "ephraimbuddy",
- "feluelle",
- "feng-tao",
- "houqp",
- "jedcunningham",
- "jgao54",
- "jghoman",
- "jhtimmins",
- "jmcarp",
- "kaxil",
- "leahecole",
- "mik-laj",
- "milton0825",
- "mistercrunch",
- "msumit",
- "potiuk",
- "r39132",
- "ryanahamilton",
- "ryw",
- "saguziel",
- "sekikn",
- "turbaszek",
- "uranusjr",
- "vikramkoka",
- "xinbinhuang",
- "yuqian90",
- "zhongjiajie"
- ]'), github.event.pull_request.user.login)
- ) && github.repository == 'apache/airflow'
- ) && 'self-hosted' || 'ubuntu-20.04' }}
- env:
- GITHUB_CONTEXT: ${{ toJson(github) }}
- outputs:
- cacheDirective: "pulled"
- targetBranch: ${{ steps.dynamic-outputs.outputs.targetBranch }}
- defaultBranch: ${{ steps.selective-checks.outputs.default-branch }}
- waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }}
- allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
- upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
- pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
- pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
- defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
- kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }}
- kubernetesVersionsListAsString: ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
- defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
- kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }}
- defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }}
- postgresVersions: ${{ steps.selective-checks.outputs.postgres-versions }}
- defaultPostgresVersion: ${{ steps.selective-checks.outputs.default-postgres-version }}
- mysqlVersions: ${{ steps.selective-checks.outputs.mysql-versions }}
- mssqlVersions: ${{ steps.selective-checks.outputs.mssql-versions }}
- defaultMySQLVersion: ${{ steps.selective-checks.outputs.default-mysql-version }}
- helmVersions: ${{ steps.selective-checks.outputs.helm-versions }}
- defaultHelmVersion: ${{ steps.selective-checks.outputs.default-helm-version }}
- kindVersions: ${{ steps.selective-checks.outputs.kind-versions }}
- defaultKindVersion: ${{ steps.selective-checks.outputs.default-kind-version }}
- testTypes: ${{ steps.selective-checks.outputs.test-types }}
- postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
- mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
- mssqlExclude: ${{ steps.selective-checks.outputs.mssql-exclude }}
- sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
- run-tests: ${{ steps.selective-checks.outputs.run-tests }}
- run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
- run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
- basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
- image-build: ${{ steps.selective-checks.outputs.image-build }}
- docs-build: ${{ steps.selective-checks.outputs.docs-build }}
- needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
- needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
- needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }}
- default-branch: ${{ steps.selective-checks.outputs.default-branch }}
- pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
- pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
- runsOn: ${{ steps.set-runs-on.outputs.runsOn }}
- runCoverage: ${{ steps.set-run-coverage.outputs.runCoverage }}
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Get information about the PR"
- uses: ./.github/actions/get-workflow-origin
- id: source-run-info
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- - name: Set wait for image
- id: wait-for-image
- run: |
- if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == 'true' ]]; then
- echo "::set-output name=wait-for-image::true"
- else
- echo "::set-output name=wait-for-image::false"
- fi
- - name: Fetch incoming commit ${{ github.sha }} with its parent
- uses: actions/checkout@v2
- with:
- ref: ${{ github.sha }}
- fetch-depth: 2
- persist-credentials: false
- if: github.event_name == 'pull_request'
- - name: Selective checks
- id: selective-checks
- env:
- PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}"
- run: |
- if [[ ${GITHUB_EVENT_NAME} == "pull_request" ]]; then
- # Run selective checks
- ./scripts/ci/selective_ci_checks.sh "${GITHUB_SHA}"
- else
- # Run all checks
- ./scripts/ci/selective_ci_checks.sh
- fi
- # Avoid having to specify the runs-on logic every time. We use the custom
- # env var AIRFLOW_SELF_HOSTED_RUNNER set only on our runners, but never
- # on the public runners
- - name: Set runs-on
- id: set-runs-on
- env:
- PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}"
- run: |
- if [[ ${PR_LABELS=} == *"use public runners"* ]]; then
- echo "Forcing running on Public Runners via `use public runners` label"
- echo "::set-output name=runsOn::\"ubuntu-20.04\""
- elif [[ ${AIRFLOW_SELF_HOSTED_RUNNER} == "" ]]; then
- echo "Regular PR running with Public Runner"
- echo "::set-output name=runsOn::\"ubuntu-20.04\""
- else
- echo "Maintainer or main run running with self-hosted runner"
- echo "::set-output name=runsOn::\"self-hosted\""
- fi
- # Avoid having to specify the coverage logic every time.
- - name: Set run coverage
- id: set-run-coverage
- run: |
- echo "::set-output name=runCoverage::true"
- if: >
- github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
- github.event_name == 'push' &&
- steps.selective-checks.outputs.default-branch == 'main'
-
- build-ci-images:
- permissions:
- packages: write
- timeout-minutes: 80
- name: "Build CI images ${{matrix.python-version}}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info]
- strategy:
- matrix:
- # We need to attempt to build all possible versions here because pull_request_target
- # event is run for both main and v1-10-tests
- python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }}
- fail-fast: true
- if: needs.build-info.outputs.image-build == 'true'
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
- BACKEND: postgres
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
- GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
- GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
- outputs: ${{toJSON(needs.build-info.outputs) }}
- steps:
- - uses: actions/checkout@v2
- with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
- persist-credentials: false
- - name: "Retrieve DEFAULTS"
- id: defaults
- run: |
- echo "DEFAULT_BRANCH=v2-2-test" >> $GITHUB_ENV
- echo "DEFAULT_CONSTRAINTS_BRANCH=constraints-2-2" >> $GITHUB_ENV
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Build CI images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Push CI images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_push_ci_images.sh
-
- build-prod-images:
- permissions:
- packages: write
- timeout-minutes: 80
- name: "Build PROD images ${{matrix.python-version}}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, build-ci-images]
- strategy:
- matrix:
- # We need to attempt to build all possible versions here because pull_request_target
- # event is run for both main and v1-10-tests
- python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }}
- fail-fast: true
- if: needs.build-info.outputs.image-build == 'true'
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
- BACKEND: postgres
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
- VERSION_SUFFIX_FOR_PYPI: ".dev0"
- GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
- GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
- INSTALL_PROVIDERS_FROM_SOURCES: >
- ${{ needs.build-info.outputs.defaultBranch == 'main' && 'true' || 'false' }}
- steps:
- - uses: actions/checkout@v2
- with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
- persist-credentials: false
- submodules: recursive
- - name: "Retrieve DEFAULTS"
- id: defaults
- run: |
- echo "DEFAULT_BRANCH=v2-2-test" >> $GITHUB_ENV
- echo "DEFAULT_CONSTRAINTS_BRANCH=constraints-2-2" >> $GITHUB_ENV
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Build CI images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- # Pull images built in the previous step (so GITHUB_REGISTRY_PULL_IMAGE_TAG needs to be overridden)
- env:
- GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true"
- GITHUB_REGISTRY_PULL_IMAGE_TAG: ${{ github.event.pull_request.head.sha || github.sha }}
- - name: "Build PROD images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- - name: "Push PROD images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_push_production_images.sh
-
- tests-ui:
- timeout-minutes: 10
- name: React UI tests
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info]
- if: >
- needs.build-info.outputs.run-ui-tests == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup node"
- uses: actions/setup-node@v2
- with:
- node-version: 14
- - name: "Cache eslint"
- uses: actions/cache@v2
- with:
- path: 'airflow/ui/node_modules'
- key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }}
- - run: yarn --cwd airflow/ui/ install --frozen-lockfile --non-interactive
- - run: yarn --cwd airflow/ui/ run test
-
-
- test-openapi-client-generation:
- timeout-minutes: 10
- name: "Test OpenAPI client generation"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info]
- if: needs.build-info.outputs.needs-api-codegen == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- fetch-depth: 2
- persist-credentials: false
- - name: "Generate client codegen diff"
- run: ./scripts/ci/openapi/client_codegen_diff.sh
-
- test-examples-of-prod-image-building:
- timeout-minutes: 60
- name: "Test examples of production image building"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info]
- if: needs.build-info.outputs.image-build == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- fetch-depth: 2
- persist-credentials: false
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- if: |
- needs.build-info.outputs.waitForImage == 'true'
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- - name: "Cache virtualenv environmnent"
- uses: actions/cache@v2
- with:
- path: '.build/.docker_venv'
- key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }}
- - name: "Test examples of PROD image building"
- run: ./scripts/ci/images/ci_test_examples_of_prod_image_building.sh
-
- test-docker-compose-quick-start:
- timeout-minutes: 60
- name: "Test docker-compose quick start"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, prod-images]
- if: needs.build-info.outputs.image-build == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- fetch-depth: 2
- persist-credentials: false
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- if: |
- needs.build-info.outputs.waitForImage == 'true'
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- - name: "Cache virtualenv environmnent"
- uses: actions/cache@v2
- with:
- path: '.build/.docker_venv'
- key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }}
- - name: "Test docker-compose quick start"
- run: ./scripts/ci/images/ci_run_docker_compose_quick_start_test.sh
-
- ci-images:
- timeout-minutes: 120
- name: "Wait for CI images"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, build-ci-images]
- if: needs.build-info.outputs.image-build == 'true'
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: sqlite
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- WAIT_FOR_IMAGE: ${{ needs.build-info.outputs.waitForImage }}
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- if: needs.build-info.outputs.waitForImage == 'true'
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- if: needs.build-info.outputs.waitForImage == 'true'
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- if: |
- needs.build-info.outputs.waitForImage == 'true'
- - name: "Cache virtualenv environmnent"
- uses: actions/cache@v2
- with:
- path: '.build/.docker_venv'
- key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }}
- - name: >
- Wait for CI images
- ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
- id: wait-for-images
- # We wait for the images to be available either from the build-ci-image step or from
- # "build-images-workflow-run.yml' run as pull_request_target.
- # We are utilising single job to wait for all images because this job merely waits
- # for the images to be available.
- # The test jobs wait for it to complete if WAIT_FOR_IMAGE is 'true'!
- run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh
- env:
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
- VERIFY_IMAGE: "true"
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
-
- static-checks:
- timeout-minutes: 30
- name: "Static checks"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- SKIP: "identity"
- MOUNT_SELECTED_LOCAL_SOURCES: "true"
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- if: needs.build-info.outputs.basic-checks-only == 'false'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Get Python version"
- run: "echo \"::set-output name=host-python-version::$(python -c
- 'import platform; print(platform.python_version())')\""
- id: host-python-version
- - name: "Cache pre-commit local-installation"
- uses: actions/cache@v2
- with:
- path: ~/.local
- key: "pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('setup.py', 'setup.cfg') }}"
- restore-keys: "\
-pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-"
- - name: "Cache pre-commit envs"
- uses: actions/cache@v2
- with:
- path: ~/.cache/pre-commit
- key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('.pre-commit-config.yaml') }}"
- restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
-
- - name: "Cache eslint"
- uses: actions/cache@v2
- with:
- path: 'airflow/ui/node_modules'
- key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }}
- - name: "Static checks"
- run: ./scripts/ci/static_checks/run_static_checks.sh
- env:
- VERBOSE: false
-
- # Those checks are run if no image needs to be built for checks. This is for simple changes that
- # Do not touch any of the python code or any of the important files that might require building
- # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host
- static-checks-basic-checks-only:
- timeout-minutes: 30
- name: "Static checks: basic checks only"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- SKIP: "build,mypy,flake8,bats-in-container-tests,identity"
- MOUNT_SELECTED_LOCAL_SOURCES: "true"
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- if: needs.build-info.outputs.basic-checks-only == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- - name: >
- Fetch incoming commit ${{ github.sha }} with its parent
- uses: actions/checkout@v2
- with:
- ref: ${{ github.sha }}
- fetch-depth: 2
- persist-credentials: false
- - name: "Get Python version"
- run: "echo \"::set-output name=host-python-version::$(python -c
- 'import platform; print(platform.python_version())')\""
- id: host-python-version
- - name: "Cache pre-commit local-installation"
- uses: actions/cache@v2
- with:
- path: ~/.local
- key: "pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('setup.py', 'setup.cfg') }}"
- restore-keys: "\
-pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-"
- - name: "Cache pre-commit envs"
- uses: actions/cache@v2
- with:
- path: ~/.cache/pre-commit
- key: "pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}}-\
-${{ hashFiles('.pre-commit-config.yaml') }}"
- restore-keys: pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}}
- - name: "Static checks: basic checks only"
- run: ./scripts/ci/static_checks/run_basic_static_checks.sh "${{ github.sha }}"
- env:
- VERBOSE: false
- docs:
- timeout-minutes: 45
- name: "Build docs"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- if: needs.build-info.outputs.docs-build == 'true'
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Fetch inventory versions"
- run: ./scripts/ci/docs/ci_docs_prepare.sh
- - uses: actions/cache@v2
- id: cache-doc-inventories
- with:
- path: ./docs/_inventory_cache/
- key: docs-inventory-v1-${{ hashFiles('constraints.txt') }}
- restore-keys: |
- docs-inventory-${{ hashFiles('constraints.txt') }}
- docs-inventory-
- - name: "Build docs"
- run: ./scripts/ci/docs/ci_docs.sh --package-filter apache-airflow --package-filter docker-stack
- - name: Configure AWS credentials
- uses: ./.github/actions/configure-aws-credentials
- if: >
- github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
- github.event_name == 'push'
- with:
- aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
- aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
- aws-region: eu-central-1
- - name: "Upload documentation to AWS S3"
- if: >
- github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
- github.event_name == 'push'
- run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs
-
- prepare-test-provider-packages-wheel:
- timeout-minutes: 40
- name: "Build and test provider packages wheel"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- AIRFLOW_EXTRAS: "all"
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- VERSION_SUFFIX_FOR_PYPI: ".dev0"
- NON_INTERACTIVE: "true"
- GENERATE_PROVIDERS_ISSUE: "true"
- if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- if: needs.build-info.outputs.default-branch == 'main'
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Prepare provider documentation"
- run: ./scripts/ci/provider_packages/ci_prepare_provider_documentation.sh
- - name: "Prepare provider packages: wheel"
- run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
- env:
- PACKAGE_FORMAT: "wheel"
- - name: "Prepare airflow package: wheel"
- run: ./scripts/ci/build_airflow/ci_build_airflow_packages.sh
- env:
- PACKAGE_FORMAT: "wheel"
- - name: "Install and test provider packages and airflow via wheel files"
- run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
- env:
- USE_AIRFLOW_VERSION: "wheel"
- PACKAGE_FORMAT: "wheel"
- - name: "Install and test provider packages and airflow on Airflow 2.1 files"
- run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
- env:
- USE_AIRFLOW_VERSION: "2.1.0"
- SKIP_TWINE_CHECK: "true"
- PACKAGE_FORMAT: "wheel"
-
- prepare-test-provider-packages-sdist:
- timeout-minutes: 40
- name: "Build and test provider packages sdist"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- AIRFLOW_EXTRAS: "all"
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- VERSION_SUFFIX_FOR_PYPI: ".dev0"
- NON_INTERACTIVE: "true"
- GENERATE_PROVIDERS_ISSUE: "true"
- if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- if: needs.build-info.outputs.default-branch == 'main'
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Prepare provider packages: sdist"
- run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
- env:
- PACKAGE_FORMAT: "sdist"
- - name: "Prepare airflow package: sdist"
- run: ./scripts/ci/build_airflow/ci_build_airflow_packages.sh
- env:
- PACKAGE_FORMAT: "sdist"
- - name: "Upload provider distribution artifacts"
- uses: actions/upload-artifact@v2
- with:
- name: airflow-provider-packages
- path: "./dist/apache-airflow-providers-*.tar.gz"
- retention-days: 1
- - name: "Install and test provider packages and airflow via sdist files"
- run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
- env:
- USE_AIRFLOW_VERSION: "sdist"
- PACKAGE_FORMAT: "sdist"
- - name: "Upload provider distribution artifacts"
- uses: actions/upload-artifact@v2
- with:
- name: airflow-provider-packages
- path: "./dist/apache-airflow-providers-*.tar.gz"
- retention-days: 1
-
- tests-helm:
- timeout-minutes: 80
- name: "Python unit tests for helm chart"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- MOUNT_SELECTED_LOCAL_SOURCES: "true"
- TEST_TYPES: "Helm"
- BACKEND: ""
- DB_RESET: "false"
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- if: >
- needs.build-info.outputs.needs-helm-tests == 'true' &&
- (github.repository == 'apache/airflow' || github.event_name != 'schedule') &&
- needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests: Helm"
- run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- - name: "Upload airflow logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: airflow-logs-helm
- path: "./files/airflow_logs*"
- retention-days: 7
- - name: "Upload container logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: container-logs-helm
- path: "./files/container_logs*"
- retention-days: 7
- - name: "Upload artifact for coverage"
- uses: actions/upload-artifact@v2
- if: needs.build-info.outputs.runCoverage == 'true'
- with:
- name: >
- coverage-helm
- path: "./files/coverage*.xml"
- retention-days: 7
-
- tests-postgres:
- timeout-minutes: 130
- name: >
- Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
- ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- strategy:
- matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }}
- fail-fast: false
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: postgres
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- POSTGRES_VERSION: ${{ matrix.postgres-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
- if: needs.build-info.outputs.run-tests == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
- run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- - name: "Upload airflow logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: airflow-logs-${{matrix.python-version}}-${{matrix.postgres-version}}
- path: "./files/airflow_logs*"
- retention-days: 7
- - name: "Upload container logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: container-logs-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
- path: "./files/container_logs*"
- retention-days: 7
- - name: "Upload artifact for coverage"
- uses: actions/upload-artifact@v2
- if: needs.build-info.outputs.runCoverage == 'true'
- with:
- name: >
- coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
- path: "./files/coverage*.xml"
- retention-days: 7
-
- tests-mysql:
- timeout-minutes: 130
- name: >
- MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- strategy:
- matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }}
- fail-fast: false
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: mysql
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- MYSQL_VERSION: ${{ matrix.mysql-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
- if: needs.build-info.outputs.run-tests == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
- run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- - name: "Upload airflow logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: airflow-logs-${{matrix.python-version}}-${{matrix.mysql-version}}
- path: "./files/airflow_logs*"
- retention-days: 7
- - name: "Upload container logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: container-logs-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
- path: "./files/container_logs*"
- retention-days: 7
- - name: "Upload artifact for coverage"
- uses: actions/upload-artifact@v2
- if: needs.build-info.outputs.runCoverage == 'true'
- with:
- name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
- path: "./files/coverage*.xml"
- retention-days: 7
-
- tests-mssql:
- timeout-minutes: 130
- name: >
- MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- strategy:
- matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- mssql-version: ${{ fromJson(needs.build-info.outputs.mssqlVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.mssqlExclude) }}
- fail-fast: false
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: mssql
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- MSSQL_VERSION: ${{ matrix.mssql-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
- if: needs.build-info.outputs.run-tests == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
- run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- - name: "Upload airflow logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: airflow-logs-${{matrix.python-version}}-${{matrix.mssql-version}}
- path: "./files/airflow_logs*"
- retention-days: 7
- - name: "Upload container logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: container-logs-mssql-${{matrix.python-version}}-${{matrix.mssql-version}}
- path: "./files/container_logs*"
- retention-days: 7
- - name: "Upload artifact for coverage"
- uses: actions/upload-artifact@v2
- if: needs.build-info.outputs.runCoverage == 'true'
- with:
- name: coverage-mssql-${{matrix.python-version}}-${{matrix.mssql-version}}
- path: "./files/coverage*.xml"
- retention-days: 7
-
- tests-sqlite:
- timeout-minutes: 130
- name: >
- Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images]
- strategy:
- matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.sqliteExclude) }}
- fail-fast: false
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: sqlite
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
- if: needs.build-info.outputs.run-tests == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
- run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- - name: "Upload airflow logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: airflow-logs-${{matrix.python-version}}
- path: './files/airflow_logs*'
- retention-days: 7
- - name: "Upload container logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: container-logs-sqlite-${{matrix.python-version}}
- path: "./files/container_logs*"
- retention-days: 7
- - name: "Upload artifact for coverage"
- uses: actions/upload-artifact@v2
- if: needs.build-info.outputs.runCoverage == 'true'
- with:
- name: coverage-sqlite-${{matrix.python-version}}
- path: ./files/coverage*.xml
- retention-days: 7
-
- tests-quarantined:
- timeout-minutes: 60
- name: "Quarantined tests"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- continue-on-error: true
- needs: [build-info, ci-images]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
- MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}}
- POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}}
- TEST_TYPES: "Quarantined"
- NUM_RUNS: 10
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: needs.build-info.outputs.run-tests == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Set issue id for main"
- if: github.ref == 'refs/heads/main'
- run: |
- echo "ISSUE_ID=10118" >> $GITHUB_ENV
- - name: "Set issue id for v1-10-stable"
- if: github.ref == 'refs/heads/v1-10-stable'
- run: |
- echo "ISSUE_ID=10127" >> $GITHUB_ENV
- - name: "Set issue id for v1-10-test"
- if: github.ref == 'refs/heads/v1-10-test'
- run: |
- echo "ISSUE_ID=10128" >> $GITHUB_ENV
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests: Quarantined"
- run: ./scripts/ci/testing/ci_run_quarantined_tests.sh
- env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- - name: "Upload Quarantine test results"
- uses: actions/upload-artifact@v2
- if: always()
- with:
- name: quarantined-tests
- path: "files/test_result-*.xml"
- retention-days: 7
- - name: "Upload airflow logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: airflow-logs-quarantined-${{ matrix.backend }}
- path: "./files/airflow_logs*"
- retention-days: 7
- - name: "Upload container logs"
- uses: actions/upload-artifact@v2
- if: failure()
- with:
- name: container-logs-quarantined-${{ matrix.backend }}
- path: "./files/container_logs*"
- retention-days: 7
- - name: "Upload artifact for coverage"
- uses: actions/upload-artifact@v2
- if: needs.build-info.outputs.runCoverage == 'true'
- with:
- name: coverage-quarantined-${{ matrix.backend }}
- path: "./files/coverage*.xml"
- retention-days: 7
-
- upload-coverage:
- timeout-minutes: 15
- name: "Upload coverage"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- continue-on-error: true
- needs:
- - build-info
- - tests-postgres
- - tests-sqlite
- - tests-mysql
- - tests-mssql
- - tests-quarantined
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- # Only upload coverage on merges to main
- if: needs.build-info.outputs.runCoverage == 'true'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Download all artifacts from the current build"
- uses: actions/download-artifact@v2
- with:
- path: ./coverage-files
- - name: "Removes unnecessary artifacts"
- run: ls ./coverage-files | grep -v coverage | xargs rm -rf
- - name: "Upload all coverage reports to codecov"
- uses: ./.github/actions/codecov-action
- with:
- directory: "./coverage-files"
-
- prod-images:
- timeout-minutes: 120
- name: "Wait for PROD images"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, ci-images, build-prod-images]
- if: needs.build-info.outputs.image-build == 'true'
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: sqlite
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- if: needs.build-info.outputs.waitForImage == 'true'
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- if: needs.build-info.outputs.waitForImage == 'true'
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- if: |
- needs.build-info.outputs.waitForImage == 'true'
- - name: "Cache virtualenv environmnent"
- uses: actions/cache@v2
- with:
- path: '.build/.docker_venv'
- key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }}
- - name: >
- Wait for PROD images
- ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
- # We wait for the images to be available either from the build-ci-image step or from
- # "build-images-workflow-run.yml' run as pull_request_target.
- # We are utilising single job to wait for all images because this job merely waits
- # For the images to be available. The test jobs wait for it to complete!
- #
- id: wait-for-images
- run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
- env:
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
- VERIFY_IMAGE: "true"
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
-
- tests-kubernetes:
- timeout-minutes: 50
- name: Helm Chart; ${{matrix.executor}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, prod-images]
- strategy:
- matrix:
- executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor]
- fail-fast: false
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: postgres
- RUN_TESTS: "true"
- RUNTIME: "kubernetes"
- KUBERNETES_MODE: "image"
- EXECUTOR: ${{matrix.executor}}
- KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
- HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
- CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.kubernetesVersionsListAsString}}
- if: >
- ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
- needs.build-info.outputs.needs-helm-tests == 'true' ) &&
- needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Get all PROD images"
- run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
- env:
- VERIFY_IMAGE: "false"
- - name: "Cache bin folder with tools for kubernetes testing"
- uses: actions/cache@v2
- with:
- path: ".build/kubernetes-bin"
- key: "kubernetes-binaries
--${{ needs.build-info.outputs.defaultKindVersion }}\
--${{ needs.build-info.outputs.defaultHelmVersion }}"
- restore-keys: "kubernetes-binaries"
- - name: "Kubernetes Tests"
- run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
- env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- - name: "Upload KinD logs"
- uses: actions/upload-artifact@v2
- if: failure() || cancelled()
- with:
- name: >
- kind-logs-${{matrix.executor}}
- path: /tmp/kind_logs_*
- retention-days: 7
-
- tests-helm-executor-upgrade:
- timeout-minutes: 80
- name: Helm Chart Executor Upgrade
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs: [build-info, prod-images]
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- BACKEND: postgres
- RUN_TESTS: "true"
- RUNTIME: "kubernetes"
- KUBERNETES_MODE: "image"
- EXECUTOR: "KubernetesExecutor"
- KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
- HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
- CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.kubernetesVersionsListAsString}}
- if: >
- needs.build-info.outputs.run-kubernetes-tests == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Get all PROD images"
- run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
- env:
- VERIFY_IMAGE: "false"
- - name: "Cache virtualenv for kubernetes testing"
- uses: actions/cache@v2
- with:
- path: ".build/.kubernetes_venv"
- key: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}\
- -${{needs.build-info.outputs.kubernetesVersionsListAsString}}
- -${{needs.build-info.outputs.pythonVersionsListAsString}}
- -${{ hashFiles('setup.py','setup.cfg') }}"
- restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-\
- -${{needs.build-info.outputs.kubernetesVersionsListAsString}}
- -${{needs.build-info.outputs.pythonVersionsListAsString}}"
- - name: "Cache bin folder with tools for kubernetes testing"
- uses: actions/cache@v2
- with:
- path: ".build/kubernetes-bin"
- key: "kubernetes-binaries
- -${{ needs.build-info.outputs.defaultKindVersion }}\
- -${{ needs.build-info.outputs.defaultHelmVersion }}"
- restore-keys: "kubernetes-binaries"
- - name: "Kubernetes Helm Chart Executor Upgrade Tests"
- run: ./scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh
- env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- - name: "Upload KinD logs"
- uses: actions/upload-artifact@v2
- if: failure() || cancelled()
- with:
- name: >
- kind-logs-KubernetesExecutor
- path: /tmp/kind_logs_*
- retention-days: 7
-
- constraints:
- permissions:
- contents: write
- timeout-minutes: 25
- name: "Constraints"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs:
- - build-info
- - ci-images
- - prod-images
- - static-checks
- - tests-sqlite
- - tests-mysql
- - tests-mssql
- - tests-postgres
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: ${{needs.build-info.outputs.pythonVersionsListAsString}}
- if: needs.build-info.outputs.upgradeToNewerDependencies != 'false'
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- submodules: recursive
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: >
- Wait for CI images
- ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
- run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh
- env:
- VERIFY_IMAGE: "false"
- - name: "Generate constraints with PyPI providers"
- run: ./scripts/ci/constraints/ci_generate_all_constraints.sh
- env:
- GENERATE_CONSTRAINTS_MODE: "pypi-providers"
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- - name: "Generate constraints with source providers"
- run: ./scripts/ci/constraints/ci_generate_all_constraints.sh
- env:
- GENERATE_CONSTRAINTS_MODE: "source-providers"
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- - name: "Generate constraints without providers"
- run: ./scripts/ci/constraints/ci_generate_all_constraints.sh
- env:
- GENERATE_CONSTRAINTS_MODE: "no-providers"
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
- - name: "Set constraints branch name"
- id: constraints-branch
- run: ./scripts/ci/constraints/ci_branch_constraints.sh
- # only actually checkout and push it when we are in apache/airflow repository
- - name: Checkout ${{ steps.constraints-branch.outputs.branch }}
- uses: actions/checkout@v2
- if: >
- github.repository == 'apache/airflow' &&
- (github.event_name == 'push' || github.event_name == 'schedule')
- with:
- path: "repo"
- ref: ${{ steps.constraints-branch.outputs.branch }}
- persist-credentials: false
- - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}"
- run: ./scripts/ci/constraints/ci_commit_constraints.sh
- if: >
- github.repository == 'apache/airflow' &&
- (github.event_name == 'push' || github.event_name == 'schedule')
- - name: "Push changes"
- uses: ./.github/actions/github-push-action
- if: >
- github.repository == 'apache/airflow' &&
- (github.event_name == 'push' || github.event_name == 'schedule')
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
- branch: ${{ steps.constraints-branch.outputs.branch }}
- directory: "repo"
-
- # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build
- # is executed as result of direct push to "main" or one of the "vX-Y-test" branches
- # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry
- # It will automatically check if a new python image was released and will pull the latest one if needed
- push-buildx-cache-to-github-registry:
- permissions:
- packages: write
- timeout-minutes: 120
- name: "Push images as cache to GitHub Registry"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- needs:
- - build-info
- - constraints
- - docs
- # Only run it for direct pushes and scheduled builds
- if: >
- (github.event_name == 'push' || github.event_name == 'schedule')
- && github.repository == 'apache/airflow'
- strategy:
- matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- # Rebuild images before push using the latest constraints (just pushed) without
- # eager upgrade. Do not wait for images, but rebuild them, but always check if
- # there is a new Python base image to pull and rebuild. This way, when latest python
- # is not pushed, we will re-use the last cache to start from and when there is
- # a new python image, we will rebuild it from scratch (same as during the "build-images.ci")
- GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
- GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
- GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
- UPGRADE_TO_NEWER_DEPENDENCIES: "false"
- PREPARE_BUILDX_CACHE: "true"
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v2
- with:
- persist-credentials: false
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- - name: "Free space"
- run: ./scripts/ci/tools/free_space.sh
- - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:latest"
- run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Prepare PROD image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:latest"
- run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- env:
- VERSION_SUFFIX_FOR_PYPI: ".dev0"
