| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| --- |
| name: Tests |
| on: # yamllint disable-line rule:truthy |
| schedule: |
| - cron: '28 0 * * *' |
| push: |
| branches: ['main', 'v[0-9]+-[0-9]+-test'] |
| pull_request: |
| branches: ['main', 'v[0-9]+-[0-9]+-test', 'v[0-9]+-[0-9]+-stable'] |
| permissions: |
| # All other permissions are set to none |
| contents: read |
| env: |
| MOUNT_SELECTED_LOCAL_SOURCES: "false" |
| FORCE_ANSWER_TO_QUESTIONS: "yes" |
| FORCE_PULL_IMAGES: "false" |
| CHECK_IMAGE_FOR_REBUILD: "true" |
| SKIP_CHECK_REMOTE_IMAGE: "true" |
| DB_RESET: "true" |
| VERBOSE: "true" |
| DOCKER_CACHE: "pulled" |
| GITHUB_REPOSITORY: ${{ github.repository }} |
| GITHUB_USERNAME: ${{ github.actor }} |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the |
| # Airflow one is going to be used |
| CONSTRAINTS_GITHUB_REPOSITORY: >- |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} |
| # In builds from forks, this token is read-only. For scheduler/direct push it is WRITE one |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| # In builds from forks, this token is empty, and this is good because such builds do not even try |
| # to push images to the registry. |
| GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" |
| GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" |
| INSTALL_PROVIDERS_FROM_SOURCES: "true" |
| AIRFLOW_LOGIN_TO_GITHUB_REGISTRY: "true" |
| ENABLE_TEST_COVERAGE: "${{ github.event_name == 'push' }}" |
| |
| # You can switch between building the image in "Build Images" workflow or building them in CI workflow |
| # Separately for each job. |
| # |
| # a) Using images build in the separate "Build Images" workflow: |
| # |
| # GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true" |
| # |
| # b) Building images in CI workflow - separately for each job: |
| # |
| # GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false" |
| # |
| # You can also switch back to building images locally and disabling the "Build Images" workflow |
| # by defining AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE secret with value set to "false" |
| GITHUB_REGISTRY_WAIT_FOR_IMAGE: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }} |
| |
| concurrency: |
| group: ci-${{ github.event.pull_request.number || github.ref }} |
| cancel-in-progress: true |
| |
| jobs: |
| |
| build-info: |
| name: "Build info" |
| # The runs-on cannot refer to env. or secrets. context, so we have no |
| # option but to specify a hard-coded list here. This is "safe", as the list |
| # is checked again by the runner using it's own list, so a PR author cannot |
| # change this and get access to our self-hosted runners |
| # |
| # When changing this list, ensure that it is kept in sync with the |
| # /runners/apache/airflow/configOverlay |
| # parameter in AWS SSM ParameterStore (which is what the runner uses) |
| # and restart the self-hosted runners. |
| # |
| # This list of committers can be generated with: |
| # https://github.com/apache/airflow-ci-infra/blob/main/scripts/list_committers |
| runs-on: >- |
| ${{ ( |
| ( |
| github.event_name == 'push' || |
| github.event_name == 'schedule' || |
| contains(fromJSON('[ |
| "BasPH", |
| "Fokko", |
| "KevinYang21", |
| "XD-DENG", |
| "aijamalnk", |
| "alexvanboxel", |
| "aoen", |
| "artwr", |
| "ashb", |
| "bbovenzi", |
| "bolkedebruin", |
| "criccomini", |
| "dimberman", |
| "dstandish", |
| "eladkal", |
| "ephraimbuddy", |
| "feluelle", |
| "feng-tao", |
| "houqp", |
| "jedcunningham", |
| "jgao54", |
| "jghoman", |
| "jhtimmins", |
| "jmcarp", |
| "kaxil", |
| "leahecole", |
| "mik-laj", |
| "milton0825", |
| "mistercrunch", |
| "msumit", |
| "potiuk", |
| "r39132", |
| "ryanahamilton", |
| "ryw", |
| "saguziel", |
| "sekikn", |
| "turbaszek", |
| "uranusjr", |
| "vikramkoka", |
| "xinbinhuang", |
| "yuqian90", |
| "zhongjiajie" |
| ]'), github.event.pull_request.user.login) |
| ) && github.repository == 'apache/airflow' |
| ) && 'self-hosted' || 'ubuntu-20.04' }} |
| env: |
| GITHUB_CONTEXT: ${{ toJson(github) }} |
| outputs: |
| waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }} |
| upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} |
| pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} |
| pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} |
| defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} |
| kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }} |
| kubernetesVersionsListAsString: ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} |
| defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }} |
| kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }} |
| defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }} |
| postgresVersions: ${{ steps.selective-checks.outputs.postgres-versions }} |
| defaultPostgresVersion: ${{ steps.selective-checks.outputs.default-postgres-version }} |
| mysqlVersions: ${{ steps.selective-checks.outputs.mysql-versions }} |
| mssqlVersions: ${{ steps.selective-checks.outputs.mssql-versions }} |
| defaultMySQLVersion: ${{ steps.selective-checks.outputs.default-mysql-version }} |
| helmVersions: ${{ steps.selective-checks.outputs.helm-versions }} |
| defaultHelmVersion: ${{ steps.selective-checks.outputs.default-helm-version }} |
| kindVersions: ${{ steps.selective-checks.outputs.kind-versions }} |
| defaultKindVersion: ${{ steps.selective-checks.outputs.default-kind-version }} |
| testTypes: ${{ steps.selective-checks.outputs.test-types }} |
| postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }} |
| mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }} |
| mssqlExclude: ${{ steps.selective-checks.outputs.mssql-exclude }} |
| sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} |
| run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} |
| run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} |
| basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} |
| image-build: ${{ steps.selective-checks.outputs.image-build }} |
| docs-build: ${{ steps.selective-checks.outputs.docs-build }} |
| needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} |
| needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} |
| needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} |
| default-branch: ${{ steps.selective-checks.outputs.default-branch }} |
| pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }} |
| pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }} |
| runsOn: ${{ steps.set-runs-on.outputs.runsOn }} |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Get information about the PR" |
| uses: ./.github/actions/get-workflow-origin |
| id: source-run-info |
| with: |
| token: ${{ secrets.GITHUB_TOKEN }} |
| - name: Set wait for image |
| id: wait-for-image |
| run: | |
| if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == 'true' ]]; then |
| echo "::set-output name=wait-for-image::true" |
| else |
| echo "::set-output name=wait-for-image::false" |
| fi |
| - name: Fetch incoming commit ${{ github.sha }} with its parent |
| uses: actions/checkout@v2 |
| with: |
| ref: ${{ github.sha }} |
| fetch-depth: 2 |
| persist-credentials: false |
| if: github.event_name == 'pull_request' |
| - name: Selective checks |
| id: selective-checks |
| env: |
| PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}" |
| run: | |
| if [[ ${GITHUB_EVENT_NAME} == "pull_request" ]]; then |
| # Run selective checks |
| ./scripts/ci/selective_ci_checks.sh "${GITHUB_SHA}" |
| else |
| # Run all checks |
| ./scripts/ci/selective_ci_checks.sh |
| fi |
| # Avoid having to specify the runs-on logic every time. We use the custom |
| # env var AIRFLOW_SELF_HOSTED_RUNNER set only on our runners, but never |
| # on the public runners |
| - name: Set runs-on |
| id: set-runs-on |
| env: |
| PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}" |
| run: | |
| if [[ ${PR_LABELS=} == *"use public runners"* ]]; then |
| echo "Forcing running on Public Runners via `use public runners` label" |
| echo "::set-output name=runsOn::\"ubuntu-20.04\"" |
| elif [[ ${AIRFLOW_SELF_HOSTED_RUNNER} == "" ]]; then |
| echo "Regular PR running with Public Runner" |
| echo "::set-output name=runsOn::\"ubuntu-20.04\"" |
| else |
| echo "Maintainer or main run running with self-hosted runner" |
| echo "::set-output name=runsOn::\"self-hosted\"" |
| fi |
| |
| run-new-breeze-tests: |
| timeout-minutes: 10 |
| name: Breeze2 tests |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info] |
| defaults: |
| run: |
| shell: bash |
| working-directory: ./dev/breeze |
| steps: |
| - uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - uses: actions/setup-python@v2 |
| with: |
| python-version: '3.7' |
| cache: 'pip' |
| - run: pip install . |
| - run: python3 -m pytest -n auto --color=yes |
| |
| tests-ui: |
| timeout-minutes: 10 |
| name: React UI tests |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info] |
| if: needs.build-info.outputs.run-ui-tests == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup node" |
| uses: actions/setup-node@v2 |
| with: |
| node-version: 14 |
| - name: "Cache eslint" |
| uses: actions/cache@v2 |
| with: |
| path: 'airflow/ui/node_modules' |
| key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }} |
| - run: yarn --cwd airflow/ui/ install --frozen-lockfile --non-interactive |
| - run: yarn --cwd airflow/ui/ run test |
| |
| tests-www: |
| timeout-minutes: 10 |
| name: React WWW tests |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info] |
| if: needs.build-info.outputs.run-www-tests == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup node" |
| uses: actions/setup-node@v2 |
| with: |
| node-version: 14 |
| - name: "Cache eslint" |
| uses: actions/cache@v2 |
| with: |
| path: 'airflow/www/node_modules' |
| key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }} |
| - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive |
| - run: yarn --cwd airflow/www/ run test |
| |
| |
| test-openapi-client-generation: |
| timeout-minutes: 10 |
| name: "Test OpenAPI client generation" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info] |
| if: needs.build-info.outputs.needs-api-codegen == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Generate client codegen diff" |
| run: ./scripts/ci/openapi/client_codegen_diff.sh |
| |
| test-examples-of-prod-image-building: |
| timeout-minutes: 60 |
| name: "Test examples of production image building" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info] |
| if: needs.build-info.outputs.image-build == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| if: | |
| needs.build-info.outputs.waitForImage == 'true' |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{needs.build-info.outputs.defaultPythonVersion}} |
| - name: "Cache virtualenv environmnent" |
| uses: actions/cache@v2 |
| with: |
| path: '.build/.docker_venv' |
| key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} |
| - name: "Test examples of PROD image building" |
| run: ./scripts/ci/images/ci_test_examples_of_prod_image_building.sh |
| |
| test-docker-compose-quick-start: |
| timeout-minutes: 60 |
| name: "Test docker-compose quick start" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, prod-images] |
| if: needs.build-info.outputs.image-build == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| if: | |
| needs.build-info.outputs.waitForImage == 'true' |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{needs.build-info.outputs.defaultPythonVersion}} |
| - name: "Cache virtualenv environmnent" |
| uses: actions/cache@v2 |
| with: |
| path: '.build/.docker_venv' |
| key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} |
| - name: "Test docker-compose quick start" |
| run: ./scripts/ci/images/ci_run_docker_compose_quick_start_test.sh |
| |
| ci-images: |
| timeout-minutes: 120 |
| name: "Wait for CI images" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info] |
| if: needs.build-info.outputs.image-build == 'true' |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| BACKEND: sqlite |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} |
| WAIT_FOR_IMAGE: ${{ needs.build-info.outputs.waitForImage }} |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| if: needs.build-info.outputs.waitForImage == 'true' |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{needs.build-info.outputs.defaultPythonVersion}} |
| if: needs.build-info.outputs.waitForImage == 'true' |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| if: | |
| needs.build-info.outputs.waitForImage == 'true' |
| - name: "Cache virtualenv environmnent" |
| uses: actions/cache@v2 |
| with: |
| path: '.build/.docker_venv' |
| key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} |
| - name: > |
| Wait for CI images |
| ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }} |
| id: wait-for-images |
| # We wait for the images to be available either from the build-ci-image step or from |
| # "build-images-workflow-run.yml' run as pull_request_target. |
| # We are utilising single job to wait for all images because this job merely waits |
| # for the images to be available. |
| # The test jobs wait for it to complete if WAIT_FOR_IMAGE is 'true'! |
| run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh |
| env: |
| CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > |
| ${{needs.build-info.outputs.pythonVersionsListAsString}} |
| VERIFY_IMAGE: "true" |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| |
| static-checks: |
| timeout-minutes: 30 |
| name: "Static checks" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, ci-images] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| SKIP: "identity" |
| MOUNT_SELECTED_LOCAL_SOURCES: "true" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} |
| if: needs.build-info.outputs.basic-checks-only == 'false' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{needs.build-info.outputs.defaultPythonVersion}} |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Get Python version" |
| run: "echo \"::set-output name=host-python-version::$(python -c |
| 'import platform; print(platform.python_version())')\"" |
| id: host-python-version |
| - name: "Cache pre-commit local-installation" |
| uses: actions/cache@v2 |
| with: |
| path: ~/.local |
| key: "pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-\ |
| ${{ hashFiles('setup.py', 'setup.cfg') }}" |
| restore-keys: "\ |
| pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-" |
| - name: "Cache pre-commit envs" |
| uses: actions/cache@v2 |
| with: |
| path: ~/.cache/pre-commit |
| key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ |
| ${{ hashFiles('.pre-commit-config.yaml') }}" |
| restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} |
| |
| - name: "Cache eslint" |
| uses: actions/cache@v2 |
| with: |
| path: 'airflow/ui/node_modules' |
| key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }} |
| - name: "Static checks" |
| run: ./scripts/ci/static_checks/run_static_checks.sh |
| env: |
| VERBOSE: false |
| |
| # Those checks are run if no image needs to be built for checks. This is for simple changes that |
| # Do not touch any of the python code or any of the important files that might require building |
| # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host |
| static-checks-basic-checks-only: |
| timeout-minutes: 30 |
| name: "Static checks: basic checks only" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| SKIP: "build,mypy,flake8,bats-in-container-tests,identity" |
| MOUNT_SELECTED_LOCAL_SOURCES: "true" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} |
| if: needs.build-info.outputs.basic-checks-only == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{needs.build-info.outputs.defaultPythonVersion}} |
| - name: > |
| Fetch incoming commit ${{ github.sha }} with its parent |
| uses: actions/checkout@v2 |
| with: |
| ref: ${{ github.sha }} |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Get Python version" |
| run: "echo \"::set-output name=host-python-version::$(python -c |
| 'import platform; print(platform.python_version())')\"" |
| id: host-python-version |
| - name: "Cache pre-commit local-installation" |
| uses: actions/cache@v2 |
| with: |
| path: ~/.local |
| key: "pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-\ |
| ${{ hashFiles('setup.py', 'setup.cfg') }}" |
| restore-keys: "\ |
| pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-" |
| - name: "Cache pre-commit envs" |
| uses: actions/cache@v2 |
| with: |
| path: ~/.cache/pre-commit |
| key: "pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}}-\ |
| ${{ hashFiles('.pre-commit-config.yaml') }}" |
| restore-keys: pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}} |
| - name: "Static checks: basic checks only" |
| run: ./scripts/ci/static_checks/run_basic_static_checks.sh "${{ github.sha }}" |
| env: |
| VERBOSE: false |
| docs: |
| timeout-minutes: 45 |
| name: "Build docs" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, ci-images] |
| if: needs.build-info.outputs.docs-build == 'true' |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Fetch inventory versions" |
| run: ./scripts/ci/docs/ci_docs_prepare.sh |
| - uses: actions/cache@v2 |
| id: cache-doc-inventories |
| with: |
| path: ./docs/_inventory_cache/ |
| key: docs-inventory-v1-${{ hashFiles('constraints.txt') }} |
| restore-keys: | |
| docs-inventory-${{ hashFiles('constraints.txt') }} |
| docs-inventory- |
| - name: "Build docs" |
| run: ./scripts/ci/docs/ci_docs.sh |
| - name: Configure AWS credentials |
| uses: ./.github/actions/configure-aws-credentials |
| if: > |
| github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && |
| github.event_name == 'push' |
| with: |
| aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} |
| aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} |
| aws-region: eu-central-1 |
| - name: "Upload documentation to AWS S3" |
| if: > |
| github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && |
| github.event_name == 'push' |
| run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs |
| |
| prepare-test-provider-packages-wheel: |
| timeout-minutes: 40 |
| name: "Build and test provider packages wheel" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, ci-images] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| AIRFLOW_EXTRAS: "all" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} |
| VERSION_SUFFIX_FOR_PYPI: ".dev0" |
| NON_INTERACTIVE: "true" |
| GENERATE_PROVIDERS_ISSUE: "true" |
| if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| if: needs.build-info.outputs.default-branch == 'main' |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Prepare provider documentation" |
| run: ./scripts/ci/provider_packages/ci_prepare_provider_documentation.sh |
| - name: "Prepare provider packages: wheel" |
| run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh |
| env: |
| PACKAGE_FORMAT: "wheel" |
| - name: "Prepare airflow package: wheel" |
| run: ./scripts/ci/build_airflow/ci_build_airflow_packages.sh |
| env: |
| PACKAGE_FORMAT: "wheel" |
| - name: "Install and test provider packages and airflow via wheel files" |
| run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh |
| env: |
| USE_AIRFLOW_VERSION: "wheel" |
| PACKAGE_FORMAT: "wheel" |
| - name: "Install and test provider packages and airflow on Airflow 2.1 files" |
| run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh |
| env: |
| USE_AIRFLOW_VERSION: "2.1.0" |
| PACKAGE_FORMAT: "wheel" |
| |
| prepare-test-provider-packages-sdist: |
| timeout-minutes: 40 |
| name: "Build and test provider packages sdist" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, ci-images] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| AIRFLOW_EXTRAS: "all" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} |
| VERSION_SUFFIX_FOR_PYPI: ".dev0" |
| NON_INTERACTIVE: "true" |
| GENERATE_PROVIDERS_ISSUE: "true" |
| if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| if: needs.build-info.outputs.default-branch == 'main' |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Prepare provider packages: sdist" |
| run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh |
| env: |
| PACKAGE_FORMAT: "sdist" |
| - name: "Prepare airflow package: sdist" |
| run: ./scripts/ci/build_airflow/ci_build_airflow_packages.sh |
| env: |
| PACKAGE_FORMAT: "sdist" |
| - name: "Install and test provider packages and airflow via sdist files" |
| run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh |
| env: |
| USE_AIRFLOW_VERSION: "sdist" |
| PACKAGE_FORMAT: "sdist" |
| - name: "Upload provider distribution artifacts" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: airflow-provider-packages |
| path: "./dist/apache-airflow-providers-*.tar.gz" |
| retention-days: 1 |
| |
| tests-helm: |
| timeout-minutes: 40 |
| name: "Python unit tests for helm chart" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, ci-images] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| MOUNT_SELECTED_LOCAL_SOURCES: "true" |
| TEST_TYPES: "Helm" |
| BACKEND: "" |
| DB_RESET: "false" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} |
| if: > |
| needs.build-info.outputs.needs-helm-tests == 'true' && |
| (github.repository == 'apache/airflow' || github.event_name != 'schedule') && |
| needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Tests: Helm" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: airflow-logs-helm |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: container-logs-helm |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: > |
| coverage-helm |
| path: "./files/coverage*.xml" |
| retention-days: 7 |
| |
| tests-postgres: |
| timeout-minutes: 130 |
| name: > |
| Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}: |
| ${{needs.build-info.outputs.testTypes}} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }} |
| fail-fast: false |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| BACKEND: postgres |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| POSTGRES_VERSION: ${{ matrix.postgres-version }} |
| TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Tests: ${{needs.build-info.outputs.testTypes}}" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: airflow-logs-${{matrix.python-version}}-${{matrix.postgres-version}} |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: container-logs-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: > |
| coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} |
| path: "./files/coverage*.xml" |
| retention-days: 7 |
| |
| tests-mysql: |
| timeout-minutes: 130 |
| name: > |
| MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }} |
| fail-fast: false |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| BACKEND: mysql |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| MYSQL_VERSION: ${{ matrix.mysql-version }} |
| TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Tests: ${{needs.build-info.outputs.testTypes}}" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: airflow-logs-${{matrix.python-version}}-${{matrix.mysql-version}} |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: container-logs-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} |
| path: "./files/coverage*.xml" |
| retention-days: 7 |
| |
| tests-mssql: |
| timeout-minutes: 130 |
| name: > |
| MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| mssql-version: ${{ fromJson(needs.build-info.outputs.mssqlVersions) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.mssqlExclude) }} |
| fail-fast: false |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| BACKEND: mssql |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| MSSQL_VERSION: ${{ matrix.mssql-version }} |
| TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Tests: ${{needs.build-info.outputs.testTypes}}" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: airflow-logs-${{matrix.python-version}}-${{matrix.mssql-version}} |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: container-logs-mssql-${{matrix.python-version}}-${{matrix.mssql-version}} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: coverage-mssql-${{matrix.python-version}}-${{matrix.mssql-version}} |
| path: "./files/coverage*.xml" |
| retention-days: 7 |
| |
| tests-sqlite: |
| timeout-minutes: 130 |
| name: > |
| Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.sqliteExclude) }} |
| fail-fast: false |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| BACKEND: sqlite |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Tests: ${{needs.build-info.outputs.testTypes}}" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: airflow-logs-${{matrix.python-version}} |
| path: './files/airflow_logs*' |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: container-logs-sqlite-${{matrix.python-version}} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: coverage-sqlite-${{matrix.python-version}} |
| path: ./files/coverage*.xml |
| retention-days: 7 |
| |
| tests-quarantined: |
| timeout-minutes: 60 |
| name: "Quarantined tests" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| continue-on-error: true |
| needs: [build-info, ci-images] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} |
| MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}} |
| POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}} |
| TEST_TYPES: "Quarantined" |
| NUM_RUNS: 10 |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Set issue id for main" |
| if: github.ref == 'refs/heads/main' |
| run: | |
| echo "ISSUE_ID=10118" >> $GITHUB_ENV |
| - name: "Set issue id for v1-10-stable" |
| if: github.ref == 'refs/heads/v1-10-stable' |
| run: | |
| echo "ISSUE_ID=10127" >> $GITHUB_ENV |
| - name: "Set issue id for v1-10-test" |
| if: github.ref == 'refs/heads/v1-10-test' |
| run: | |
| echo "ISSUE_ID=10128" >> $GITHUB_ENV |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Tests: Quarantined" |
| run: ./scripts/ci/testing/ci_run_quarantined_tests.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| - name: "Upload Quarantine test results" |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: quarantined-tests |
| path: "files/test_result-*.xml" |
| retention-days: 7 |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: airflow-logs-quarantined-${{ matrix.backend }} |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: container-logs-quarantined-${{ matrix.backend }} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: coverage-quarantined-${{ matrix.backend }} |
| path: "./files/coverage*.xml" |
| retention-days: 7 |
| |
| upload-coverage: |
| timeout-minutes: 15 |
| name: "Upload coverage" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| continue-on-error: true |
| needs: |
| - build-info |
| - tests-postgres |
| - tests-sqlite |
| - tests-mysql |
| - tests-mssql |
| - tests-quarantined |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| # Only upload coverage on merges to main |
| if: > |
| github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && |
| github.event_name == 'push' && |
| needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Download all artifacts from the current build" |
| uses: actions/download-artifact@v2 |
| with: |
| path: ./coverage-files |
| - name: "Removes unnecessary artifacts" |
| run: ls ./coverage-files | grep -v coverage | xargs rm -rf |
| - name: "Upload all coverage reports to codecov" |
| uses: ./.github/actions/codecov-action |
| with: |
| directory: "./coverage-files" |
| |
| prod-images: |
| timeout-minutes: 120 |
| name: "Wait for PROD images" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, ci-images] |
| if: needs.build-info.outputs.image-build == 'true' |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| BACKEND: sqlite |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| if: needs.build-info.outputs.waitForImage == 'true' |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| if: needs.build-info.outputs.waitForImage == 'true' |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| if: | |
| needs.build-info.outputs.waitForImage == 'true' |
| - name: "Cache virtualenv environmnent" |
| uses: actions/cache@v2 |
| with: |
| path: '.build/.docker_venv' |
| key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} |
| - name: > |
| Wait for PROD images |
| ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }} |
| # We wait for the images to be available either from the build-ci-image step or from |
| # "build-images-workflow-run.yml' run as pull_request_target. |
| # We are utilising single job to wait for all images because this job merely waits |
| # For the images to be available. The test jobs wait for it to complete! |
| # |
| id: wait-for-images |
| run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh |
| env: |
| CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > |
| ${{needs.build-info.outputs.pythonVersionsListAsString}} |
| VERIFY_IMAGE: "true" |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| |
| tests-kubernetes: |
| timeout-minutes: 50 |
| name: Helm Chart; ${{matrix.executor}} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, prod-images] |
| strategy: |
| matrix: |
| executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] |
| fail-fast: false |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| BACKEND: postgres |
| RUN_TESTS: "true" |
| RUNTIME: "kubernetes" |
| KUBERNETES_MODE: "image" |
| EXECUTOR: ${{matrix.executor}} |
| KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" |
| HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" |
| CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > |
| ${{needs.build-info.outputs.pythonVersionsListAsString}} |
| CURRENT_KUBERNETES_VERSIONS_AS_STRING: > |
| ${{needs.build-info.outputs.kubernetesVersionsListAsString}} |
| if: > |
| ( needs.build-info.outputs.run-kubernetes-tests == 'true' || |
| needs.build-info.outputs.needs-helm-tests == 'true' ) && |
| needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Get all PROD images" |
| run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh |
| env: |
| VERIFY_IMAGE: "false" |
| - name: "Cache bin folder with tools for kubernetes testing" |
| uses: actions/cache@v2 |
| with: |
| path: ".build/kubernetes-bin" |
| key: "kubernetes-binaries |
| -${{ needs.build-info.outputs.defaultKindVersion }}\ |
| -${{ needs.build-info.outputs.defaultHelmVersion }}" |
| restore-keys: "kubernetes-binaries" |
| - name: "Kubernetes Tests" |
| run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| - name: "Upload KinD logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() || cancelled() |
| with: |
| name: > |
| kind-logs-${{matrix.executor}} |
| path: /tmp/kind_logs_* |
| retention-days: 7 |
| |
| tests-helm-executor-upgrade: |
| timeout-minutes: 50 |
| name: Helm Chart Executor Upgrade |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: [build-info, prod-images] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| BACKEND: postgres |
| RUN_TESTS: "true" |
| RUNTIME: "kubernetes" |
| KUBERNETES_MODE: "image" |
| EXECUTOR: "KubernetesExecutor" |
| KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" |
| HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" |
| CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > |
| ${{needs.build-info.outputs.pythonVersionsListAsString}} |
| CURRENT_KUBERNETES_VERSIONS_AS_STRING: > |
| ${{needs.build-info.outputs.kubernetesVersionsListAsString}} |
| if: > |
| needs.build-info.outputs.run-kubernetes-tests == 'true' && |
| needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Get all PROD images" |
| run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh |
| env: |
| VERIFY_IMAGE: "false" |
| - name: "Cache virtualenv for kubernetes testing" |
| uses: actions/cache@v2 |
| with: |
| path: ".build/.kubernetes_venv" |
| key: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}\ |
| -${{needs.build-info.outputs.kubernetesVersionsListAsString}} |
| -${{needs.build-info.outputs.pythonVersionsListAsString}} |
| -${{ hashFiles('setup.py','setup.cfg') }}" |
| restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-\ |
| -${{needs.build-info.outputs.kubernetesVersionsListAsString}} |
| -${{needs.build-info.outputs.pythonVersionsListAsString}}" |
| - name: "Cache bin folder with tools for kubernetes testing" |
| uses: actions/cache@v2 |
| with: |
| path: ".build/kubernetes-bin" |
| key: "kubernetes-binaries |
| -${{ needs.build-info.outputs.defaultKindVersion }}\ |
| -${{ needs.build-info.outputs.defaultHelmVersion }}" |
| restore-keys: "kubernetes-binaries" |
| - name: "Kubernetes Helm Chart Executor Upgrade Tests" |
| run: ./scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| - name: "Upload KinD logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() || cancelled() |
| with: |
| name: > |
| kind-logs-KubernetesExecutor |
| path: /tmp/kind_logs_* |
| retention-days: 7 |
| |
| constraints: |
| permissions: |
| contents: write |
| timeout-minutes: 10 |
| name: "Constraints" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: |
| - build-info |
| - ci-images |
| - prod-images |
| - static-checks |
| - tests-sqlite |
| - tests-mysql |
| - tests-mssql |
| - tests-postgres |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: ${{needs.build-info.outputs.pythonVersionsListAsString}} |
| # Only run it for direct pushes and scheduled builds |
| if: github.event_name == 'push' || github.event_name == 'schedule' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: > |
| Wait for CI images |
| ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }} |
| run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh |
| env: |
| VERIFY_IMAGE: "false" |
| - name: "Generate constraints with PyPI providers" |
| run: ./scripts/ci/constraints/ci_generate_all_constraints.sh |
| env: |
| GENERATE_CONSTRAINTS_MODE: "pypi-providers" |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| - name: "Generate constraints with source providers" |
| run: ./scripts/ci/constraints/ci_generate_all_constraints.sh |
| env: |
| GENERATE_CONSTRAINTS_MODE: "source-providers" |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| - name: "Generate constraints without providers" |
| run: ./scripts/ci/constraints/ci_generate_all_constraints.sh |
| env: |
| GENERATE_CONSTRAINTS_MODE: "no-providers" |
| PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" |
| - name: "Set constraints branch name" |
| id: constraints-branch |
| run: ./scripts/ci/constraints/ci_branch_constraints.sh |
| # only actually push it when we are in apache/airflow repository |
| - name: Checkout ${{ steps.constraints-branch.outputs.branch }} |
| uses: actions/checkout@v2 |
| if: github.repository == 'apache/airflow' |
| with: |
| path: "repo" |
| ref: ${{ steps.constraints-branch.outputs.branch }} |
| persist-credentials: false |
| - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}" |
| run: ./scripts/ci/constraints/ci_commit_constraints.sh |
| if: github.repository == 'apache/airflow' |
| - name: "Push changes" |
| uses: ./.github/actions/github-push-action |
| if: github.repository == 'apache/airflow' |
| with: |
| github_token: ${{ secrets.GITHUB_TOKEN }} |
| branch: ${{ steps.constraints-branch.outputs.branch }} |
| directory: "repo" |
| |
| # Push images to GitHub Registry in Apache repository, if all tests are successful and build |
| # is executed as result of direct push to "main" or one of the "test" branches |
| # It actually rebuilds all images using just-pushed constraints if they changed |
| # It will also check if a new python image was released and will pull the latest one if needed |
| # Same as build-images.yaml |
| push-images-to-github-registry: |
| permissions: |
| packages: write |
| timeout-minutes: 40 |
| name: "Push images as cache to GitHub Registry" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| needs: |
| - build-info |
| - constraints |
| - docs |
| # Only run it for direct pushes and scheduled builds |
| if: github.event_name == 'push' || github.event_name == 'schedule' |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| # Rebuild images before push using the latest constraints (just pushed) without |
| # eager upgrade. Do not wait for images, but rebuild them, but always check if |
| # there is a new Python base image to pull and rebuild. This way, when latest python |
| # is not pushed, we will re-use the last cache to start from and when there is |
| # a new python image, we will rebuild it from scratch (same as during the "build-images.ci") |
| GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" |
| GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" |
| PUSH_PYTHON_BASE_IMAGE: "true" |
| FORCE_PULL_IMAGES: "true" |
| CHECK_IF_BASE_PYTHON_IMAGE_UPDATED: "true" |
| GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false" |
| UPGRADE_TO_NEWER_DEPENDENCIES: "false" |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/free_space.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:latest" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Prepare PROD image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:latest" |
| run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh |
| env: |
| VERSION_SUFFIX_FOR_PYPI: ".dev0" |
| - name: "Push CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:latest" |
| run: ./scripts/ci/images/ci_push_ci_images.sh |
| - name: "Push PROD images ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:latest" |
| run: ./scripts/ci/images/ci_push_production_images.sh |
