| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| --- |
| name: CI Build |
| on: # yamllint disable-line rule:truthy |
| schedule: |
| - cron: '28 0 * * *' |
| push: |
| branches: ['master', 'v1-10-test', 'v1-10-stable', 'v2-0-test'] |
| pull_request: |
| branches: ['master', 'v1-10-test', 'v1-10-stable', 'v2-0-test'] |
| |
| env: |
| |
| MOUNT_LOCAL_SOURCES: "false" |
| FORCE_ANSWER_TO_QUESTIONS: "yes" |
| FORCE_PULL_IMAGES: "true" |
| CHECK_IMAGE_FOR_REBUILD: "true" |
| SKIP_CHECK_REMOTE_IMAGE: "true" |
| DB_RESET: "true" |
| VERBOSE: "true" |
| DOCKER_CACHE: "pulled" |
| USE_GITHUB_REGISTRY: "true" |
| GITHUB_REPOSITORY: ${{ github.repository }} |
| GITHUB_USERNAME: ${{ github.actor }} |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.run_id }}" |
| GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" |
| INSTALL_PROVIDERS_FROM_SOURCES: "true" |
| |
| # You can switch between building the image in "Build Images" workflow or building them in CI workflow |
| # Separately for each job. |
| # |
| # a) Using images build in the separate "Build Image" workflow: |
| # |
| # GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true" |
| # |
| # b) Building images in CI workflow - separately for each job: |
| # |
| # GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false" |
| # |
| # You can also switch back to building images locally and disabling the "Build Images" workflow |
| # by defining AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE secret with value set to "false" |
| |
| GITHUB_REGISTRY_WAIT_FOR_IMAGE: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }} |
| |
| jobs: |
| |
| build-info: |
| name: "Build info" |
| runs-on: ubuntu-20.04 |
| env: |
| GITHUB_CONTEXT: ${{ toJson(github) }} |
| outputs: |
| waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }} |
| upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} |
| pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} |
| pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} |
| defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} |
| kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }} |
| defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }} |
| kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }} |
| defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }} |
| postgresVersions: ${{ steps.selective-checks.outputs.postgres-versions }} |
| defaultPostgresVersion: ${{ steps.selective-checks.outputs.default-postgres-version }} |
| mysqlVersions: ${{ steps.selective-checks.outputs.mysql-versions }} |
| defaultMySQLVersion: ${{ steps.selective-checks.outputs.default-mysql-version }} |
| helmVersions: ${{ steps.selective-checks.outputs.helm-versions }} |
| defaultHelmVersion: ${{ steps.selective-checks.outputs.default-helm-version }} |
| kindVersions: ${{ steps.selective-checks.outputs.kind-versions }} |
| defaultKindVersion: ${{ steps.selective-checks.outputs.default-kind-version }} |
| testTypes: ${{ steps.selective-checks.outputs.test-types }} |
| postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }} |
| mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }} |
| sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} |
| kubernetesExclude: ${{ steps.selective-checks.outputs.kubernetes-exclude }} |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} |
| basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} |
| image-build: ${{ steps.selective-checks.outputs.image-build }} |
| docs-build: ${{ steps.selective-checks.outputs.docs-build }} |
| needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} |
| needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} |
| needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} |
| pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }} |
| pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }} |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Get information about the PR" |
| uses: ./.github/actions/get-workflow-origin |
| id: source-run-info |
| with: |
| token: ${{ secrets.GITHUB_TOKEN }} |
| - name: > |
| Event: ${{ github.event_name }} |
| Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} |
| Branch: ${{ github.head_ref }} |
| Run id: ${{ github.run_id }} |
| Sha: ${{ github.sha }} |
| Ref: ${{ github.ref }} |
| run: printenv |
| - name: Set wait for image |
| id: wait-for-image |
| run: | |
| if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == 'true' ]]; then |
| echo "::set-output name=wait-for-image::true" |
| else |
| echo "::set-output name=wait-for-image::false" |
| fi |
| - name: Fetch incoming commit ${{ github.sha }} with its parent |
| uses: actions/checkout@v2 |
| with: |
| ref: ${{ github.sha }} |
| fetch-depth: 2 |
| persist-credentials: false |
| if: github.event_name == 'pull_request' |
| - name: Selective checks |
| id: selective-checks |
| env: |
| EVENT_NAME: ${{ github.event_name }} |
| TARGET_COMMIT_SHA: ${{ github.sha }} |
| PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}" |
| run: | |
| if [[ ${EVENT_NAME} == "pull_request" ]]; then |
| # Run selective checks |
| ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}" |
| else |
| # Run all checks |
| ./scripts/ci/selective_ci_checks.sh |
| fi |
| |
| test-openapi-client-generation: |
| name: "Test OpenAPI client generation" |
| runs-on: ubuntu-20.04 |
| needs: [build-info] |
| if: needs.build-info.outputs.needs-api-codegen == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Generate client codegen diff" |
| run: ./scripts/ci/openapi/client_codegen_diff.sh |
| |
| ci-images: |
| timeout-minutes: 120 |
| name: "Wait for CI images" |
| runs-on: ubuntu-20.04 |
| needs: [build-info] |
| if: needs.build-info.outputs.image-build == 'true' |
| env: |
| BACKEND: sqlite |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| if: needs.build-info.outputs.waitForImage == 'true' |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{needs.build-info.outputs.defaultPythonVersion}} |
| if: needs.build-info.outputs.waitForImage == 'true' |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| if: needs.build-info.outputs.waitForImage == 'true' |
| - name: > |
| Wait for CI images |
| ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }} |
| env: |
| CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > |
| ${{needs.build-info.outputs.pythonVersionsListAsString}} |
| # We wait for the images to be available either from the build-ci-image step or from |
| # "build-images-workflow-run.yml' run as pull_request_target (it has the write |
| # permissions in case pull_request from fork is run. |
| # We are utilising single job to wait for all images because this job merely waits |
| # For the images to be available. The test jobs wait for it to complete! |
| run: ./scripts/ci/images/ci_wait_for_all_ci_images.sh |
| if: needs.build-info.outputs.waitForImage == 'true' |
| |
| verify-ci-images: |
| timeout-minutes: 20 |
| name: "Verify CI Image Py${{matrix.python-version}}" |
| runs-on: ubuntu-20.04 |
| needs: [build-info, ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| env: |
| BACKEND: sqlite |
| if: needs.build-info.outputs.image-build == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| if: needs.build-info.outputs.waitForImage == 'true' |
| with: |
| persist-credentials: false |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| if: needs.build-info.outputs.waitForImage == 'true' |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Verify CI image Py${{matrix.python-version}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_verify_ci_image.sh |
| if: needs.build-info.outputs.waitForImage == 'true' |
| |
| static-checks: |
| timeout-minutes: 30 |
| name: "Static checks" |
| runs-on: ubuntu-20.04 |
| needs: [build-info, ci-images] |
| env: |
| SKIP: "pylint,identity" |
| MOUNT_LOCAL_SOURCES: "true" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} |
| if: needs.build-info.outputs.basic-checks-only == 'false' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{needs.build-info.outputs.defaultPythonVersion}} |
| - name: Cache pre-commit env |
| uses: actions/cache@v2 |
| with: |
| path: ~/.cache/pre-commit |
| key: pre-commit-no-pylint-${{ hashFiles('.pre-commit-config.yaml') }} |
| restore-keys: pre-commit-no-pylint- |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Static checks: except pylint" |
| run: ./scripts/ci/static_checks/run_static_checks.sh |
| env: |
| VERBOSE: false |
| |
| # Those checks are run if no image needs to be built for checks. This is for simple changes that |
| # Do not touch any of the python code or any of the important files that might require building |
| # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host |
| static-checks-basic-checks-only: |
| timeout-minutes: 30 |
| name: "Static checks: basic checks only" |
| runs-on: ubuntu-20.04 |
| needs: [build-info] |
| env: |
| SKIP: "build,mypy,flake8,pylint,bats-in-container-tests,identity" |
| MOUNT_LOCAL_SOURCES: "true" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} |
| if: needs.build-info.outputs.basic-checks-only == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{needs.build-info.outputs.defaultPythonVersion}} |
| - name: Cache pre-commit env |
| uses: actions/cache@v2 |
| with: |
| path: ~/.cache/pre-commit |
| key: pre-commit-basic-${{ hashFiles('.pre-commit-config.yaml') }} |
| restore-keys: pre-commit-no-basic- |
| - name: > |
| Fetch incoming commit ${{ github.sha }} with its parent |
| uses: actions/checkout@v2 |
| with: |
| ref: ${{ github.sha }} |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Static checks: basic checks only" |
| run: | |
| ./scripts/ci/static_checks/run_basic_static_checks.sh "${{ github.sha }}" |
| env: |
| VERBOSE: false |
| |
| |
| static-checks-pylint: |
| timeout-minutes: 30 |
| name: "Pylint" |
| runs-on: ubuntu-20.04 |
| needs: [build-info, ci-images] |
| if: needs.build-info.outputs.basic-checks-only == 'false' |
| env: |
| # We want to make sure we have latest sources as only in_container scripts are added |
| # to the image but we want to static-check all of them |
| MOUNT_LOCAL_SOURCES: "true" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{needs.build-info.outputs.defaultPythonVersion}} |
| - name: "Cache pre-commit env" |
| uses: actions/cache@v2 |
| with: |
| path: ~/.cache/pre-commit |
| key: pre-commit-pylint-${{ hashFiles('.pre-commit-config.yaml') }} |
| restore-keys: pre-commit-pylint- |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Static checks: pylint" |
| run: ./scripts/ci/static_checks/run_static_checks.sh pylint |
| env: |
| VERBOSE: false |
| |
| docs: |
| timeout-minutes: 45 |
| name: "Build docs" |
| runs-on: ubuntu-20.04 |
| needs: [build-info, ci-images] |
| if: needs.build-info.outputs.docs-build == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Build docs" |
| run: ./scripts/ci/docs/ci_docs.sh |
| - name: "Upload documentation" |
| uses: actions/upload-artifact@v2 |
| if: always() && github.event_name == 'pull_request' |
| with: |
| name: airflow-documentation |
| path: "./files/documentation" |
| retention-days: 7 |
| - name: Configure AWS credentials |
| uses: ./.github/actions/configure-aws-credentials |
| if: > |
| github.ref == 'refs/heads/master' && github.repository == 'apache/airflow' && |
| github.event_name == 'push' |
| with: |
| aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} |
| aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} |
| aws-region: eu-central-1 |
| - name: "Upload documentation to AWS S3" |
| if: > |
| github.ref == 'refs/heads/master' && github.repository == 'apache/airflow' && |
| github.event_name == 'push' |
| run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs |
| |
| prepare-backport-provider-packages: |
| timeout-minutes: 30 |
| name: "Backport packages: ${{ matrix.package-format }}" |
| runs-on: ubuntu-20.04 |
| needs: [build-info, ci-images] |
| strategy: |
| matrix: |
| package-format: ['wheel', 'sdist'] |
| env: |
| # In this case we want to install airflow from the latest released 1.10 version |
| # all provider packages are installed from wheels or .tar.gz files |
| INSTALL_AIRFLOW_VERSION: "1.10.14" |
| AIRFLOW_EXTRAS: "all" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} |
| BACKPORT_PACKAGES: "true" |
| VERSION_SUFFIX_FOR_PYPI: "dev" |
| PACKAGE_FORMAT: ${{ matrix.package-format }} |
| if: needs.build-info.outputs.image-build == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Prepare provider readmes" |
| run: ./scripts/ci/provider_packages/ci_prepare_provider_readmes.sh |
| - name: "Prepare provider packages: ${{ matrix.package-format }}" |
| run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh |
| - name: "Install and test provider packages and airflow via ${{ matrix.package-format }} files" |
| run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh |
| env: |
| INSTALL_PROVIDERS_FROM_SOURCES: "false" |
| - name: "Upload package artifacts" |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: airflow-backport-packages |
| path: "./dist/apache*" |
| retention-days: 7 |
| - name: "Upload readme artifacts" |
| uses: actions/upload-artifact@v2 |
| if: always() && matrix.package-format == 'wheel' |
| with: |
| name: airflow-backport-readmes |
| path: "./files/airflow-readme-*" |
| retention-days: 7 |
| |
| prepare-provider-packages: |
| timeout-minutes: 30 |
| name: "Provider packages ${{ matrix.package-format }}" |
| runs-on: ubuntu-20.04 |
| needs: [build-info, ci-images] |
| env: |
| INSTALL_AIRFLOW_VERSION: "${{ matrix.package-format }}" |
| AIRFLOW_EXTRAS: "all" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} |
| VERSION_SUFFIX_FOR_PYPI: "dev" |
| PACKAGE_FORMAT: ${{ matrix.package-format }} |
| strategy: |
| matrix: |
| package-format: ['wheel', 'sdist'] |
| if: needs.build-info.outputs.image-build == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Prepare provider readmes" |
| run: ./scripts/ci/provider_packages/ci_prepare_provider_readmes.sh |
| - name: "Prepare provider packages: ${{ matrix.package-format }}" |
| run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh |
| - name: "Prepare airflow packages: ${{ matrix.package-format }}" |
| run: ./scripts/ci/build_airflow/ci_build_airflow_package.sh |
| - name: "Install and test provider packages and airflow via ${{ matrix.package-format }} files" |
| run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh |
| - name: "Upload package artifacts" |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: airflow-provider-packages |
| path: "./dist/apache-*" |
| retention-days: 7 |
| - name: "Upload readme artifacts" |
| uses: actions/upload-artifact@v2 |
| if: always() && matrix.package-format == 'wheel' |
| with: |
| name: airflow-provider-readmes |
| path: "./files/airflow-readme-*" |
| retention-days: 7 |
| |
| test-provider-packages-released-airflow: |
| timeout-minutes: 30 |
| name: "Test Provider packages with 2.0.0 version ${{ matrix.package-format }}" |
| runs-on: ubuntu-20.04 |
| needs: [build-info, ci-images] |
| env: |
| INSTALL_AIRFLOW_VERSION: "2.0.0" |
| AIRFLOW_EXTRAS: "all" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} |
| VERSION_SUFFIX_FOR_PYPI: "dev" |
| PACKAGE_FORMAT: ${{ matrix.package-format }} |
| strategy: |
| matrix: |
| package-format: ['wheel', 'sdist'] |
| if: needs.build-info.outputs.image-build == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Prepare provider readmes" |
| run: ./scripts/ci/provider_packages/ci_prepare_provider_readmes.sh |
| - name: "Prepare provider packages: ${{ matrix.package-format }}" |
| run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh |
| - name: "Install and test provider packages and airflow via ${{ matrix.package-format }} files" |
| run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh |
| |
| tests-helm: |
| timeout-minutes: 20 |
| name: "Python unit tests for helm chart" |
| runs-on: ubuntu-20.04 |
| needs: [build-info, ci-images] |
| env: |
| MOUNT_LOCAL_SOURCES: "true" |
| RUN_TESTS: true |
| TEST_TYPES: "Helm" |
| BACKEND: "sqlite" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} |
| if: > |
| needs.build-info.outputs.needs-helm-tests == 'true' && |
| (github.repository == 'apache/airflow' || github.event_name != 'schedule') |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Tests: ${{needs.build-info.outputs.testTypes}}" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: airflow-logs-helm |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: container-logs-helm |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: > |
| coverage-helm |
| path: "./files/coverage.xml" |
| retention-days: 7 |
| |
| tests-postgres: |
| timeout-minutes: 130 |
| name: > |
| Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}: |
| ${{needs.build-info.outputs.testTypes}} |
| runs-on: ubuntu-20.04 |
| needs: [build-info, ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }} |
| fail-fast: false |
| env: |
| BACKEND: postgres |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| POSTGRES_VERSION: ${{ matrix.postgres-version }} |
| RUN_TESTS: true |
| TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" |
| TEST_TYPE: "" |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Tests: ${{needs.build-info.outputs.testTypes}}" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: airflow-logs-${{matrix.python-version}}-${{matrix.postgres-version}} |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: container-logs-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: > |
| coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} |
| path: "./files/coverage.xml" |
| retention-days: 7 |
| |
| tests-mysql: |
| timeout-minutes: 130 |
| name: > |
| MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} |
| runs-on: ubuntu-20.04 |
| needs: [build-info, ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }} |
| fail-fast: false |
| env: |
| BACKEND: mysql |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| MYSQL_VERSION: ${{ matrix.mysql-version }} |
| RUN_TESTS: true |
| TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" |
| TEST_TYPE: "" |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Tests: ${{needs.build-info.outputs.testTypes}}" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: airflow-logs-${{matrix.python-version}}-${{matrix.mysql-version}} |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: container-logs-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} |
| path: "./files/coverage.xml" |
| retention-days: 7 |
| |
| tests-sqlite: |
| timeout-minutes: 130 |
| name: > |
| Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} |
| runs-on: ubuntu-20.04 |
| needs: [build-info, ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.sqliteExclude) }} |
| fail-fast: false |
| env: |
| BACKEND: sqlite |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| RUN_TESTS: true |
| TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" |
| TEST_TYPE: "" |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Tests: ${{needs.build-info.outputs.testTypes}}" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: airflow-logs-${{matrix.python-version}} |
| path: './files/airflow_logs*' |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: container-logs-sqlite-${{matrix.python-version}} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: coverage-sqlite-${{matrix.python-version}} |
| path: ./files/coverage.xml |
| retention-days: 7 |
| |
| tests-quarantined: |
| timeout-minutes: 60 |
| name: "Quarantined tests" |
| runs-on: ubuntu-20.04 |
| continue-on-error: true |
| needs: [build-info, ci-images] |
| strategy: |
| matrix: |
| include: |
| - backend: mysql |
| - backend: postgres |
| - backend: sqlite |
| env: |
| BACKEND: ${{ matrix.backend }} |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} |
| MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}} |
| POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}} |
| RUN_TESTS: true |
| TEST_TYPES: "Quarantined" |
| TEST_TYPE: "" |
| NUM_RUNS: 10 |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Set issue id for master" |
| if: github.ref == 'refs/heads/master' |
| run: | |
| echo "ISSUE_ID=10118" >> $GITHUB_ENV |
| - name: "Set issue id for v1-10-stable" |
| if: github.ref == 'refs/heads/v1-10-stable' |
| run: | |
| echo "ISSUE_ID=10127" >> $GITHUB_ENV |
| - name: "Set issue id for v1-10-test" |
| if: github.ref == 'refs/heads/v1-10-test' |
| run: | |
| echo "ISSUE_ID=10128" >> $GITHUB_ENV |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Tests: Quarantined" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| - name: "Upload Quarantine test results" |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: quarantined_tests |
| path: "files/test_result.xml" |
| retention-days: 7 |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: airflow-logs-quarantined-${{ matrix.backend }} |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: container-logs-quarantined-${{ matrix.backend }} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: coverage-quarantined-${{ matrix.backend }} |
| path: "./files/coverage.xml" |
| retention-days: 7 |
| |
| upload-coverage: |
| timeout-minutes: 5 |
| name: "Upload coverage" |
| runs-on: ubuntu-20.04 |
| continue-on-error: true |
| needs: |
| - tests-kubernetes |
| - tests-postgres |
| - tests-sqlite |
| - tests-mysql |
| - tests-quarantined |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Download all artifacts from the current build" |
| uses: actions/download-artifact@v2 |
| with: |
| path: ./coverage-files |
| - name: "Removes unnecessary artifacts" |
| run: ls ./coverage-files | grep -v coverage | xargs rm -rf |
| - name: "Upload all coverage reports to codecov" |
| uses: ./.github/actions/codecov-action |
| with: |
| directory: "./coverage-files" |
| |
| prod-images: |
| timeout-minutes: 120 |
| name: "Wait for PROD images" |
| runs-on: ubuntu-20.04 |
| needs: [build-info, ci-images] |
| env: |
| BACKEND: sqlite |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} |
| if: needs.build-info.outputs.image-build == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| if: needs.build-info.outputs.waitForImage == 'true' |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| if: needs.build-info.outputs.waitForImage == 'true' |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| if: needs.build-info.outputs.waitForImage == 'true' |
| - name: > |
| Wait for PROD images |
| ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }} |
| env: |
| CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > |
| ${{needs.build-info.outputs.pythonVersionsListAsString}} |
| run: ./scripts/ci/images/ci_wait_for_all_prod_images.sh |
| if: needs.build-info.outputs.waitForImage == 'true' |
| |
| verify-prod-images: |
| timeout-minutes: 20 |
| name: "Verify Prod Image Py${{matrix.python-version}}" |
| runs-on: ubuntu-20.04 |
| needs: [build-info, prod-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| env: |
| BACKEND: sqlite |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| if: needs.build-info.outputs.waitForImage == 'true' |
| with: |
| persist-credentials: false |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| if: needs.build-info.outputs.waitForImage == 'true' |
| - name: "Prepare PROD Image" |
| run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh |
| - name: "Verify PROD image Py${{matrix.python-version}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_verify_prod_image.sh |
| if: needs.build-info.outputs.waitForImage == 'true' |
| |
| tests-kubernetes: |
| timeout-minutes: 50 |
| name: K8s ${{matrix.python-version}} ${{matrix.kubernetes-version}} ${{matrix.kubernetes-mode}} |
| runs-on: ubuntu-20.04 |
| needs: [build-info, prod-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| kubernetes-version: ${{ fromJson(needs.build-info.outputs.kubernetesVersions) }} |
| kubernetes-mode: ${{ fromJson(needs.build-info.outputs.kubernetesModes) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.kubernetesExclude) }} |
| fail-fast: false |
| env: |
| BACKEND: postgres |
| RUN_TESTS: "true" |
| RUNTIME: "kubernetes" |
| PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" |
| KUBERNETES_MODE: "${{ matrix.kubernetes-mode }}" |
| KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}" |
| KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" |
| HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" |
| if: needs.build-info.outputs.run-kubernetes-tests == 'true' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare PROD Image" |
| run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh |
| - name: "Setup cluster and deploy Airflow" |
| id: setp-cluster-deploy-app |
| run: ./scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh |
| env: |
| # We have the right image pulled already by the previous step |
| SKIP_BUILDING_PROD_IMAGE: "true" |
| - name: "Cache virtualenv for kubernetes testing" |
| uses: actions/cache@v2 |
| with: |
| path: ".build/.kubernetes_venv_ ${{ needs.build-info.outputs.defaultPythonVersion }}" |
| key: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}\ |
| -${{ hashFiles('setup.py','setup.cfg') }}" |
| restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-" |
| - name: "Cache bin folder with tools for kubernetes testing" |
| uses: actions/cache@v2 |
| with: |
| path: ".build/bin" |
| key: "bin-${{ matrix.kubernetes-version }}\ |
| -${{ needs.build-info.outputs.defaultKindVersion }}\ |
| -${{ needs.build-info.outputs.defaultHelmVersion }}" |
| restore-keys: "bin-${{ matrix.kubernetes-version }}" |
| - name: "Kubernetes Tests" |
| run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh |
| - name: "Upload KinD logs" |
| uses: actions/upload-artifact@v2 |
| if: failure() |
| with: |
| name: > |
| kind-logs-${{matrix.kubernetes-mode}}-${{matrix.python-version}}-${{matrix.kubernetes-version}} |
| path: /tmp/kind_logs_* |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: > |
| coverage-k8s-${{matrix.kubernetes-mode}}-${{matrix.python-version}}-${{matrix.kubernetes-version}} |
| path: "./files/coverage.xml" |
| retention-days: 7 |
| |
| push-prod-images-to-github-registry: |
| timeout-minutes: 10 |
| name: "Push PROD images as cache to GitHub Registry" |
| runs-on: ubuntu-20.04 |
| needs: |
| - build-info |
| - static-checks |
| - static-checks-pylint |
| - tests-sqlite |
| - tests-postgres |
| - tests-mysql |
| - tests-kubernetes |
| - prepare-backport-provider-packages |
| - prepare-provider-packages |
| - test-provider-packages-released-airflow |
| - prod-images |
| - verify-prod-images |
| - docs |
| if: > |
| (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' || |
| github.ref == 'refs/heads/v2-0-test') && |
| github.event_name != 'schedule' |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| env: |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: |
| "Prepare PROD image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh |
| env: |
| # Since we are going to push both final image and build image segment, we need to pull the |
| # build image, in case we are pulling from registry rather than building. |
| WAIT_FOR_PROD_BUILD_IMAGE: "true" |
| - name: "Push PROD images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_push_production_images.sh |
| |
| push-ci-images-to-github-registry: |
| timeout-minutes: 10 |
| name: "Push CI images as cache to GitHub Registry" |
| runs-on: ubuntu-20.04 |
| needs: |
| - build-info |
| - static-checks |
| - static-checks-pylint |
| - tests-sqlite |
| - tests-postgres |
| - tests-mysql |
| - tests-kubernetes |
| - prepare-backport-provider-packages |
| - ci-images |
| - verify-ci-images |
| - docs |
| if: > |
| (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' || |
| github.ref == 'refs/heads/v2-0-test') && |
| github.event_name != 'schedule' |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| env: |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Push CI image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" |
| run: ./scripts/ci/images/ci_push_ci_images.sh |
| |
| constraints: |
| timeout-minutes: 10 |
| name: "Constraints" |
| runs-on: ubuntu-20.04 |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} |
| fail-fast: false |
| needs: |
| - build-info |
| - ci-images |
| env: |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| if: > |
| github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' || |
| github.ref == 'refs/heads/v2-0-test' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ github.sha }}" |
| run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh |
| - name: "Generate constraints" |
| run: ./scripts/ci/constraints/ci_generate_constraints.sh |
| - name: "Upload constraint artifacts" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: 'constraints-${{matrix.python-version}}' |
| path: './files/constraints-${{matrix.python-version}}/constraints-${{matrix.python-version}}.txt' |
| retention-days: 7 |
| |
| constraints-push: |
| timeout-minutes: 10 |
| name: "Constraints push" |
| runs-on: ubuntu-20.04 |
| needs: |
| - build-info |
| - constraints |
| - verify-ci-images |
| - verify-prod-images |
| - static-checks |
| - static-checks-pylint |
| - tests-sqlite |
| - tests-mysql |
| - tests-postgres |
| - tests-kubernetes |
| if: > |
| github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' || |
| github.ref == 'refs/heads/v2-0-test' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Set constraints branch name" |
| id: constraints-branch |
| run: ./scripts/ci/constraints/ci_branch_constraints.sh |
| - name: Checkout ${{ steps.constraints-branch.outputs.branch }} |
| uses: actions/checkout@v2 |
| with: |
| path: "repo" |
| ref: ${{ steps.constraints-branch.outputs.branch }} |
| persist-credentials: false |
| - name: "Get all artifacts (constraints)" |
| uses: actions/download-artifact@v2 |
| with: |
| path: 'artifacts' |
| - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}" |
| run: ./scripts/ci/constraints/ci_commit_constraints.sh |
| - name: "Push changes" |
| uses: ./.github/actions/github-push-action |
| with: |
| github_token: ${{ secrets.GITHUB_TOKEN }} |
| branch: ${{ steps.constraints-branch.outputs.branch }} |
| directory: "repo" |
| |
| tag-repo-nightly: |
| timeout-minutes: 10 |
| name: "Tag repo nightly" |
| runs-on: ubuntu-20.04 |
| needs: |
| - docs |
| - static-checks |
| - static-checks-pylint |
| - tests-sqlite |
| - tests-postgres |
| - tests-mysql |
| - tests-kubernetes |
| - constraints-push |
| - prepare-provider-packages |
| if: github.event_name == 'schedule' && github.repository == 'apache/airflow' |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Free space" |
| run: ./scripts/ci/tools/ci_free_space_on_ci.sh |
| - name: "Tag commit" |
| run: | |
| BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///') |
| echo "Tagging ${BRANCH_NAME}" |
| git tag -f nightly-${BRANCH_NAME} HEAD |
| - name: "Push tags" |
| uses: ./.github/actions/github-push-action |
| with: |
| github_token: ${{ secrets.GITHUB_TOKEN }} |
| tags: true |
| force: true |
| branch: master |
