| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| --- |
| name: Tests |
| on: # yamllint disable-line rule:truthy |
| schedule: |
| - cron: '28 0 * * *' |
| push: |
| branches: ['main', 'v[0-9]+-[0-9]+-test'] |
| pull_request: |
| branches: ['main', 'v[0-9]+-[0-9]+-test', 'v[0-9]+-[0-9]+-stable'] |
| permissions: |
| # All other permissions are set to none |
| contents: read |
| packages: read |
| env: |
| MOUNT_SELECTED_LOCAL_SOURCES: "false" |
| ANSWER: "yes" |
| CHECK_IMAGE_FOR_REBUILD: "true" |
| SKIP_CHECK_REMOTE_IMAGE: "true" |
| DEBIAN_VERSION: "bullseye" |
| DB_RESET: "true" |
| VERBOSE: "true" |
| GITHUB_REPOSITORY: ${{ github.repository }} |
| GITHUB_USERNAME: ${{ github.actor }} |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the |
| # Airflow one is going to be used |
| CONSTRAINTS_GITHUB_REPOSITORY: >- |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} |
| # In builds from forks, this token is read-only. For scheduler/direct push it is WRITE one |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| ENABLE_TEST_COVERAGE: "${{ github.event_name == 'push' }}" |
| IMAGE_TAG_FOR_THE_BUILD: "${{ github.event.pull_request.head.sha || github.sha }}" |
| USE_SUDO: "true" |
| |
| concurrency: |
| group: ci-${{ github.event.pull_request.number || github.ref }} |
| cancel-in-progress: true |
| |
| jobs: |
| |
| build-info: |
| name: "Build info" |
| # The runs-on cannot refer to env. or secrets. context, so we have no |
| # option but to specify a hard-coded list here. This is "safe", as the list |
| # is checked again by the runner using it's own list, so a PR author cannot |
| # change this and get access to our self-hosted runners |
| # |
| # When changing this list, ensure that it is kept in sync with the |
| # /runners/apache/airflow/configOverlay |
| # parameter in AWS SSM ParameterStore (which is what the runner uses) |
| # and restart the self-hosted runners. |
| # |
| # This list of committers can be generated with: |
| # https://github.com/apache/airflow-ci-infra/blob/main/scripts/list_committers |
| runs-on: >- |
| ${{ ( |
| ( |
| github.event_name == 'push' || |
| github.event_name == 'schedule' || |
| contains(fromJSON('[ |
| "BasPH", |
| "Fokko", |
| "KevinYang21", |
| "XD-DENG", |
| "aijamalnk", |
| "alexvanboxel", |
| "aoen", |
| "artwr", |
| "ashb", |
| "bbovenzi", |
| "bolkedebruin", |
| "criccomini", |
| "dimberman", |
| "dstandish", |
| "eladkal", |
| "ephraimbuddy", |
| "feluelle", |
| "feng-tao", |
| "houqp", |
| "jedcunningham", |
| "jgao54", |
| "jghoman", |
| "jhtimmins", |
| "jmcarp", |
| "josh-fell", |
| "kaxil", |
| "leahecole", |
| "malthe", |
| "mik-laj", |
| "milton0825", |
| "mistercrunch", |
| "msumit", |
| "pingzh", |
| "potiuk", |
| "r39132", |
| "ryanahamilton", |
| "ryw", |
| "saguziel", |
| "sekikn", |
| "turbaszek", |
| "uranusjr", |
| "vikramkoka", |
| "xinbinhuang", |
| "yuqian90", |
| "zhongjiajie" |
| ]'), github.event.pull_request.user.login) |
| ) && github.repository == 'apache/airflow' |
| ) && 'self-hosted' || 'ubuntu-20.04' }} |
| env: |
| GITHUB_CONTEXT: ${{ toJson(github) }} |
| outputs: |
| debian-version: ${{ steps.selective-checks.outputs.debian-version }} |
| cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} |
| upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} |
| python-versions: ${{ steps.selective-checks.outputs.python-versions }} |
| python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} |
| all-python-versions-list-as-string: >- |
| ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} |
| default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} |
| min-max-python-versions-as-string: >- |
| ${{ steps.selective-checks.outputs.min-max-python-versions-as-string }} |
| kubernetes-versions-list-as-string: >- |
| ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} |
| default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} |
| min-max-kubernetes-versions-as-string: >- |
| ${{ steps.selective-checks.outputs.min-max-kubernetes-versions-as-string }} |
| postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} |
| default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} |
| mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} |
| mssql-versions: ${{ steps.selective-checks.outputs.mssql-versions }} |
| default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} |
| default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} |
| default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} |
| test-types: ${{ steps.selective-checks.outputs.test-types }} |
| postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} |
| mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} |
| mssql-exclude: ${{ steps.selective-checks.outputs.mssql-exclude }} |
| sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} |
| run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} |
| run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} |
| basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} |
| image-build: ${{ steps.selective-checks.outputs.image-build }} |
| docs-build: ${{ steps.selective-checks.outputs.docs-build }} |
| needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} |
| needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} |
| needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} |
| default-branch: ${{ steps.selective-checks.outputs.default-branch }} |
| default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} |
| docs-filter: ${{ steps.selective-checks.outputs.docs-filter }} |
| skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} |
| source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} |
| pull-request-labels: ${{ steps.source-run-info.outputs.pullRequestLabels }} |
| runs-on: ${{ steps.set-runs-on.outputs.runs-on }} |
| run-coverage: ${{ steps.set-run-coverage.outputs.run-coverage }} |
| in-workflow-build: ${{ steps.set-in-workflow-build.outputs.in-workflow-build }} |
| build-job-description: ${{ steps.set-in-workflow-build.outputs.build-job-description }} |
| merge-run: ${{ steps.set-merge-run.outputs.merge-run }} |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Get information about the PR" |
| uses: ./.github/actions/get-workflow-origin |
| id: source-run-info |
| with: |
| token: ${{ secrets.GITHUB_TOKEN }} |
| - name: Fetch incoming commit ${{ github.sha }} with its parent |
| uses: actions/checkout@v3 |
| with: |
| ref: ${{ github.sha }} |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| # We do not have output from selective checks yet, so we need to hardcode python |
| python-version: 3.7 |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - name: "Retrieve defaults from branch_defaults.py" |
| id: defaults |
| # We could retrieve it differently here - by just importing the variables and |
| # printing them from python code, however we want to have the same code as used in |
| # the build-images.yml (there we cannot import python code coming from the PR - we need to |
| # treat the python code as text and extract the variables from there. |
| run: | |
| python - <<EOF >>$GITHUB_ENV |
| from pathlib import Path |
| import re |
| import sys |
| |
| DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text() |
| BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$' |
| CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$' |
| DEBIAN_VERSION_PATTERN = r'^DEBIAN_VERSION = "(.*)"$' |
| |
| branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) |
| constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) |
| debian_version = re.search(DEBIAN_VERSION_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) |
| |
| output = f""" |
| DEFAULT_BRANCH={branch} |
| DEFAULT_CONSTRAINTS_BRANCH={constraints_branch} |
| DEBIAN_VERSION={debian_version} |
| """.strip() |
| |
| print(output) |
| # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log |
| print(output, file=sys.stderr) |
| EOF |
| - run: ./scripts/ci/install_breeze.sh |
| - name: Selective checks |
| id: selective-checks |
| env: |
| PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}" |
| COMMIT_REF: "${{ github.sha }}" |
| run: breeze ci selective-check |
| # Avoid having to specify the runs-on logic every time. We use the custom |
| # env var AIRFLOW_SELF_HOSTED_RUNNER set only on our runners, but never |
| # on the public runners |
| - name: Set runs-on |
| id: set-runs-on |
| env: |
| PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}" |
| run: | |
| if [[ ${PR_LABELS=} == *"use public runners"* ]]; then |
| echo "Forcing running on Public Runners via `use public runners` label" |
| echo "::set-output name=runs-on::\"ubuntu-20.04\"" |
| elif [[ ${AIRFLOW_SELF_HOSTED_RUNNER} == "" ]]; then |
| echo "Regular PR running with Public Runner" |
| echo "::set-output name=runs-on::\"ubuntu-20.04\"" |
| else |
| echo "Maintainer or main run running with self-hosted runner" |
| echo "::set-output name=runs-on::\"self-hosted\"" |
| fi |
| # Avoid having to specify the coverage logic every time. |
| - name: Set run coverage |
| id: set-run-coverage |
| run: echo "::set-output name=run-coverage::true" |
| if: > |
| github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && |
| github.event_name == 'push' && |
| steps.selective-checks.outputs.default-branch == 'main' |
| - name: Determine where to run image builds |
| id: set-in-workflow-build |
| # Run in-workflow build image when: |
| # * direct push is run |
| # * schedule build is run |
| # * pull request is run not from fork |
| run: | |
| set -x |
| if [[ ${GITHUB_EVENT_NAME} == "push" || ${GITHUB_EVENT_NAME} == "push" || \ |
| ${{steps.source-run-info.outputs.sourceHeadRepo}} == "apache/airflow" ]]; then |
| echo "Images will be built in current workflow" |
| echo "::set-output name=in-workflow-build::true" |
| echo "::set-output name=build-job-description::Build" |
| else |
| echo "Images will be built in pull_request_target workflow" |
| echo "::set-output name=in-workflow-build::false" |
| echo "::set-output name=build-job-description::Skip Build (pull_request_target)" |
| fi |
| - name: Determine if this is merge run |
| id: set-merge-run |
| run: echo "::set-output name=merge-run::true" |
| # Only in Apache Airflow repo, when there is a merge run to main or any of v2*test branches |
| if: | |
| github.repository == 'apache/airflow' && github.event_name == 'push' && |
| ( |
| github.ref_name == 'main' || |
| startsWith(github.ref_name, 'v2') && endsWith(github.ref_name, 'test') |
| ) |
| - name: env |
| run: printenv |
| env: |
| PR_LABELS: ${{ steps.source-run-info.outputs.pullRequestLabels }} |
| GITHUB_CONTEXT: ${{ toJson(github) }} |
| |
| build-ci-images: |
| permissions: |
| packages: write |
| timeout-minutes: 80 |
| name: >- |
| ${{needs.build-info.outputs.build-job-description}} CI images |
| ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info] |
| env: |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} |
| DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }} |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }} |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - uses: actions/checkout@v3 |
| with: |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} |
| persist-credentials: false |
| submodules: recursive |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - run: ./scripts/ci/install_breeze.sh |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: "Free space" |
| run: breeze ci free-space |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: Cache pre-commit envs |
| uses: actions/cache@v3 |
| with: |
| path: ~/.cache/pre-commit |
| key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ |
| ${{ hashFiles('.pre-commit-config.yaml') }}" |
| restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: "Regenerate dependencies in case they was modified manually so that we can build an image" |
| run: > |
| breeze static-checks --type update-providers-dependencies --all-files |
| --show-diff-on-failure --color always || true |
| if: > |
| needs.build-info.outputs.in-workflow-build == 'true' && |
| needs.build-info.outputs.default-branch == 'main' |
| - name: Compile www assets |
| run: breeze compile-www-assets |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: > |
| Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| run: breeze ci-image build --push --tag-as-latest --run-in-parallel |
| env: |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: "Candidates for pip resolver backtrack triggers" |
| if: failure() || cancelled() |
| run: > |
| breeze ci find-newer-dependencies --max-age 1 |
| --python "${{ needs.build-info.outputs.default-python-version }}" |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() && needs.build-info.outputs.in-workflow-build == 'true' |
| |
| build-prod-images: |
| permissions: |
| packages: write |
| timeout-minutes: 80 |
| name: > |
| ${{needs.build-info.outputs.build-job-description}} PROD images |
| ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, build-ci-images] |
| env: |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} |
| DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }} |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }} |
| BACKEND: sqlite |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| VERSION_SUFFIX_FOR_PYPI: "dev0" |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - uses: actions/checkout@v3 |
| with: |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} |
| persist-credentials: false |
| submodules: recursive |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - run: ./scripts/ci/install_breeze.sh |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: "Free space" |
| run: breeze ci free-space |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: Cache pre-commit envs |
| uses: actions/cache@v3 |
| with: |
| path: ~/.cache/pre-commit |
| key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ |
| ${{ hashFiles('.pre-commit-config.yaml') }}" |
| restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} |
| if: > |
| needs.build-info.outputs.in-workflow-build == 'true' && |
| needs.build-info.outputs.default-branch == 'main' |
| - name: "Regenerate dependencies in case they was modified manually so that we can build an image" |
| run: > |
| breeze static-checks --type update-providers-dependencies --all-files |
| --show-diff-on-failure --color always || true |
| if: > |
| needs.build-info.outputs.in-workflow-build == 'true' && |
| needs.build-info.outputs.default-branch == 'main' |
| - name: > |
| Pull CI image for PROD build: |
| ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}" |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| # Always use default Python version of CI image for preparing packages |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: "Cleanup dist and context file" |
| run: rm -fv ./dist/* ./docker-context-files/* |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: "Prepare providers packages" |
| run: > |
| breeze release-management prepare-provider-packages |
| --package-list-file ./scripts/ci/installed_providers.txt |
| --package-format wheel --version-suffix-for-pypi dev0 |
| if: > |
| needs.build-info.outputs.in-workflow-build == 'true' && |
| needs.build-info.outputs.default-branch == 'main' |
| - name: "Prepare airflow package" |
| run: > |
| breeze release-management prepare-airflow-package |
| --package-format wheel --version-suffix-for-pypi dev0 |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: "Move dist packages to docker-context files" |
| run: mv -v ./dist/*.whl ./docker-context-files |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: > |
| Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| run: > |
| breeze prod-image build |
| --tag-as-latest |
| --run-in-parallel |
| --push |
| --install-packages-from-context |
| env: |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() && needs.build-info.outputs.in-workflow-build == 'true' |
| |
| run-new-breeze-tests: |
| timeout-minutes: 10 |
| name: Breeze unit tests |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info] |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: python -m pip install --editable ./dev/breeze/ |
| - run: python -m pytest ./dev/breeze/ -n auto --color=yes |
| - run: breeze setup version |
| |
| tests-ui: |
| timeout-minutes: 10 |
| name: React UI tests |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info] |
| if: needs.build-info.outputs.run-ui-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup node" |
| uses: actions/setup-node@v3 |
| with: |
| node-version: 14 |
| - name: "Cache eslint" |
| uses: actions/cache@v2 |
| with: |
| path: 'airflow/ui/node_modules' |
| key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }} |
| - run: yarn --cwd airflow/ui/ install --frozen-lockfile --non-interactive |
| - run: yarn --cwd airflow/ui/ run test |
| env: |
| FORCE_COLOR: 2 |
| |
| tests-www: |
| timeout-minutes: 10 |
| name: React WWW tests |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info] |
| if: needs.build-info.outputs.run-www-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup node" |
| uses: actions/setup-node@v3 |
| with: |
| node-version: 14 |
| - name: "Cache eslint" |
| uses: actions/cache@v3 |
| with: |
| path: 'airflow/www/node_modules' |
| key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }} |
| - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive |
| - run: yarn --cwd airflow/www/ run test |
| env: |
| FORCE_COLOR: 2 |
| |
| |
| test-openapi-client-generation: |
| timeout-minutes: 10 |
| name: "Test OpenAPI client generation" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info] |
| if: needs.build-info.outputs.needs-api-codegen == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Generate client codegen diff" |
| run: ./scripts/ci/openapi/client_codegen_diff.sh |
| |
| test-examples-of-prod-image-building: |
| timeout-minutes: 60 |
| name: "Test examples of production image building" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info] |
| if: needs.build-info.outputs.image-build == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/requirements.txt |
| - name: "Test examples of PROD image building" |
| run: > |
| python -m pip install -r ./docker_tests/requirements.txt && |
| python -m pytest docker_tests/test_examples_of_prod_image_building.py -n auto --color=yes |
| |
| wait-for-ci-images: |
| timeout-minutes: 120 |
| name: "Wait for CI images" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, build-ci-images] |
| if: needs.build-info.outputs.image-build == 'true' |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| BACKEND: sqlite |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| id: wait-for-images |
| run: breeze ci-image pull --run-in-parallel --verify --wait-for-image --tag-as-latest |
| env: |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| static-checks: |
| timeout-minutes: 30 |
| name: "Static checks" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} |
| if: needs.build-info.outputs.basic-checks-only == 'false' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - name: Cache pre-commit envs |
| uses: actions/cache@v3 |
| with: |
| path: ~/.cache/pre-commit |
| key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ |
| ${{ hashFiles('.pre-commit-config.yaml') }}" |
| restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: > |
| Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Get Python version" |
| run: "echo \"::set-output name=host-python-version::$(python -c |
| 'import platform; print(platform.python_version())')\"" |
| id: host-python-version |
| - name: "Static checks" |
| run: breeze static-checks --all-files --show-diff-on-failure --color always |
| env: |
| VERBOSE: "false" |
| SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} |
| COLUMNS: "250" |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| # Those checks are run if no image needs to be built for checks. This is for simple changes that |
| # Do not touch any of the python code or any of the important files that might require building |
| # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host |
| static-checks-basic-checks-only: |
| timeout-minutes: 30 |
| name: "Static checks: basic checks only" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| if: needs.build-info.outputs.basic-checks-only == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: Cache pre-commit envs |
| uses: actions/cache@v3 |
| with: |
| path: ~/.cache/pre-commit |
| key: "pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}}-\ |
| ${{ hashFiles('.pre-commit-config.yaml') }}" |
| restore-keys: pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}} |
| - name: Fetch incoming commit ${{ github.sha }} with its parent |
| uses: actions/checkout@v3 |
| with: |
| ref: ${{ github.sha }} |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: "Get Python version" |
| run: "echo \"::set-output name=host-python-version::$(python -c |
| 'import platform; print(platform.python_version())')\"" |
| id: host-python-version |
| - name: "Static checks: basic checks only" |
| run: > |
| breeze static-checks --all-files --show-diff-on-failure --color always |
| --commit-ref "${{ github.sha }}" |
| env: |
| VERBOSE: "false" |
| SKIP_IMAGE_PRE_COMMITS: "true" |
| SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} |
| COLUMNS: "250" |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| docs: |
| timeout-minutes: 45 |
| name: "Build docs" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-ci-images] |
| if: needs.build-info.outputs.docs-build == 'true' |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - uses: actions/setup-python@v4 |
| with: |
| python-version: ${{needs.build-info.outputs.default-python-version}} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - uses: actions/cache@v3 |
| id: cache-doc-inventories |
| with: |
| path: ./docs/_inventory_cache/ |
| key: docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }} |
| restore-keys: | |
| docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }} |
| docs-inventory- |
| - name: "Build docs" |
| run: breeze build-docs ${{ needs.build-info.outputs.docs-filter }} |
| - name: Configure AWS credentials |
| uses: ./.github/actions/configure-aws-credentials |
| if: > |
| github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && |
| github.event_name == 'push' |
| with: |
| aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} |
| aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} |
| aws-region: eu-central-1 |
| - name: "Upload documentation to AWS S3" |
| if: > |
| github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && |
| github.event_name == 'push' |
| run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| prepare-test-provider-packages-wheel: |
| timeout-minutes: 40 |
| name: "Build and test provider packages wheel" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} |
| if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| if: needs.build-info.outputs.default-branch == 'main' |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: > |
| Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Cleanup dist files" |
| run: rm -fv ./dist/* |
| - name: "Prepare provider documentation" |
| run: breeze release-management prepare-provider-documentation --answer yes |
| - name: "Prepare provider packages: wheel" |
| run: > |
| breeze release-management prepare-provider-packages |
| --package-format wheel --version-suffix-for-pypi dev0 |
| - name: "Prepare airflow package: wheel" |
| run: > |
| breeze release-management prepare-airflow-package |
| --package-format wheel --version-suffix-for-pypi dev0 |
| - name: "Install and test provider packages and airflow via wheel files" |
| run: > |
| breeze release-management verify-provider-packages |
| --use-airflow-version wheel --use-packages-from-dist --package-format wheel |
| env: |
| SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" |
| - name: "Remove airflow package and replace providers with 2.2-compliant versions" |
| run: | |
| rm -vf dist/apache_airflow-*.whl \ |
| dist/apache_airflow_providers_cncf_kubernetes*.whl \ |
| dist/apache_airflow_providers_celery*.whl |
| pip download --no-deps --dest dist \ |
| apache-airflow-providers-cncf-kubernetes==3.0.0 \ |
| apache-airflow-providers-celery==2.1.3 |
| - name: "Install and test provider packages and airflow on Airflow 2.2 files" |
| run: > |
| breeze release-management verify-provider-packages --use-airflow-version 2.2.0 |
| --use-packages-from-dist --package-format wheel --airflow-constraints-reference constraints-2.2.0 |
| env: |
| # The extras below are all extras that should be installed with Airflow 2.2.0 |
| AIRFLOW_EXTRAS: "airbyte,alibaba,amazon,apache.atlas.apache.beam,apache.cassandra,apache.drill,\ |
| apache.druid,apache.hdfs,apache.hive,apache.kylin,apache.livy,apache.pig,apache.pinot,\ |
| apache.spark,apache.sqoop,apache.webhdfs,arangodb,asana,async,\ |
| celery,cgroups,cloudant,cncf.kubernetes,dask,databricks,datadog,dbt.cloud,\ |
| deprecated_api,dingding,discord,docker,\ |
| elasticsearch,exasol,facebook,ftp,github,github_enterprise,google,google_auth,\ |
| grpc,hashicorp,http,imap,influxdb,jdbc,jenkins,jira,kerberos,ldap,\ |
| leveldb,microsoft.azure,microsoft.mssql,microsoft.psrp,microsoft.winrm,mongo,mysql,\ |
| neo4j,odbc,openfaas,opsgenie,oracle,pagerduty,pandas,papermill,password,plexus,\ |
| postgres,presto,qubole,rabbitmq,redis,salesforce,samba,segment,sendgrid,sentry,\ |
| sftp,singularity,slack,snowflake,sqlite,ssh,statsd,tableau,telegram,trino,vertica,\ |
| virtualenv,yandex,zendesk" |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| prepare-test-provider-packages-sdist: |
| timeout-minutes: 80 |
| name: "Build and test provider packages sdist" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} |
| if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| if: needs.build-info.outputs.default-branch == 'main' |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: > |
| Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Cleanup dist files" |
| run: rm -fv ./dist/* |
| - name: "Prepare provider packages: sdist" |
| run: > |
| breeze release-management prepare-provider-packages |
| --package-format sdist --version-suffix-for-pypi dev0 |
| - name: "Prepare airflow package: sdist" |
| run: > |
| breeze release-management prepare-airflow-package |
| --package-format sdist --version-suffix-for-pypi dev0 |
| - name: "Upload provider distribution artifacts" |
| uses: actions/upload-artifact@v2 |
| with: |
| name: airflow-provider-packages |
| path: "./dist/apache-airflow-providers-*.tar.gz" |
| retention-days: 1 |
| - name: "Install and test provider packages and airflow via sdist files" |
| run: > |
| breeze release-management verify-provider-packages |
| --use-airflow-version sdist --use-packages-from-dist --package-format sdist |
| env: |
| SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| tests-helm: |
| timeout-minutes: 80 |
| name: "Python unit tests for Helm chart" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| MOUNT_SELECTED_LOCAL_SOURCES: "true" |
| TEST_TYPES: "Helm" |
| BACKEND: "" |
| DB_RESET: "false" |
| PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.default-python-version}} |
| if: > |
| needs.build-info.outputs.needs-helm-tests == 'true' && |
| (github.repository == 'apache/airflow' || github.event_name != 'schedule') && |
| needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: > |
| Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Tests: Helm" |
| run: breeze testing helm-tests |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() |
| with: |
| name: airflow-logs-helm |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() |
| with: |
| name: container-logs-helm |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| tests-postgres: |
| timeout-minutes: 130 |
| name: > |
| Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}: |
| ${{needs.build-info.outputs.test-types}} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} |
| postgres-version: ${{ fromJson(needs.build-info.outputs.postgres-versions) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.postgres-exclude) }} |
| fail-fast: false |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| BACKEND: postgres |
| POSTGRES_VERSION: ${{ matrix.postgres-version }} |
| TEST_TYPES: "${{needs.build-info.outputs.test-types}}" |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Test downgrade" |
| run: ./scripts/ci/testing/run_downgrade_test.sh |
| - name: "Test Offline SQL generation" |
| run: ./scripts/ci/testing/run_offline_sql_test.sh |
| - name: "Tests: ${{needs.build-info.outputs.test-types}}" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() |
| with: |
| name: airflow-logs-${{matrix.python-version}}-${{matrix.postgres-version}} |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() |
| with: |
| name: container-logs-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v3 |
| if: needs.build-info.outputs.run-coverage == 'true' |
| with: |
| name: coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} |
| path: "./files/coverage*.xml" |
| retention-days: 7 |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| tests-mysql: |
| timeout-minutes: 130 |
| name: > |
| MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} |
| mysql-version: ${{ fromJson(needs.build-info.outputs.mysql-versions) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.mysql-exclude) }} |
| fail-fast: false |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| BACKEND: mysql |
| MYSQL_VERSION: ${{ matrix.mysql-version }} |
| TEST_TYPES: "${{needs.build-info.outputs.test-types}}" |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Test downgrade" |
| run: ./scripts/ci/testing/run_downgrade_test.sh |
| - name: "Test Offline SQL generation" |
| run: ./scripts/ci/testing/run_offline_sql_test.sh |
| - name: "Tests: ${{needs.build-info.outputs.test-types}}" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() |
| with: |
| name: airflow-logs-${{matrix.python-version}}-${{matrix.mysql-version}} |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() |
| with: |
| name: container-logs-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v3 |
| if: needs.build-info.outputs.run-coverage == 'true' |
| with: |
| name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} |
| path: "./files/coverage*.xml" |
| retention-days: 7 |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| tests-mssql: |
| timeout-minutes: 130 |
| name: > |
| MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} |
| mssql-version: ${{ fromJson(needs.build-info.outputs.mssql-versions) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.mssql-exclude) }} |
| fail-fast: false |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| BACKEND: mssql |
| MSSQL_VERSION: ${{ matrix.mssql-version }} |
| TEST_TYPES: "${{needs.build-info.outputs.test-types}}" |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Test downgrade" |
| run: ./scripts/ci/testing/run_downgrade_test.sh |
| - name: "Tests: ${{needs.build-info.outputs.test-types}}" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() |
| with: |
| name: airflow-logs-${{matrix.python-version}}-${{matrix.mssql-version}} |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() |
| with: |
| name: container-logs-mssql-${{matrix.python-version}}-${{matrix.mssql-version}} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v3 |
| if: needs.build-info.outputs.run-coverage == 'true' |
| with: |
| name: coverage-mssql-${{matrix.python-version}}-${{matrix.mssql-version}} |
| path: "./files/coverage*.xml" |
| retention-days: 7 |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| tests-sqlite: |
| timeout-minutes: 130 |
| name: > |
| Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.sqlite-exclude) }} |
| fail-fast: false |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| BACKEND: sqlite |
| TEST_TYPES: "${{needs.build-info.outputs.test-types}}" |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Test downgrade" |
| run: ./scripts/ci/testing/run_downgrade_test.sh |
| - name: "Tests: ${{needs.build-info.outputs.test-types}}" |
| run: ./scripts/ci/testing/ci_run_airflow_testing.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() |
| with: |
| name: airflow-logs-${{matrix.python-version}} |
| path: './files/airflow_logs*' |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() |
| with: |
| name: container-logs-sqlite-${{matrix.python-version}} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v3 |
| if: needs.build-info.outputs.run-coverage == 'true' |
| with: |
| name: coverage-sqlite-${{matrix.python-version}} |
| path: ./files/coverage*.xml |
| retention-days: 7 |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| tests-quarantined: |
| timeout-minutes: 60 |
| name: "Quarantined tests" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| continue-on-error: true |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| MYSQL_VERSION: ${{needs.build-info.outputs.default-mysql-version}} |
| POSTGRES_VERSION: ${{needs.build-info.outputs.default-postgres-version}} |
| TEST_TYPES: "Quarantined" |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - name: "Set issue id for main" |
| if: github.ref == 'refs/heads/main' |
| run: | |
| echo "ISSUE_ID=10118" >> $GITHUB_ENV |
| - name: "Set issue id for v1-10-stable" |
| if: github.ref == 'refs/heads/v1-10-stable' |
| run: | |
| echo "ISSUE_ID=10127" >> $GITHUB_ENV |
| - name: "Set issue id for v1-10-test" |
| if: github.ref == 'refs/heads/v1-10-test' |
| run: | |
| echo "ISSUE_ID=10128" >> $GITHUB_ENV |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Tests: Quarantined" |
| run: ./scripts/ci/testing/ci_run_quarantined_tests.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" |
| - name: "Upload Quarantine test results" |
| uses: actions/upload-artifact@v3 |
| if: always() |
| with: |
| name: quarantined-tests |
| path: "files/test_result-*.xml" |
| retention-days: 7 |
| - name: "Upload airflow logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() |
| with: |
| name: airflow-logs-quarantined-${{ matrix.backend }} |
| path: "./files/airflow_logs*" |
| retention-days: 7 |
| - name: "Upload container logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() |
| with: |
| name: container-logs-quarantined-${{ matrix.backend }} |
| path: "./files/container_logs*" |
| retention-days: 7 |
| - name: "Upload artifact for coverage" |
| uses: actions/upload-artifact@v3 |
| if: needs.build-info.outputs.run-coverage == 'true' |
| with: |
| name: coverage-quarantined-${{ matrix.backend }} |
| path: "./files/coverage*.xml" |
| retention-days: 7 |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| upload-coverage: |
| timeout-minutes: 15 |
| name: "Upload coverage" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| continue-on-error: true |
| needs: |
| - build-info |
| - tests-postgres |
| - tests-sqlite |
| - tests-mysql |
| - tests-mssql |
| - tests-quarantined |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| # Only upload coverage on merges to main |
| if: needs.build-info.outputs.run-coverage == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Download all artifacts from the current build" |
| uses: actions/download-artifact@v3 |
| with: |
| path: ./coverage-files |
| - name: "Removes unnecessary artifacts" |
| run: ls ./coverage-files | grep -v coverage | xargs rm -rfv |
| - name: "Upload all coverage reports to codecov" |
| uses: ./.github/actions/codecov-action |
| with: |
| directory: "./coverage-files" |
| |
| wait-for-prod-images: |
| timeout-minutes: 120 |
| name: "Wait for PROD images" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-ci-images, build-prod-images] |
| if: needs.build-info.outputs.image-build == 'true' |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| BACKEND: sqlite |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: "Cache virtualenv environment" |
| uses: actions/cache@v3 |
| with: |
| path: '.build/.docker_venv' |
| key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} |
| - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| # We wait for the images to be available either from "build-images.yml' run as pull_request_target |
| # or from build-prod-images above. |
| # We are utilising single job to wait for all images because this job merely waits |
| # For the images to be available and test them. |
| run: breeze prod-image pull --verify --wait-for-image --run-in-parallel |
| env: |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| test-docker-compose-quick-start: |
| timeout-minutes: 60 |
| name: "Test docker-compose quick start" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-prod-images] |
| if: needs.build-info.outputs.image-build == 'true' |
| env: |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Pull PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze prod-image pull --tag-as-latest |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Test docker-compose quick start" |
| run: breeze testing docker-compose-tests |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| tests-kubernetes: |
| timeout-minutes: 240 |
| name: Helm Chart; ${{matrix.executor}} - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-prod-images] |
| strategy: |
| matrix: |
| executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] |
| fail-fast: false |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| BACKEND: postgres |
| RUN_TESTS: "true" |
| RUNTIME: "kubernetes" |
| KUBERNETES_MODE: "image" |
| EXECUTOR: ${{matrix.executor}} |
| KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}" |
| HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}" |
| CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > |
| ${{needs.build-info.outputs.python-versions-list-as-string}} |
| CURRENT_KUBERNETES_VERSIONS_AS_STRING: > |
| ${{needs.build-info.outputs.kubernetes-versions-list-as-string}} |
| if: > |
| ( needs.build-info.outputs.run-kubernetes-tests == 'true' || |
| needs.build-info.outputs.needs-helm-tests == 'true' ) && |
| needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze prod-image pull --run-in-parallel --tag-as-latest |
| env: |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Cache bin folder with tools for kubernetes testing" |
| uses: actions/cache@v3 |
| with: |
| path: ".build/kubernetes-bin" |
| key: "kubernetes-binaries |
| -${{ needs.build-info.outputs.default-kind-version }}\ |
| -${{ needs.build-info.outputs.default-helm-version }}" |
| restore-keys: "kubernetes-binaries" |
| - name: "Kubernetes Tests" |
| run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" |
| - name: "Upload KinD logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() || cancelled() |
| with: |
| name: kind-logs-${{matrix.executor}} |
| path: /tmp/kind_logs_* |
| retention-days: 7 |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| tests-helm-executor-upgrade: |
| timeout-minutes: 150 |
| name: Helm Chart Executor Upgrade - ${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, wait-for-prod-images] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| BACKEND: postgres |
| RUN_TESTS: "true" |
| RUNTIME: "kubernetes" |
| KUBERNETES_MODE: "image" |
| EXECUTOR: "KubernetesExecutor" |
| KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}" |
| HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}" |
| CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >- |
| ${{needs.build-info.outputs.min-max-python-versions-as-string}} |
| CURRENT_KUBERNETES_VERSIONS_AS_STRING: >- |
| ${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}} |
| if: > |
| needs.build-info.outputs.run-kubernetes-tests == 'true' && |
| needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze prod-image pull --run-in-parallel --tag-as-latest |
| env: |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Cache virtualenv for kubernetes testing" |
| uses: actions/cache@v3 |
| with: |
| path: ".build/.kubernetes_venv" |
| key: "kubernetes-${{ needs.build-info.outputs.default-python-version }}\ |
| -${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}} |
| -${{needs.build-info.outputs.min-max-python-versions-as-string}} |
| -${{ hashFiles('setup.py','setup.cfg') }}" |
| restore-keys: "kubernetes-${{ needs.build-info.outputs.default-python-version }}-\ |
| -${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}} \ |
| -${{needs.build-info.outputs.min-max-python-versions-as-string}}" |
| - name: "Cache bin folder with tools for kubernetes testing" |
| uses: actions/cache@v3 |
| with: |
| path: ".build/kubernetes-bin" |
| key: "kubernetes-binaries |
| -${{ needs.build-info.outputs.default-kind-version }}\ |
| -${{ needs.build-info.outputs.default-helm-version }}" |
| restore-keys: "kubernetes-binaries" |
| - name: "Kubernetes Helm Chart Executor Upgrade Tests" |
| run: ./scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh |
| env: |
| PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" |
| - name: "Upload KinD logs" |
| uses: actions/upload-artifact@v3 |
| if: failure() || cancelled() |
| with: |
| name: kind-logs-KubernetesExecutor |
| path: /tmp/kind_logs_* |
| retention-days: 7 |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| constraints: |
| permissions: |
| contents: write |
| timeout-minutes: 40 |
| name: "Constraints" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: |
| - build-info |
| - docs |
| - wait-for-ci-images |
| - wait-for-prod-images |
| - static-checks |
| - tests-sqlite |
| - tests-mysql |
| - tests-mssql |
| - tests-postgres |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze ci-image pull --run-in-parallel --tag-as-latest |
| env: |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Generate constraints" |
| run: | |
| breeze release-management generate-constraints --run-in-parallel \ |
| --airflow-constraints-mode constraints-source-providers |
| breeze release-management generate-constraints \ |
| --run-in-parallel --airflow-constraints-mode constraints-no-providers |
| breeze release-management generate-constraints \ |
| --run-in-parallel --airflow-constraints-mode constraints |
| env: |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| - name: "Set constraints branch name" |
| id: constraints-branch |
| run: ./scripts/ci/constraints/ci_branch_constraints.sh |
| if: needs.build-info.outputs.merge-run == 'true' |
| - name: Checkout ${{ steps.constraints-branch.outputs.branch }} |
| uses: actions/checkout@v3 |
| if: needs.build-info.outputs.merge-run == 'true' |
| with: |
| path: "repo" |
| ref: ${{ steps.constraints-branch.outputs.branch }} |
| persist-credentials: false |
| - name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}" |
| run: ./scripts/ci/constraints/ci_commit_constraints.sh |
| if: needs.build-info.outputs.merge-run == 'true' |
| - name: "Push changes" |
| uses: ./.github/actions/github-push-action |
| if: needs.build-info.outputs.merge-run == 'true' |
| with: |
| github_token: ${{ secrets.GITHUB_TOKEN }} |
| branch: ${{ steps.constraints-branch.outputs.branch }} |
| directory: "repo" |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build |
| # is executed as result of direct push to "main" or one of the "vX-Y-test" branches |
| # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry |
| # It will automatically check if a new python image was released and will pull the latest one if needed |
| push-buildx-cache-to-github-registry: |
| permissions: |
| packages: write |
| timeout-minutes: 50 |
| name: "Push Image Cache" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: |
| - build-info |
| - constraints |
| - docs |
| if: needs.build-info.outputs.merge-run == 'true' |
| strategy: |
| fail-fast: false |
| matrix: |
| platform: ["linux/amd64", "linux/arm64"] |
| env: |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: > |
| Pull CI image for PROD build |
| ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}" |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| # Always use default Python version of CI image for preparing packages |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Cleanup dist and context file" |
| run: rm -fv ./dist/* ./docker-context-files/* |
| - name: "Prepare airflow package for PROD build" |
| run: breeze release-management prepare-airflow-package --package-format wheel |
| env: |
| VERSION_SUFFIX_FOR_PYPI: "dev0" |
| - name: "Prepare providers packages for PROD build" |
| run: > |
| breeze release-management prepare-provider-packages |
| --package-list-file ./scripts/ci/installed_providers.txt |
| --package-format wheel |
| env: |
| VERSION_SUFFIX_FOR_PYPI: "dev0" |
| if: needs.build-info.outputs.default-branch == 'main' |
| - name: "Start ARM instance" |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh |
| if: matrix.platform == 'linux/arm64' |
| - name: "Push CI cache ${{ matrix.platform }}" |
| run: > |
| breeze ci-image build |
| --builder airflow_cache |
| --prepare-buildx-cache |
| --run-in-parallel |
| --force-build |
| --platform ${{ matrix.platform }} |
| - name: "Push CI latest image ${{ matrix.platform }}" |
| run: > |
| breeze ci-image build |
| --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }} |
| if: matrix.platform == 'linux/amd64' |
| - name: "Move dist packages to docker-context files" |
| run: mv -v ./dist/*.whl ./docker-context-files |
| - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}" |
| run: > |
| breeze prod-image build |
| --builder airflow_cache |
| --install-packages-from-context |
| --prepare-buildx-cache |
| --platform ${{ matrix.platform }} |
| if: needs.build-info.outputs.default-branch == 'main' |
| - name: "Push PROD latest image ${{ matrix.platform }}" |
| run: > |
| breeze prod-image build --tag-as-latest --install-packages-from-context |
| --push --run-in-parallel --platform ${{ matrix.platform }} |
| if: matrix.platform == 'linux/amd64' |
| - name: "Stop ARM instance" |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh |
| if: always() && matrix.platform == 'linux/arm64' |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| build-ci-arm-images: |
| timeout-minutes: 50 |
| name: > |
| Build CI ARM images |
| ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: |
| - build-info |
| - wait-for-ci-images |
| - wait-for-prod-images |
| - static-checks |
| - tests-sqlite |
| - tests-mysql |
| - tests-mssql |
| - tests-postgres |
| env: |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} |
| DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }} |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }} |
| if: > |
| needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' && |
| needs.build-info.outputs.in-workflow-build == 'true' && |
| needs.build-info.outputs.merge-run != 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - uses: actions/checkout@v2 |
| with: |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} |
| persist-credentials: false |
| submodules: recursive |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: "Start ARM instance" |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh |
| - name: > |
| Build CI ARM images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| run: > |
| breeze ci-image build |
| --run-in-parallel |
| --builder airflow_cache |
| --platform "linux/arm64" |
| env: |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| - name: "Stop ARM instance" |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh |
| if: always() |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
