| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| --- |
| name: Tests |
| on: # yamllint disable-line rule:truthy |
| schedule: |
| - cron: '28 0 * * *' |
| push: |
| branches: ['main', 'v[0-9]+-[0-9]+-test'] |
| pull_request: |
| branches: ['main', 'v[0-9]+-[0-9]+-test', 'v[0-9]+-[0-9]+-stable'] |
| permissions: |
| # All other permissions are set to none |
| contents: read |
| # Technically read access while waiting for images should be more than enough. However, |
| # there is a bug in GitHub Actions/Packages and in case private repositories are used, you get a permission |
| # denied error when attempting to just pull private image, changing the token permission to write solves the |
| # issue. This is not dangerous, because if it is for "apache/airflow", only maintainers can push ci.yml |
| # changes. If it is for a fork, then the token is read-only anyway. |
| packages: write |
| env: |
| ANSWER: "yes" |
| DB_RESET: "true" |
| VERBOSE: "true" |
| GITHUB_REPOSITORY: ${{ github.repository }} |
| GITHUB_USERNAME: ${{ github.actor }} |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the |
| # Airflow one is going to be used |
| CONSTRAINTS_GITHUB_REPOSITORY: >- |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} |
| # In builds from forks, this token is read-only. For scheduler/direct push it is WRITE one |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| ENABLE_TEST_COVERAGE: "${{ github.event_name == 'push' }}" |
| IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" |
| USE_SUDO: "true" |
| INCLUDE_SUCCESS_OUTPUTS: "true" |
| AIRFLOW_ENABLE_AIP_44: "true" |
| |
| concurrency: |
| group: ci-${{ github.event.pull_request.number || github.ref }} |
| cancel-in-progress: true |
| |
| jobs: |
| |
| build-info: |
| name: "Build info" |
| # The runs-on cannot refer to env. or secrets. context, so we have no |
| # option but to specify a hard-coded list here. This is "safe", as the list |
| # is checked again by the runner using it's own list, so a PR author cannot |
| # change this and get access to our self-hosted runners |
| # |
| # This list is kept up-to-date from the list of authors found in the |
| # 'airflow-ci-infra' by the 'sync_authors' Github workflow. It uses a regexp |
| # to find the list of authors and replace them, so any changes to the |
| # formatting of the contains(fromJSON()) structure below will need to be |
| # reflected in that workflow too. |
| runs-on: "ubuntu-22.04" |
| env: |
| GITHUB_CONTEXT: ${{ toJson(github) }} |
| outputs: |
| cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} |
| affected-providers-list-as-string: >- |
| ${{ steps.selective-checks.outputs.affected-providers-list-as-string }} |
| upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} |
| python-versions: ${{ steps.selective-checks.outputs.python-versions }} |
| python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} |
| all-python-versions-list-as-string: >- |
| ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} |
| default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} |
| kubernetes-versions-list-as-string: >- |
| ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} |
| kubernetes-combos-list-as-string: >- |
| ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} |
| default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} |
| postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} |
| default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} |
| mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} |
| mssql-versions: ${{ steps.selective-checks.outputs.mssql-versions }} |
| default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} |
| default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} |
| default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} |
| full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} |
| parallel-test-types-list-as-string: >- |
| ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }} |
| mssql-parallelism: ${{ steps.selective-checks.outputs.mssql-parallelism }} |
| postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} |
| mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} |
| mssql-exclude: ${{ steps.selective-checks.outputs.mssql-exclude }} |
| sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} |
| skip-provider-tests: ${{ steps.selective-checks.outputs.skip-provider-tests }} |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} |
| run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} |
| run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} |
| basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} |
| image-build: ${{ steps.selective-checks.outputs.image-build }} |
| docs-build: ${{ steps.selective-checks.outputs.docs-build }} |
| needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} |
| needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} |
| needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} |
| default-branch: ${{ steps.selective-checks.outputs.default-branch }} |
| default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} |
| docs-filter-list-as-string: ${{ steps.selective-checks.outputs.docs-filter-list-as-string }} |
| skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} |
| helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} |
| debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} |
| runs-on: ${{ steps.selective-checks.outputs.runs-on }} |
| source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} |
| pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} |
| in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }} |
| build-job-description: ${{ steps.source-run-info.outputs.build-job-description }} |
| canary-run: ${{ steps.source-run-info.outputs.canary-run }} |
| run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: Fetch incoming commit ${{ github.sha }} with its parent |
| uses: actions/checkout@v3 |
| with: |
| ref: ${{ github.sha }} |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| - name: "Retrieve defaults from branch_defaults.py" |
| id: defaults |
| # We could retrieve it differently here - by just importing the variables and |
| # printing them from python code, however we want to have the same code as used in |
| # the build-images.yml (there we cannot import python code coming from the PR - we need to |
| # treat the python code as text and extract the variables from there. |
| run: | |
| python - <<EOF >> ${GITHUB_ENV} |
| from pathlib import Path |
| import re |
| import sys |
| |
| DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text() |
| BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$' |
| CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$' |
| |
| branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) |
| constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) |
| |
| output = f""" |
| DEFAULT_BRANCH={branch} |
| DEFAULT_CONSTRAINTS_BRANCH={constraints_branch} |
| """.strip() |
| |
| print(output) |
| # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log |
| print(output, file=sys.stderr) |
| EOF |
| - name: "Get information about the Workflow" |
| id: source-run-info |
| run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} |
| - name: Selective checks |
| id: selective-checks |
| env: |
| PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" |
| COMMIT_REF: "${{ github.sha }}" |
| VERBOSE: "false" |
| run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} |
| - name: env |
| run: printenv |
| env: |
| PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} |
| GITHUB_CONTEXT: ${{ toJson(github) }} |
| |
| # Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the |
| # tests to complete - it is run very early in the build process for "main" merges in order to refresh |
| # cache using the current constraints. This will speed up cache refresh in cases when setup.py |
| # changes or in case of Dockerfile changes. Failure in this step is not a problem (at most it will |
| # delay cache refresh. It does not attempt to upgrade to newer dependencies. |
| # We only push CI cache as PROD cache usually does not gain as much from fresh cache because |
| # it uses prepared airflow and provider packages that invalidate the cache anyway most of the time |
| push-early-buildx-cache-to-github-registry: |
| timeout-minutes: 50 |
| name: "Push Early Image Cache" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: |
| - build-info |
| strategy: |
| fail-fast: false |
| matrix: |
| platform: ["linux/amd64", "linux/arm64"] |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| UPGRADE_TO_NEWER_DEPENDENCIES: false |
| continue-on-error: true |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' |
| - name: "Start ARM instance" |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh |
| if: > |
| matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true' |
| && needs.build-info.outputs.default-branch == 'main' |
| - name: "Push CI cache ${{ matrix.platform }}" |
| run: > |
| breeze ci-image build |
| --builder airflow_cache |
| --prepare-buildx-cache |
| --run-in-parallel |
| --force-build |
| --platform ${{ matrix.platform }} |
| env: |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' |
| - name: "Push CI latest image ${{ matrix.platform }}" |
| run: > |
| breeze ci-image build |
| --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }} |
| env: |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} |
| # We only push "amd" image as it is really only needed for any kind of automated builds in CI |
| # and currently there is not an easy way to make multi-platform image from two separate builds |
| if: > |
| matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true' |
| && needs.build-info.outputs.default-branch == 'main' |
| - name: "Stop ARM instance" |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh |
| if: > |
| always() && matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true' |
| && needs.build-info.outputs.default-branch == 'main' |
| - name: "Clean docker cache for ${{ matrix.platform }}" |
| run: docker system prune --all --force |
| if: > |
| matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true' |
| && needs.build-info.outputs.default-branch == 'main' |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: > |
| always() && needs.build-info.outputs.canary-run == 'true' |
| && needs.build-info.outputs.default-branch == 'main' |
| # Check that after earlier cache push, breeze command will build quickly |
| check-that-image-builds-quickly: |
| timeout-minutes: 5 |
| name: "Check that image builds quickly" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: |
| - build-info |
| - push-early-buildx-cache-to-github-registry |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| UPGRADE_TO_NEWER_DEPENDENCIES: false |
| PLATFORM: "linux/amd64" |
| if: needs.build-info.outputs.canary-run == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| - name: "Check that image builds quickly" |
| run: breeze shell --max-time 120 |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| build-ci-images: |
| timeout-minutes: 80 |
| name: >- |
| ${{needs.build-info.outputs.build-job-description}} CI images |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info] |
| env: |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| # Force more parallelism for build even on public images |
| PARALLELISM: 6 |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - uses: actions/checkout@v3 |
| with: |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} |
| persist-credentials: false |
| submodules: recursive |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: > |
| Build CI Images |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/build-ci-images |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| env: |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} |
| |
| build-prod-images: |
| timeout-minutes: 80 |
| name: > |
| ${{needs.build-info.outputs.build-job-description}} PROD images |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, build-ci-images] |
| env: |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| BACKEND: sqlite |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| VERSION_SUFFIX_FOR_PYPI: "dev0" |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} |
| # Force more parallelism for build even on public images |
| PARALLELISM: 6 |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - uses: actions/checkout@v3 |
| with: |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} |
| persist-credentials: false |
| submodules: recursive |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| - name: > |
| Build PROD Images |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/build-prod-images |
| if: needs.build-info.outputs.in-workflow-build == 'true' |
| with: |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} |
| env: |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} |
| |
| run-breeze-tests: |
| timeout-minutes: 10 |
| name: Breeze unit tests |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info] |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - uses: actions/setup-python@v4 |
| with: |
| python-version: "${{needs.build-info.outputs.default-python-version}}" |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: python -m pip install --editable ./dev/breeze/ |
| - run: breeze setup version |
| - run: python -m pytest ./dev/breeze/ -n auto --color=yes |
| - run: breeze setup check-all-params-in-groups |
| |
| tests-www: |
| timeout-minutes: 10 |
| name: React WWW tests |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info] |
| if: needs.build-info.outputs.run-www-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup node" |
| uses: actions/setup-node@v3 |
| with: |
| node-version: 14 |
| - name: "Cache eslint" |
| uses: actions/cache@v3 |
| with: |
| path: 'airflow/www/node_modules' |
| key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }} |
| - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive |
| - run: yarn --cwd airflow/www/ run test |
| env: |
| FORCE_COLOR: 2 |
| |
| |
| test-openapi-client-generation: |
| timeout-minutes: 10 |
| name: "Test OpenAPI client generation" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info] |
| if: needs.build-info.outputs.needs-api-codegen == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Generate client codegen diff" |
| run: ./scripts/ci/openapi/client_codegen_diff.sh |
| |
| test-examples-of-prod-image-building: |
| timeout-minutes: 60 |
| name: "Test examples of production image building" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info] |
| if: needs.build-info.outputs.image-build == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: "${{needs.build-info.outputs.default-python-version}}" |
| cache: 'pip' |
| cache-dependency-path: ./dev/requirements.txt |
| - name: "Test examples of PROD image building" |
| run: > |
| cd ./docker_tests && |
| python -m pip install -r requirements.txt && |
| python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes |
| |
| test-git-clone-on-windows: |
| timeout-minutes: 5 |
| name: "Test git clone on Windows" |
| runs-on: windows-latest |
| needs: [build-info] |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| fetch-depth: 2 |
| persist-credentials: false |
| if: needs.build-info.outputs.runs-on != 'self-hosted' |
| |
| wait-for-ci-images: |
| timeout-minutes: 120 |
| name: "Wait for CI images" |
| runs-on: "ubuntu-22.04" |
| needs: [build-info, build-ci-images] |
| if: needs.build-info.outputs.image-build == 'true' |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| BACKEND: sqlite |
| # Force more parallelism for pull even on public images |
| PARALLELISM: 6 |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} |
| id: wait-for-images |
| run: breeze ci-image pull --run-in-parallel --verify --wait-for-image --tag-as-latest |
| env: |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| static-checks: |
| timeout-minutes: 30 |
| name: "Static checks" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" |
| if: needs.build-info.outputs.basic-checks-only == 'false' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: > |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/prepare_breeze_and_image |
| id: breeze |
| - name: Cache pre-commit envs |
| uses: actions/cache@v3 |
| with: |
| path: ~/.cache/pre-commit |
| # yamllint disable-line rule:line-length |
| key: "pre-commit-full-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" |
| restore-keys: | |
| pre-commit-full-${{steps.breeze.outputs.host-python-version}}- |
| - name: "Static checks" |
| run: breeze static-checks --all-files --show-diff-on-failure --color always --initialize-environment |
| env: |
| VERBOSE: "false" |
| SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} |
| COLUMNS: "250" |
| SKIP_GROUP_OUTPUT: "true" |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| RUFF_FORMAT: "github" |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| # Those checks are run if no image needs to be built for checks. This is for simple changes that |
| # Do not touch any of the python code or any of the important files that might require building |
| # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host |
| static-checks-basic-checks-only: |
| timeout-minutes: 30 |
| name: "Static checks: basic checks only" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info] |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| if: needs.build-info.outputs.basic-checks-only == 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: "${{needs.build-info.outputs.default-python-version}}" |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| id: breeze |
| - name: Cache pre-commit envs |
| uses: actions/cache@v3 |
| with: |
| path: ~/.cache/pre-commit |
| # yamllint disable-line rule:line-length |
| key: "pre-commit-basic-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" |
| restore-keys: "\ |
| pre-commit-full-${{steps.breeze.outputs.host-python-version}}-\ |
| ${{ hashFiles('.pre-commit-config.yaml') }}\n |
| pre-commit-basic-${{steps.breeze.outputs.host-python-version}}-\n |
| pre-commit-full-${{steps.breeze.outputs.host-python-version}}-" |
| - name: Fetch incoming commit ${{ github.sha }} with its parent |
| uses: actions/checkout@v3 |
| with: |
| ref: ${{ github.sha }} |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: "Static checks: basic checks only" |
| run: > |
| breeze static-checks --show-diff-on-failure --color always --initialize-environment |
| --commit-ref "${{ github.sha }}" |
| env: |
| VERBOSE: "false" |
| SKIP_IMAGE_PRE_COMMITS: "true" |
| SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} |
| COLUMNS: "250" |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| docs: |
| timeout-minutes: 90 |
| name: "Build docs" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| if: needs.build-info.outputs.docs-build == 'true' |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: > |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/prepare_breeze_and_image |
| - uses: actions/cache@v3 |
| id: cache-doc-inventories |
| with: |
| path: ./docs/_inventory_cache/ |
| key: docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }} |
| restore-keys: | |
| docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }} |
| docs-inventory- |
| - name: "Build docs" |
| run: breeze build-docs ${{ needs.build-info.outputs.docs-filter-list-as-string }} |
| - name: Configure AWS credentials |
| uses: ./.github/actions/configure-aws-credentials |
| if: > |
| github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && |
| github.event_name == 'push' |
| with: |
| aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} |
| aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} |
| aws-region: eu-central-1 |
| - name: "Upload documentation to AWS S3" |
| if: > |
| github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && |
| github.event_name == 'push' |
| run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| prepare-test-provider-packages-wheel: |
| timeout-minutes: 80 |
| name: "Provider packages wheel (verify)" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| if: needs.build-info.outputs.skip-provider-tests != 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: > |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: "Cleanup dist files" |
| run: rm -fv ./dist/* |
| - name: "Prepare provider documentation" |
| run: > |
| breeze release-management prepare-provider-documentation |
| ${{ needs.build-info.outputs.affected-providers-list-as-string }} |
| - name: "Prepare provider packages: wheel" |
| run: > |
| breeze release-management prepare-provider-packages --version-suffix-for-pypi dev0 |
| --package-format wheel ${{ needs.build-info.outputs.affected-providers-list-as-string }} |
| - name: "Prepare airflow package: wheel" |
| run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0 |
| - name: "Verify wheel packages with twine" |
| run: pipx install twine --force && twine check dist/*.whl |
| - name: "Test providers issue generation automatically" |
| run: > |
| breeze release-management generate-issue-content-providers |
| --only-available-in-dist --disable-progress |
| - name: "Install and verify all provider packages and airflow via wheel files" |
| run: > |
| breeze release-management verify-provider-packages --use-packages-from-dist |
| --package-format wheel --use-airflow-version wheel |
| if: needs.build-info.outputs.affected-providers-list-as-string == '' |
| env: |
| SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" |
| - name: "Install affected provider packages and airflow via wheel files" |
| run: > |
| breeze release-management install-provider-packages |
| --package-format wheel --use-airflow-version wheel --run-in-parallel |
| if: needs.build-info.outputs.affected-providers-list-as-string != '' |
| env: |
| SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| provider-airflow-compatibility-check: |
| timeout-minutes: 80 |
| name: "Providers Airflow 2.4 compatibility check" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| if: needs.build-info.outputs.skip-provider-tests != 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: > |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: "Cleanup dist files" |
| run: rm -fv ./dist/* |
| - name: "Prepare provider packages: wheel" |
| run: > |
| breeze release-management prepare-provider-packages --version-suffix-for-pypi dev0 |
| --package-format wheel ${{ needs.build-info.outputs.affected-providers-list-as-string }} |
| - name: "Fix incompatible 2.4 provider packages" |
| run: | |
| # This step should remove the provider packages that are not compatible with 2.4 |
| # or replace them with 2.4 compatible versions. Sometimes we have good reasons to bump |
| # the min airflow versions for some providers and then we need to add exclusions here. |
| # |
| # The Removal can be done with: |
| # |
| # rm -vf dist/apache_airflow_providers_<PROVIDER>*.whl |
| # |
| # Then it can be followed by downloading a compatible version from PyPI in case other |
| # providers depend on it and fail with import errors (you need to download compatible version): |
| # |
| # pip download --no-deps --dest dist apache-airflow-providers-<PROVIDER>==3.1.0 |
| # |
| rm -vf dist/apache_airflow_providers_openlineage*.whl |
| - name: "Get all provider extras as AIRFLOW_EXTRAS env variable" |
| # Extras might be different on S3 so rather than relying on "all" we should get the list of |
| # packages to be installed from the current provider_dependencies.json file |
| run: | |
| python -c 'from pathlib import Path; import json |
| providers = json.loads(Path("generated/provider_dependencies.json").read_text()) |
| provider_keys = ",".join(providers.keys()) |
| print("AIRFLOW_EXTRAS={}".format(provider_keys))' >> $GITHUB_ENV |
| - name: "Install and verify all provider packages and airflow on Airflow 2.4 files" |
| run: > |
| breeze release-management verify-provider-packages --use-airflow-version 2.4.0 |
| --use-packages-from-dist --airflow-constraints-reference constraints-2.4.0 |
| if: needs.build-info.outputs.affected-providers-list-as-string == '' |
| - name: "Install affected provider packages and airflow on Airflow 2.4 files" |
| run: > |
| breeze release-management install-provider-packages --use-airflow-version 2.4.0 |
| --airflow-constraints-reference constraints-2.4.0 --run-in-parallel |
| # Make sure to skip the run if the only provider to be installed has been removed |
| # in the previous step |
| if: > |
| needs.build-info.outputs.affected-providers-list-as-string != '' && |
| needs.build-info.outputs.affected-providers-list-as-string != 'openlineage' |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| prepare-install-provider-packages-sdist: |
| timeout-minutes: 80 |
| name: "Provider packages sdist (install)" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| USE_AIRFLOW_VERSION: "sdist" |
| if: > |
| needs.build-info.outputs.canary-run == 'true' && |
| needs.build-info.outputs.skip-provider-tests != 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: > |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: "Cleanup dist files" |
| run: rm -fv ./dist/* |
| - name: "Prepare provider packages: sdist" |
| run: > |
| breeze release-management prepare-provider-packages |
| --version-suffix-for-pypi dev0 --package-format sdist |
| ${{ needs.build-info.outputs.affected-providers-list-as-string }} |
| - name: "Prepare airflow package: sdist" |
| run: > |
| breeze release-management prepare-airflow-package |
| --version-suffix-for-pypi dev0 --package-format sdist |
| - name: "Verify sdist packages with twine" |
| run: pipx install twine --force && twine check dist/*.tar.gz |
| - name: "Install all provider packages and airflow via sdist files" |
| run: > |
| breeze release-management install-provider-packages |
| --package-format sdist --use-airflow-version sdist --run-in-parallel |
| if: needs.build-info.outputs.affected-providers-list-as-string == '' |
| - name: "Install affected provider packages and airflow via sdist files" |
| run: > |
| breeze release-management install-provider-packages |
| --package-format sdist --use-airflow-version sdist --run-in-parallel |
| if: needs.build-info.outputs.affected-providers-list-as-string != '' |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| |
| test-airflow-release-commands: |
| timeout-minutes: 80 |
| name: "Test Airflow release commands" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| if: needs.build-info.outputs.runs-on == 'self-hosted' |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: > |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: "Cleanup dist files" |
| run: rm -fv ./dist/* |
| - name: "Install required packages" |
| run: python -m pip install --editable ./dev/breeze/ |
| - name: "Check Airflow create minor branch command" |
| run: breeze release-management create-minor-branch --version-branch 2-4 -a y |
| - name: "Check Airflow RC process command" |
| run: | |
| breeze release-management start-rc-process --version 2.4.3rc1 --previous-version 2.4.2 -a y |
| - name: "Check Airflow release process command" |
| run: | |
| breeze release-management start-release --release-candidate 2.4.3rc1 --previous-release 2.4.2 -a y |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| tests-helm: |
| timeout-minutes: 80 |
| name: "Unit tests Helm: ${{matrix.helm-test-package}}" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| strategy: |
| fail-fast: false |
| matrix: |
| helm-test-package: ${{fromJson(needs.build-info.outputs.helm-test-packages)}} |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PARALLEL_TEST_TYPES: "Helm" |
| BACKEND: "" |
| DB_RESET: "false" |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| JOB_ID: "helm-tests" |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" |
| if: > |
| needs.build-info.outputs.needs-helm-tests == 'true' && |
| (github.repository == 'apache/airflow' || github.event_name != 'schedule') && |
| needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: > |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: "Helm Unit Tests: ${{ matrix.helm-test-package }}" |
| run: breeze testing helm-tests --helm-test-package "${{ matrix.helm-test-package }}" |
| - name: "Post Helm Tests" |
| uses: ./.github/actions/post_tests |
| |
| tests-postgres: |
| timeout-minutes: 130 |
| name: > |
| Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}: |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| strategy: |
| matrix: |
| python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" |
| postgres-version: "${{fromJson(needs.build-info.outputs.postgres-versions)}}" |
| exclude: "${{fromJson(needs.build-info.outputs.postgres-exclude)}}" |
| fail-fast: false |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" |
| BACKEND: "postgres" |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" |
| POSTGRES_VERSION: "${{matrix.postgres-version}}" |
| BACKEND_VERSION: "${{matrix.postgres-version}}" |
| JOB_ID: "postgres-${{matrix.postgres-version}}-${{matrix.python-version}}" |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| shell: bash |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: > |
| Migration Tests: |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| uses: ./.github/actions/migration_tests |
| - name: > |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| run: breeze testing tests --run-in-parallel |
| - name: "Tests ARM Pytest collection: ${{matrix.python-version}}" |
| run: breeze testing tests --run-in-parallel --collect-only --remove-arm-packages |
| if: matrix.postgres-version == needs.build-info.outputs.default-postgres-version |
| - name: > |
| Post Tests: |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| uses: ./.github/actions/post_tests |
| |
| tests-postgres-boto: |
| timeout-minutes: 130 |
| name: > |
| LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}}, |
| Py${{needs.build-info.outputs.default-python-version}}: |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" |
| BACKEND: "postgres" |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" |
| POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" |
| UPGRADE_BOTO: "true" |
| JOB_ID: > |
| postgres-boto-${{needs.build-info.outputs.default-python-version}}- |
| ${{needs.build-info.outputs.default-postgres-version}} |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" |
| if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| shell: bash |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: > |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: > |
| Tests: ${{needs.build-info.outputs.default-python-version}}: |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| run: breeze testing tests --run-in-parallel |
| - name: > |
| Post Tests: ${{needs.build-info.outputs.default-python-version}}: |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| uses: ./.github/actions/post_tests |
| |
| tests-postgres-in-progress-features-disabled: |
| timeout-minutes: 130 |
| name: > |
| InProgressDisabledPostgres${{needs.build-info.outputs.default-postgres-version}}, |
| Py${{needs.build-info.outputs.default-python-version}}: |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" |
| BACKEND: "postgres" |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" |
| POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" |
| AIRFLOW_ENABLE_AIP_44: "false" |
| JOB_ID: > |
| postgres-in-progress-disabled-${{needs.build-info.outputs.default-python-version}}- |
| ${{needs.build-info.outputs.default-postgres-version}} |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" |
| if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| shell: bash |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: > |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: > |
| Tests: ${{needs.build-info.outputs.default-python-version}}: |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| run: breeze testing tests --run-in-parallel |
| - name: > |
| Post Tests: ${{needs.build-info.outputs.default-python-version}}: |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| uses: ./.github/actions/post_tests |
| |
| tests-mysql: |
| timeout-minutes: 130 |
| name: > |
| MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| strategy: |
| matrix: |
| python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" |
| mysql-version: "${{fromJson(needs.build-info.outputs.mysql-versions)}}" |
| exclude: "${{fromJson(needs.build-info.outputs.mysql-exclude)}}" |
| fail-fast: false |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" |
| BACKEND: "mysql" |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" |
| MYSQL_VERSION: "${{matrix.mysql-version}}" |
| BACKEND_VERSION: "${{matrix.mysql-version}}" |
| JOB_ID: "mysql-${{matrix.mysql-version}}-${{matrix.python-version}}" |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| shell: bash |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: > |
| Migration Tests: |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| uses: ./.github/actions/migration_tests |
| - name: > |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| run: breeze testing tests --run-in-parallel |
| - name: > |
| Post Tests: |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| uses: ./.github/actions/post_tests |
| |
| tests-mssql: |
| timeout-minutes: 130 |
| name: > |
| MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| strategy: |
| matrix: |
| python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" |
| mssql-version: "${{fromJson(needs.build-info.outputs.mssql-versions)}}" |
| exclude: "${{fromJson(needs.build-info.outputs.mssql-exclude)}}" |
| fail-fast: false |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" |
| BACKEND: "mssql" |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" |
| MSSQL_VERSION: "${{matrix.mssql-version}}" |
| BACKEND_VERSION: "${{matrix.mssql-version}}" |
| JOB_ID: "mssql-${{matrix.mssql-version}}-${{matrix.python-version}}" |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" |
| # The below (and corresponding selective checks code) can be removed once |
| # https://github.com/apache/airflow/issues/31575 is fixed. |
| # This is a temporary workaround for flaky tests that occur in MSSQL tests for public runners |
| PARALLELISM: "${{needs.build-info.outputs.mssql-parallelism}}" |
| if: > |
| needs.build-info.outputs.run-tests == 'true' && |
| (needs.build-info.outputs.runs-on == 'self-hosted' || |
| needs.build-info.outputs.full-tests-needed == 'true') |
| steps: |
| - name: Cleanup repo |
| shell: bash |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: > |
| Migration Tests: |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| uses: ./.github/actions/migration_tests |
| - name: > |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| run: breeze testing tests --run-in-parallel |
| - name: > |
| Post Tests: |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| uses: ./.github/actions/post_tests |
| |
| tests-sqlite: |
| timeout-minutes: 130 |
| name: > |
| Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| strategy: |
| matrix: |
| python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} |
| exclude: ${{ fromJson(needs.build-info.outputs.sqlite-exclude) }} |
| fail-fast: false |
| if: needs.build-info.outputs.run-tests == 'true' |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" |
| BACKEND: "sqlite" |
| BACKEND_VERSION: "" |
| JOB_ID: "sqlite-${{matrix.python-version}}" |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" |
| steps: |
| - name: Cleanup repo |
| shell: bash |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: > |
| Migration Tests: |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| uses: ./.github/actions/migration_tests |
| - name: > |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| run: breeze testing tests --run-in-parallel |
| - name: > |
| Post Tests: |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| uses: ./.github/actions/post_tests |
| |
| tests-integration-postgres: |
| timeout-minutes: 130 |
| name: Integration Tests Postgres |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" |
| BACKEND: "postgres" |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" |
| JOB_ID: "integration" |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" |
| SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}" |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| shell: bash |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: "Integration Tests Postgres: cassandra" |
| run: | |
| breeze testing integration-tests --integration cassandra |
| breeze down |
| if: needs.build-info.outputs.runs-on != 'self-hosted' |
| - name: "Integration Tests Postgres: mongo" |
| run: | |
| breeze testing integration-tests --integration mongo |
| breeze down |
| if: needs.build-info.outputs.runs-on != 'self-hosted' |
| - name: "Integration Tests Postgres: pinot" |
| run: | |
| breeze testing integration-tests --integration pinot |
| breeze down |
| if: needs.build-info.outputs.runs-on != 'self-hosted' |
| - name: "Integration Tests Postgres: celery" |
| run: | |
| breeze testing integration-tests --integration celery |
| breeze down |
| if: needs.build-info.outputs.runs-on != 'self-hosted' |
| - name: "Integration Tests Postgres: trino, kerberos" |
| run: | |
| breeze testing integration-tests --integration trino --integration kerberos |
| breeze down |
| if: needs.build-info.outputs.runs-on != 'self-hosted' |
| - name: "Integration Tests Postgres: Kafka" |
| run: | |
| breeze testing integration-tests --integration kafka |
| breeze down |
| if: needs.build-info.outputs.runs-on != 'self-hosted' |
| - name: "Integration Tests Postgres: all-testable" |
| run: breeze testing integration-tests --integration all-testable |
| if: needs.build-info.outputs.runs-on == 'self-hosted' |
| - name: > |
| Post Tests: |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| uses: ./.github/actions/post_tests |
| |
| tests-integration-mysql: |
| timeout-minutes: 130 |
| name: Integration Tests MySQL |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" |
| BACKEND: "mysql" |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| MYSQL_VERSION: "${{needs.build-info.outputs.default-mysql-version}}" |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-mysql-version}}" |
| JOB_ID: "integration" |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" |
| SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}" |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| shell: bash |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| if: needs.build-info.outputs.runs-on == 'self-hosted' |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| if: needs.build-info.outputs.runs-on == 'self-hosted' |
| - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" |
| uses: ./.github/actions/prepare_breeze_and_image |
| if: needs.build-info.outputs.runs-on == 'self-hosted' |
| - name: "Integration Tests MySQL: all-testable" |
| run: breeze testing integration-tests --integration all-testable |
| if: needs.build-info.outputs.runs-on == 'self-hosted' |
| - name: > |
| Post Tests: |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} |
| uses: ./.github/actions/post_tests |
| if: needs.build-info.outputs.runs-on == 'self-hosted' |
| |
| |
| tests-quarantined: |
| timeout-minutes: 60 |
| name: "Quarantined tests" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| continue-on-error: true |
| needs: [build-info, wait-for-ci-images] |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| PARALLEL_TEST_TYPES: "Quarantined" |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" |
| BACKEND: "sqlite" |
| BACKEND_VERSION: "" |
| JOB_ID: "quarantined-${{needs.build-info.outputs.default-python-version}}" |
| COVERAGE: "${{needs.build-info.outputs.run-coverage}}" |
| if: needs.build-info.outputs.run-tests == 'true' |
| steps: |
| - name: Cleanup repo |
| shell: bash |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: > |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/prepare_breeze_and_image |
| - name: > |
| Tests: ${{needs.build-info.outputs.default-python-version}}:Quarantined |
| run: breeze testing tests --run-in-parallel || true |
| - name: > |
| Post Tests: ${{needs.build-info.outputs.default-python-version}}:Quarantined" |
| uses: ./.github/actions/post_tests |
| |
| summarize-warnings: |
| timeout-minutes: 15 |
| name: "Summarize warnings" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: |
| - build-info |
| - tests-postgres |
| - tests-sqlite |
| - tests-mysql |
| - tests-quarantined |
| - tests-integration-postgres |
| - tests-integration-mysql |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Download all artifacts from the current build" |
| uses: actions/download-artifact@v3 |
| with: |
| path: ./artifacts |
| - name: "Summarize all warnings" |
| run: | |
| ls -R ./artifacts/ |
| cat ./artifacts/test-warnings*/* | sort | uniq |
| echo |
| echo Total number of unique warnings $(cat ./artifacts/test-warnings*/* | sort | uniq | wc -l) |
| |
| |
| wait-for-prod-images: |
| timeout-minutes: 80 |
| name: "Wait for PROD images" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-ci-images, build-prod-images] |
| if: needs.build-info.outputs.image-build == 'true' |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| BACKEND: sqlite |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| # Force more parallelism for pull even on public images |
| PARALLELISM: 6 |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} |
| # We wait for the images to be available either from "build-images.yml' run as pull_request_target |
| # or from build-prod-images above. |
| # We are utilising single job to wait for all images because this job merely waits |
| # For the images to be available and test them. |
| run: breeze prod-image pull --verify --wait-for-image --run-in-parallel |
| env: |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| test-docker-compose-quick-start: |
| timeout-minutes: 60 |
| name: "Test docker-compose quick start" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-prod-images] |
| if: needs.build-info.outputs.image-build == 'true' |
| env: |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| fetch-depth: 2 |
| persist-credentials: false |
| - name: > |
| Prepare breeze & PROD image: |
| ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/prepare_breeze_and_image |
| with: |
| pull-image-type: 'PROD' |
| - name: "Test docker-compose quick start" |
| run: breeze testing docker-compose-tests |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| tests-kubernetes: |
| timeout-minutes: 240 |
| name: "Helm: ${{matrix.executor}} - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: [build-info, wait-for-prod-images] |
| strategy: |
| matrix: |
| executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] |
| fail-fast: false |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} |
| if: > |
| ( needs.build-info.outputs.run-kubernetes-tests == 'true' || |
| needs.build-info.outputs.needs-helm-tests == 'true' ) && |
| needs.build-info.outputs.default-branch == 'main' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| id: breeze |
| - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} |
| run: breeze prod-image pull --run-in-parallel --tag-as-latest |
| env: |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| - name: "Cache bin folder with tools for kubernetes testing" |
| uses: actions/cache@v3 |
| with: |
| path: ".build/.k8s-env" |
| key: "\ |
| k8s-env-${{steps.breeze.outputs.host-python-version}}-\ |
| ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','setup.cfg',\ |
| 'setup.py','pyproject.toml','generated/provider_dependencies.json') }}" |
| - name: Run complete K8S tests ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} |
| run: breeze k8s run-complete-tests --run-in-parallel --upgrade |
| env: |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}} |
| EXECUTOR: ${{matrix.executor}} |
| VERBOSE: false |
| - name: Upload KinD logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} |
| uses: actions/upload-artifact@v3 |
| if: failure() || cancelled() |
| with: |
| name: kind-logs-${{matrix.executor}} |
| path: /tmp/kind_logs_* |
| retention-days: 7 |
| - name: > |
| Upload test resource logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} |
| uses: actions/upload-artifact@v3 |
| if: failure() || cancelled() |
| with: |
| name: k8s-test-resources-${{matrix.executor}} |
| path: /tmp/k8s_test_resources_* |
| retention-days: 7 |
| - name: "Delete clusters just in case they are left" |
| run: breeze k8s delete-cluster --all |
| if: always() |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| constraints: |
| permissions: |
| contents: write |
| packages: write |
| timeout-minutes: 80 |
| name: "Constraints" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: |
| - build-info |
| - docs |
| - wait-for-ci-images |
| - wait-for-prod-images |
| - static-checks |
| - tests-sqlite |
| - tests-mysql |
| - tests-postgres |
| - tests-integration-postgres |
| - tests-integration-mysql |
| - push-early-buildx-cache-to-github-registry |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} |
| if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} |
| run: breeze ci-image pull --run-in-parallel --tag-as-latest |
| env: |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| - name: "Generate constraints" |
| run: | |
| breeze release-management generate-constraints --run-in-parallel \ |
| --airflow-constraints-mode constraints-source-providers |
| breeze release-management generate-constraints \ |
| --run-in-parallel --airflow-constraints-mode constraints-no-providers |
| breeze release-management generate-constraints \ |
| --run-in-parallel --airflow-constraints-mode constraints |
| env: |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| - name: "Set constraints branch name" |
| id: constraints-branch |
| run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} |
| if: needs.build-info.outputs.canary-run == 'true' |
| - name: Checkout ${{ steps.constraints-branch.outputs.branch }} |
| uses: actions/checkout@v3 |
| if: needs.build-info.outputs.canary-run == 'true' |
| with: |
| path: "repo" |
| ref: ${{ steps.constraints-branch.outputs.branch }} |
| persist-credentials: false |
| - name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}" |
| run: ./scripts/ci/constraints/ci_commit_constraints.sh |
| if: needs.build-info.outputs.canary-run == 'true' |
| - name: "Push changes" |
| uses: ./.github/actions/github-push-action |
| if: needs.build-info.outputs.canary-run == 'true' |
| with: |
| github_token: ${{ secrets.GITHUB_TOKEN }} |
| branch: ${{ steps.constraints-branch.outputs.branch }} |
| directory: "repo" |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build |
| # is executed as result of direct push to "main" or one of the "vX-Y-test" branches |
| # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry |
| # It will automatically check if a new python image was released and will pull the latest one if needed |
| push-buildx-cache-to-github-registry: |
| timeout-minutes: 50 |
| name: "Push Image Cache" |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: |
| - build-info |
| - constraints |
| - docs |
| if: needs.build-info.outputs.canary-run == 'true' |
| strategy: |
| fail-fast: false |
| matrix: |
| platform: ["linux/amd64", "linux/arm64"] |
| env: |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v3 |
| with: |
| persist-credentials: false |
| - name: > |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/prepare_breeze_and_image |
| env: |
| # Always use default Python version of CI image for preparing packages |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" |
| - name: "Cleanup dist and context file" |
| run: rm -fv ./dist/* ./docker-context-files/* |
| - name: "Prepare airflow package for PROD build" |
| run: breeze release-management prepare-airflow-package --package-format wheel |
| env: |
| VERSION_SUFFIX_FOR_PYPI: "dev0" |
| - name: "Prepare providers packages for PROD build" |
| run: > |
| breeze release-management prepare-provider-packages |
| --package-list-file ./scripts/ci/installed_providers.txt |
| --package-format wheel |
| env: |
| VERSION_SUFFIX_FOR_PYPI: "dev0" |
| if: needs.build-info.outputs.default-branch == 'main' |
| - name: "Start ARM instance" |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh |
| if: matrix.platform == 'linux/arm64' |
| - name: "Push CI cache ${{ matrix.platform }}" |
| run: > |
| breeze ci-image build |
| --builder airflow_cache |
| --prepare-buildx-cache |
| --run-in-parallel |
| --force-build |
| --platform ${{ matrix.platform }} |
| env: |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} |
| - name: "Push CI latest image ${{ matrix.platform }}" |
| run: > |
| breeze ci-image build |
| --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }} |
| env: |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} |
| if: matrix.platform == 'linux/amd64' |
| - name: "Move dist packages to docker-context files" |
| run: mv -v ./dist/*.whl ./docker-context-files |
| - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}" |
| run: > |
| breeze prod-image build |
| --builder airflow_cache |
| --install-packages-from-context |
| --run-in-parallel |
| --prepare-buildx-cache |
| --platform ${{ matrix.platform }} |
| - name: "Push PROD latest image ${{ matrix.platform }}" |
| run: > |
| breeze prod-image build --tag-as-latest --install-packages-from-context |
| --push --run-in-parallel --platform ${{ matrix.platform }} |
| env: |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} |
| if: matrix.platform == 'linux/amd64' |
| - name: "Stop ARM instance" |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh |
| if: always() && matrix.platform == 'linux/arm64' |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| build-ci-arm-images: |
| timeout-minutes: 50 |
| name: > |
| Build CI ARM images |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} |
| runs-on: "${{needs.build-info.outputs.runs-on}}" |
| needs: |
| - build-info |
| - wait-for-ci-images |
| - wait-for-prod-images |
| - static-checks |
| - tests-sqlite |
| - tests-mysql |
| - tests-postgres |
| - tests-integration-postgres |
| - tests-integration-mysql |
| env: |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" |
| # Force more parallelism for build even on small instances |
| PARALLELISM: 6 |
| if: > |
| needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' && |
| needs.build-info.outputs.in-workflow-build == 'true' && |
| needs.build-info.outputs.canary-run != 'true' |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - uses: actions/checkout@v3 |
| with: |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} |
| persist-credentials: false |
| submodules: recursive |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| - name: "Start ARM instance" |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh |
| - name: > |
| Build CI ARM images ${{ env.IMAGE_TAG }} |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} |
| run: > |
| breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64" |
| env: |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} |
| - name: "Stop ARM instance" |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh |
| if: always() |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
