| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| --- |
| name: "Build Images" |
| on: # yamllint disable-line rule:truthy |
| pull_request_target: |
| permissions: |
| # all other permissions are set to none |
| contents: read |
| pull-requests: read |
| env: |
| MOUNT_SELECTED_LOCAL_SOURCES: "false" |
| ANSWER: "yes" |
| CHECK_IMAGE_FOR_REBUILD: "true" |
| SKIP_CHECK_REMOTE_IMAGE: "true" |
| DB_RESET: "true" |
| VERBOSE: "true" |
| GITHUB_REPOSITORY: ${{ github.repository }} |
| GITHUB_USERNAME: ${{ github.actor }} |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the |
| # Airflow one is going to be used |
| CONSTRAINTS_GITHUB_REPOSITORY: >- |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} |
| # This token is WRITE one - pull_request_target type of events always have the WRITE token |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| IMAGE_TAG_FOR_THE_BUILD: "${{ github.event.pull_request.head.sha || github.sha }}" |
| USE_SUDO: "true" |
| |
| concurrency: |
| group: build-${{ github.event.pull_request.number || github.ref }} |
| cancel-in-progress: true |
| |
| jobs: |
| build-info: |
| timeout-minutes: 10 |
| name: "Build Info" |
| runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }} |
| env: |
| TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} |
| outputs: |
| runs-on: ${{ github.repository == 'apache/airflow' && '["self-hosted"]' || '["ubuntu-20.04"]' }} |
| python-versions: "${{ steps.selective-checks.python-versions }}" |
| upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} |
| all-python-versions-list-as-string: >- |
| ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} |
| default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} |
| image-build: ${{ steps.selective-checks.outputs.image-build }} |
| cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} |
| default-branch: ${{ steps.selective-checks.outputs.default-branch }} |
| default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} |
| debian-version: ${{ steps.selective-checks.outputs.debian-version }} |
| target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha || |
| github.event.pull_request.head.sha || |
| github.sha |
| }}" |
| steps: |
| - name: Discover PR merge commit |
| id: discover-pr-merge-commit |
| run: | |
| TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)" |
| echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> $GITHUB_ENV |
| echo "::set-output name=target-commit-sha::${TARGET_COMMIT_SHA}" |
| if: github.event_name == 'pull_request_target' |
| # The labels in the event aren't updated when re-triggering the job, So lets hit the API to get |
| # up-to-date values |
| - name: Get latest PR labels |
| id: get-latest-pr-labels |
| run: | |
| echo -n "::set-output name=pull-request-labels::" |
| gh api graphql --paginate -F node_id=${{github.event.pull_request.node_id}} -f query=' |
| query($node_id: ID!, $endCursor: String) { |
| node(id:$node_id) { |
| ... on PullRequest { |
| labels(first: 100, after: $endCursor) { |
| nodes { name } |
| pageInfo { hasNextPage endCursor } |
| } |
| } |
| } |
| }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' |
| if: github.event_name == 'pull_request_target' |
| # Retrieve it to be able to determine which files has changed in the incoming commit of the PR |
| # we checkout the target commit and it's parent to be able to compare them |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - uses: actions/checkout@v2 |
| with: |
| ref: ${{ env.TARGET_COMMIT_SHA }} |
| persist-credentials: false |
| fetch-depth: 2 |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| # We do not have output from selective checks yet, so we need to hardcode python |
| python-version: 3.7 |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - name: "Retrieve defaults from branch_defaults.py" |
| # We cannot "execute" the branch_defaults.py python code here because that would be |
| # a security problem (we cannot run any code that comes from the sources coming from the PR. |
| # Therefore, we extract the branches via embedded Python code |
| # we need to do it before next step replaces checked-out breeze and scripts code coming from |
| # the PR, because the PR defaults have to be retrieved here. |
| id: defaults |
| run: | |
| python - <<EOF >>$GITHUB_ENV |
| from pathlib import Path |
| import re |
| import sys |
| |
| DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text() |
| BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$' |
| CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$' |
| DEBIAN_VERSION_PATTERN = r'^DEBIAN_VERSION = "(.*)"$' |
| |
| branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) |
| constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) |
| debian_version = re.search(DEBIAN_VERSION_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) |
| |
| output = f""" |
| DEFAULT_BRANCH={branch} |
| DEFAULT_CONSTRAINTS_BRANCH={constraints_branch} |
| DEBIAN_VERSION={debian_version} |
| """.strip() |
| |
| print(output) |
| # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log |
| print(output, file=sys.stderr) |
| EOF |
| - name: Checkout main branch to use breeze from there. |
| uses: actions/checkout@v3 |
| with: |
| ref: "main" |
| persist-credentials: false |
| submodules: recursive |
| - run: ./scripts/ci/install_breeze.sh |
| - name: Selective checks |
| id: selective-checks |
| env: |
| PR_LABELS: "${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}" |
| COMMIT_REF: "${{ env.TARGET_COMMIT_SHA }}" |
| run: breeze ci selective-check |
| - name: env |
| run: printenv |
| env: |
| PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }} |
| GITHUB_CONTEXT: ${{ toJson(github) }} |
| |
| build-ci-images: |
| permissions: |
| packages: write |
| timeout-minutes: 80 |
| name: "Build CI images ${{ needs.build-info.outputs.all-python-versions-list-as-string }}" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info] |
| if: | |
| needs.build-info.outputs.image-build == 'true' && |
| github.event.pull_request.head.repo.full_name != 'apache/airflow' |
| env: |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} |
| DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }} |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }} |
| BACKEND: sqlite |
| outputs: ${{toJSON(needs.build-info.outputs) }} |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - uses: actions/checkout@v3 |
| with: |
| ref: ${{ needs.build-info.outputs.target-commit-sha }} |
| persist-credentials: false |
| submodules: recursive |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| - name: > |
| Checkout "main branch to 'main-airflow' folder |
| to use ci/scripts from there. |
| uses: actions/checkout@v3 |
| with: |
| path: "main-airflow" |
| ref: "main" |
| persist-credentials: false |
| submodules: recursive |
| - name: > |
| Override "scripts/ci" with the "main" branch |
| so that the PR does not override it |
| # We should not override those scripts which become part of the image as they will not be |
| # changed in the image built - we should only override those that are executed to build |
| # the image. |
| run: | |
| rm -rfv "scripts/ci" |
| rm -rfv "dev" |
| mv -v "main-airflow/scripts/ci" "scripts" |
| mv -v "main-airflow/dev" "." |
| - uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Cache pre-commit envs |
| uses: actions/cache@v3 |
| with: |
| path: ~/.cache/pre-commit |
| key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ |
| ${{ hashFiles('.pre-commit-config.yaml') }}" |
| restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} |
| - name: "Regenerate dependencies in case they was modified manually so that we can build an image" |
| run: > |
| breeze static-checks --type update-providers-dependencies --all-files |
| --show-diff-on-failure --color always || true |
| if: needs.build-info.outputs.default-branch == 'main' |
| - name: Compile www assets |
| run: breeze compile-www-assets |
| - name: >- |
| Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| run: breeze ci-image build --push --tag-as-latest --run-in-parallel |
| env: |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| - name: Push empty CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| if: failure() || cancelled() |
| run: breeze ci-image build --push --empty-image --run-in-parallel |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Candidates for pip resolver backtrack triggers" |
| if: failure() || cancelled() |
| run: > |
| breeze ci find-newer-dependencies --max-age 1 |
| --python "${{ needs.build-info.outputs.default-python-version }}" |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| build-prod-images: |
| permissions: |
| packages: write |
| timeout-minutes: 80 |
| name: "Build PROD images ${{ needs.build-info.outputs.all-python-versions-list-as-string }}" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, build-ci-images] |
| if: | |
| needs.build-info.outputs.image-build == 'true' && |
| github.event.pull_request.head.repo.full_name != 'apache/airflow' |
| env: |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} |
| DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }} |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }} |
| BACKEND: sqlite |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - uses: actions/checkout@v3 |
| with: |
| ref: ${{ needs.build-info.outputs.target-commit-sha }} |
| persist-credentials: false |
| submodules: recursive |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| - name: > |
| Checkout "main" branch to 'main-airflow' folder |
| to use ci/scripts from there. |
| uses: actions/checkout@v3 |
| with: |
| path: "main-airflow" |
| ref: "main" |
| persist-credentials: false |
| submodules: recursive |
| - name: > |
| Override "scripts/ci" with the "main" branch |
| so that the PR does not override it |
| # We should not override those scripts which become part of the image as they will not be |
| # changed in the image built - we should only override those that are executed to build |
| # the image. |
| run: | |
| rm -rfv "scripts/ci" |
| rm -rfv "dev" |
| mv -v "main-airflow/scripts/ci" "scripts" |
| mv -v "main-airflow/dev" "." |
| - uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| cache: 'pip' |
| cache-dependency-path: ./dev/breeze/setup* |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: Cache pre-commit envs |
| uses: actions/cache@v3 |
| with: |
| path: ~/.cache/pre-commit |
| key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ |
| ${{ hashFiles('.pre-commit-config.yaml') }}" |
| restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} |
| - name: "Regenerate dependencies in case they was modified manually so that we can build an image" |
| run: > |
| breeze static-checks --type update-providers-dependencies --all-files |
| --show-diff-on-failure --color always || true |
| - name: > |
| Pull CI image for PROD build: |
| ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| run: breeze ci-image pull --tag-as-latest |
| env: |
| # Always use default Python version of CI image for preparing packages |
| PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Cleanup dist and context file" |
| run: rm -fv ./dist/* ./docker-context-files/* |
| - name: "Prepare providers packages" |
| run: > |
| breeze release-management prepare-provider-packages |
| --package-list-file ./scripts/ci/installed_providers.txt |
| --package-format wheel |
| --version-suffix-for-pypi dev0 |
| if: needs.build-info.outputs.default-branch == 'main' |
| - name: "Prepare airflow package" |
| run: > |
| breeze release-management prepare-airflow-package |
| --package-format wheel --version-suffix-for-pypi dev0 |
| - name: "Move dist packages to docker-context files" |
| run: mv -v ./dist/*.whl ./docker-context-files |
| - name: >- |
| Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| run: > |
| breeze prod-image build |
| --run-in-parallel |
| --tag-as-latest |
| --push |
| --install-packages-from-context |
| env: |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| - name: Push empty PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| if: failure() || cancelled() |
| run: breeze prod-image build --cleanup-context --push --empty-image --run-in-parallel |
| env: |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
| |
| |
| build-ci-images-arm: |
| timeout-minutes: 50 |
| name: "Build ARM CI images ${{ needs.build-info.outputs.all-python-versions-list-as-string }}" |
| runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} |
| needs: [build-info, build-prod-images] |
| if: | |
| needs.build-info.outputs.image-build == 'true' && |
| needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' && |
| github.event.pull_request.head.repo.full_name != 'apache/airflow' |
| env: |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} |
| DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }} |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }} |
| BACKEND: sqlite |
| outputs: ${{toJSON(needs.build-info.outputs) }} |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - uses: actions/checkout@v2 |
| with: |
| ref: ${{ needs.build-info.outputs.target-commit-sha }} |
| persist-credentials: false |
| submodules: recursive |
| - name: > |
| Checkout "main" branch to 'main-airflow' folder |
| to use ci/scripts from there. |
| uses: actions/checkout@v2 |
| with: |
| path: "main-airflow" |
| ref: "main" |
| persist-credentials: false |
| submodules: recursive |
| - name: > |
| Override "scripts/ci" with the "main" branch |
| so that the PR does not override it |
| # We should not override those scripts which become part of the image as they will not be |
| # changed in the image built - we should only override those that are executed to build |
| # the image. |
| run: | |
| rm -rfv "scripts/ci" |
| rm -rfv "dev" |
| mv -v "main-airflow/scripts/ci" "scripts" |
| mv -v "main-airflow/dev" "." |
| - name: "Setup python" |
| uses: actions/setup-python@v2 |
| with: |
| python-version: ${{ needs.build-info.outputs.default-python-version }} |
| - run: ./scripts/ci/install_breeze.sh |
| - name: "Free space" |
| run: breeze ci free-space |
| - name: "Start ARM instance" |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh |
| - name: > |
| Build ARM CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| run: > |
| breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64" |
| env: |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} |
| - name: "Stop ARM instance" |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh |
| if: always() |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
