| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| --- |
| name: "Build Images" |
| run-name: > |
| Build images for ${{ github.event.pull_request.title }} ${{ github.event.pull_request._links.html.href }} |
| on: # yamllint disable-line rule:truthy |
| pull_request_target: |
| permissions: |
| # all other permissions are set to none |
| contents: read |
| pull-requests: read |
| packages: read |
| env: |
| ANSWER: "yes" |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the |
| # Airflow one is going to be used |
| CONSTRAINTS_GITHUB_REPOSITORY: >- |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} |
| # This token is WRITE one - pull_request_target type of events always have the WRITE token |
| DB_RESET: "true" |
| GITHUB_REPOSITORY: ${{ github.repository }} |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| GITHUB_USERNAME: ${{ github.actor }} |
| IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" |
| INCLUDE_SUCCESS_OUTPUTS: "true" |
| USE_SUDO: "true" |
| VERBOSE: "true" |
| |
| concurrency: |
| group: build-${{ github.event.pull_request.number || github.ref }} |
| cancel-in-progress: true |
| |
| jobs: |
| build-info: |
| timeout-minutes: 10 |
| name: "Build Info" |
| # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners |
| runs-on: ["ubuntu-22.04"] |
| env: |
| TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} |
| outputs: |
| image-tag: ${{ github.event.pull_request.head.sha || github.sha }} |
| python-versions: ${{ steps.selective-checks.outputs.python-versions }} |
| python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} |
| default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} |
| upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} |
| ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} |
| prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} |
| docker-cache: ${{ steps.selective-checks.outputs.docker-cache }} |
| default-branch: ${{ steps.selective-checks.outputs.default-branch }} |
| constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} |
| runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }} |
| runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }} |
| runs-on-as-json-self-hosted: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted }} |
| is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }} |
| is-committer-build: ${{ steps.selective-checks.outputs.is-committer-build }} |
| is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} |
| is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} |
| is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} |
| is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} |
| is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} |
| chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} |
| target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha || |
| github.event.pull_request.head.sha || |
| github.sha |
| }}" |
| if: github.repository == 'apache/airflow' |
| steps: |
| - name: "Cleanup repo" |
| shell: bash |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: Discover PR merge commit |
| id: discover-pr-merge-commit |
| run: | |
| # Sometimes target-commit-sha cannot be |
| TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)" |
| if [[ ${TARGET_COMMIT_SHA} == "" ]]; then |
| # Sometimes retrieving the merge commit SHA from PR fails. We retry it once. Otherwise we |
| # fall-back to github.event.pull_request.head.sha |
| echo |
| echo "Could not retrieve merge commit SHA from PR, waiting for 3 seconds and retrying." |
| echo |
| sleep 3 |
| TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)" |
| if [[ ${TARGET_COMMIT_SHA} == "" ]]; then |
| echo |
| echo "Could not retrieve merge commit SHA from PR, falling back to PR head SHA." |
| echo |
| TARGET_COMMIT_SHA="${{ github.event.pull_request.head.sha }}" |
| fi |
| fi |
| echo "TARGET_COMMIT_SHA=${TARGET_COMMIT_SHA}" |
| echo "TARGET_COMMIT_SHA=${TARGET_COMMIT_SHA}" >> ${GITHUB_ENV} |
| echo "target-commit-sha=${TARGET_COMMIT_SHA}" >> ${GITHUB_OUTPUT} |
| if: github.event_name == 'pull_request_target' |
| # The labels in the event aren't updated when re-triggering the job, So lets hit the API to get |
| # up-to-date values |
| - name: Get latest PR labels |
| id: get-latest-pr-labels |
| run: | |
| echo -n "pull-request-labels=" >> ${GITHUB_OUTPUT} |
| gh api graphql --paginate -F node_id=${{github.event.pull_request.node_id}} -f query=' |
| query($node_id: ID!, $endCursor: String) { |
| node(id:$node_id) { |
| ... on PullRequest { |
| labels(first: 100, after: $endCursor) { |
| nodes { name } |
| pageInfo { hasNextPage endCursor } |
| } |
| } |
| } |
| }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' >> ${GITHUB_OUTPUT} |
| if: github.event_name == 'pull_request_target' |
| - uses: actions/checkout@v4 |
| with: |
| ref: ${{ env.TARGET_COMMIT_SHA }} |
| persist-credentials: false |
| fetch-depth: 2 |
| #################################################################################################### |
| # WE ONLY DO THAT CHECKOUT ABOVE TO RETRIEVE THE TARGET COMMIT AND IT'S PARENT. DO NOT RUN ANY CODE |
| # RIGHT AFTER THAT AS WE ARE GOING TO RESTORE THE TARGET BRANCH CODE IN THE NEXT STEP. |
| #################################################################################################### |
| - name: Checkout target branch to use ci/scripts and breeze from there. |
| uses: actions/checkout@v4 |
| with: |
| ref: ${{ github.base_ref }} |
| persist-credentials: false |
| #################################################################################################### |
| # HERE EVERYTHING IS PERFECTLY SAFE TO RUN. AT THIS POINT WE HAVE THE TARGET BRANCH CHECKED OUT |
| # AND WE CAN RUN ANY CODE FROM IT. WE CAN RUN BREEZE COMMANDS, WE CAN RUN SCRIPTS, WE CAN RUN |
| # COMPOSITE ACTIONS. WE CAN RUN ANYTHING THAT IS IN THE TARGET BRANCH AND THERE IS NO RISK THAT |
| # CODE WILL BE RUN FROM THE PR. |
| #################################################################################################### |
| - name: "Cleanup docker" |
| run: ./scripts/ci/cleanup_docker.sh |
| - name: "Setup python" |
| uses: actions/setup-python@v5 |
| with: |
| python-version: 3.8 |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| #################################################################################################### |
| # WE RUN SELECTIVE CHECKS HERE USING THE TARGET COMMIT AND ITS PARENT TO BE ABLE TO COMPARE THEM |
| # AND SEE WHAT HAS CHANGED IN THE PR. THE CODE IS STILL RUN FROM THE TARGET BRANCH, SO IT IS SAFE |
| # TO RUN IT, WE ONLY PASS TARGET_COMMIT_SHA SO THAT SELECTIVE CHECKS CAN SEE WHAT'S COMING IN THE PR |
| #################################################################################################### |
| - name: Selective checks |
| id: selective-checks |
| env: |
| PR_LABELS: "${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}" |
| COMMIT_REF: "${{ env.TARGET_COMMIT_SHA }}" |
| VERBOSE: "false" |
| AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}" |
| run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} |
| - name: env |
| run: printenv |
| env: |
| PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }} |
| GITHUB_CONTEXT: ${{ toJson(github) }} |
| |
| |
| build-ci-images: |
| name: Build CI images |
| permissions: |
| contents: read |
| packages: write |
| secrets: inherit |
| needs: [build-info] |
| uses: ./.github/workflows/ci-image-build.yml |
| # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow" |
| if: | |
| needs.build-info.outputs.ci-image-build == 'true' && |
| github.event.pull_request.head.repo.full_name != 'apache/airflow' |
| with: |
| runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} |
| runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} |
| do-build: ${{ needs.build-info.outputs.ci-image-build }} |
| target-commit-sha: ${{ needs.build-info.outputs.target-commit-sha }} |
| pull-request-target: "true" |
| is-committer-build: ${{ needs.build-info.outputs.is-committer-build }} |
| push-image: "true" |
| use-uv: "true" |
| image-tag: ${{ needs.build-info.outputs.image-tag }} |
| platform: "linux/amd64" |
| python-versions: ${{ needs.build-info.outputs.python-versions }} |
| branch: ${{ needs.build-info.outputs.default-branch }} |
| constraints-branch: ${{ needs.build-info.outputs.constraints-branch }} |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| docker-cache: ${{ needs.build-info.outputs.docker-cache }} |
| |
| generate-constraints: |
| name: "Generate constraints" |
| needs: [build-info, build-ci-images] |
| uses: ./.github/workflows/generate-constraints.yml |
| with: |
| runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} |
| python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} |
| # For regular PRs we do not need "no providers" constraints - they are only needed in canary builds |
| generate-no-providers-constraints: "false" |
| image-tag: ${{ needs.build-info.outputs.image-tag }} |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} |
| |
| build-prod-images: |
| name: Build PROD images |
| permissions: |
| contents: read |
| packages: write |
| secrets: inherit |
| needs: [build-info, generate-constraints] |
| uses: ./.github/workflows/prod-image-build.yml |
| # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow" |
| if: | |
| needs.build-info.outputs.prod-image-build == 'true' && |
| github.event.pull_request.head.repo.full_name != 'apache/airflow' |
| with: |
| runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} |
| build-type: "Regular" |
| do-build: ${{ needs.build-info.outputs.ci-image-build }} |
| upload-package-artifact: "true" |
| target-commit-sha: ${{ needs.build-info.outputs.target-commit-sha }} |
| pull-request-target: "true" |
| is-committer-build: ${{ needs.build-info.outputs.is-committer-build }} |
| push-image: "true" |
| use-uv: ${{ needs.build-info.outputs.default-branch == 'main' && 'true' || 'false' }} |
| image-tag: ${{ needs.build-info.outputs.image-tag }} |
| platform: "linux/amd64" |
| python-versions: ${{ needs.build-info.outputs.python-versions }} |
| default-python-version: ${{ needs.build-info.outputs.default-python-version }} |
| branch: ${{ needs.build-info.outputs.default-branch }} |
| constraints-branch: ${{ needs.build-info.outputs.constraints-branch }} |
| build-provider-packages: "true" |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} |
| docker-cache: ${{ needs.build-info.outputs.docker-cache }} |