| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| --- |
| name: "Build Images" |
| run-name: > |
| Build images for ${{ github.event.pull_request.title }} ${{ github.event.pull_request._links.html.href }} |
| on: # yamllint disable-line rule:truthy |
| pull_request_target: |
| permissions: |
| # all other permissions are set to none |
| contents: read |
| pull-requests: read |
| env: |
| ANSWER: "yes" |
| DB_RESET: "true" |
| VERBOSE: "true" |
| GITHUB_REPOSITORY: ${{ github.repository }} |
| GITHUB_USERNAME: ${{ github.actor }} |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the |
| # Airflow one is going to be used |
| CONSTRAINTS_GITHUB_REPOSITORY: >- |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} |
| # This token is WRITE one - pull_request_target type of events always have the WRITE token |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" |
| USE_SUDO: "true" |
| INCLUDE_SUCCESS_OUTPUTS: "true" |
| |
| concurrency: |
| group: build-${{ github.event.pull_request.number || github.ref }} |
| cancel-in-progress: true |
| |
| jobs: |
| build-info: |
| timeout-minutes: 10 |
| name: "Build Info" |
| runs-on: 'ubuntu-20.04' |
| env: |
| TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} |
| outputs: |
| python-versions: "${{ steps.selective-checks.python-versions }}" |
| upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} |
| all-python-versions-list-as-string: >- |
| ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} |
| default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} |
| image-build: ${{ steps.selective-checks.outputs.image-build }} |
| cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} |
| default-branch: ${{ steps.selective-checks.outputs.default-branch }} |
| default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} |
| runs-on: ${{ steps.selective-checks.outputs.runs-on }} |
| target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha || |
| github.event.pull_request.head.sha || |
| github.sha |
| }}" |
| steps: |
| - name: Discover PR merge commit |
| id: discover-pr-merge-commit |
| run: | |
| TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)" |
| echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> ${GITHUB_ENV} |
| echo "target-commit-sha=${TARGET_COMMIT_SHA}" >> ${GITHUB_OUTPUT} |
| if: github.event_name == 'pull_request_target' |
| # The labels in the event aren't updated when re-triggering the job, So lets hit the API to get |
| # up-to-date values |
| - name: Get latest PR labels |
| id: get-latest-pr-labels |
| run: | |
| echo -n "pull-request-labels=" >> ${GITHUB_OUTPUT} |
| gh api graphql --paginate -F node_id=${{github.event.pull_request.node_id}} -f query=' |
| query($node_id: ID!, $endCursor: String) { |
| node(id:$node_id) { |
| ... on PullRequest { |
| labels(first: 100, after: $endCursor) { |
| nodes { name } |
| pageInfo { hasNextPage endCursor } |
| } |
| } |
| } |
| }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' >> ${GITHUB_OUTPUT} |
| if: github.event_name == 'pull_request_target' |
| # Retrieve it to be able to determine which files has changed in the incoming commit of the PR |
| # we checkout the target commit and it's parent to be able to compare them |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - uses: actions/checkout@v3 |
| with: |
| ref: ${{ env.TARGET_COMMIT_SHA }} |
| persist-credentials: false |
| fetch-depth: 2 |
| - name: "Setup python" |
| uses: actions/setup-python@v4 |
| with: |
| python-version: 3.7 |
| - name: "Retrieve defaults from branch_defaults.py" |
| # We cannot "execute" the branch_defaults.py python code here because that would be |
| # a security problem (we cannot run any code that comes from the sources coming from the PR. |
| # Therefore, we extract the branches via embedded Python code |
| # we need to do it before next step replaces checked-out breeze and scripts code coming from |
| # the PR, because the PR defaults have to be retrieved here. |
| id: defaults |
| run: | |
| python - <<EOF >> ${GITHUB_ENV} |
| from pathlib import Path |
| import re |
| import sys |
| |
| DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text() |
| BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$' |
| CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$' |
| |
| branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) |
| constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) |
| |
| output = f""" |
| DEFAULT_BRANCH={branch} |
| DEFAULT_CONSTRAINTS_BRANCH={constraints_branch} |
| """.strip() |
| |
| print(output) |
| # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log |
| print(output, file=sys.stderr) |
| EOF |
| - name: "Get target branch build scripts" |
| uses: ./.github/actions/get-target-branch-build-scripts |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| - name: Selective checks |
| id: selective-checks |
| env: |
| PR_LABELS: "${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}" |
| COMMIT_REF: "${{ env.TARGET_COMMIT_SHA }}" |
| VERBOSE: "false" |
| run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} |
| - name: env |
| run: printenv |
| env: |
| PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }} |
| GITHUB_CONTEXT: ${{ toJson(github) }} |
| |
| build-ci-images: |
| permissions: |
| packages: write |
| timeout-minutes: 80 |
| name: > |
| Build CI images ${{needs.build-info.outputs.all-python-versions-list-as-string}} |
| runs-on: ${{ needs.build-info.outputs.runs-on }} |
| needs: [build-info] |
| if: | |
| needs.build-info.outputs.image-build == 'true' && |
| github.event.pull_request.head.repo.full_name != 'apache/airflow' |
| env: |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} |
| RUNS_ON: ${{ needs.build-info.outputs.runs-on }} |
| BACKEND: sqlite |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - uses: actions/checkout@v3 |
| with: |
| ref: ${{ needs.build-info.outputs.target-commit-sha }} |
| persist-credentials: false |
| submodules: recursive |
| - name: "Get target branch build scripts" |
| uses: ./.github/actions/get-target-branch-build-scripts |
| - name: > |
| Build CI Images ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/build-ci-images |
| env: |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} |
| |
| build-prod-images: |
| permissions: |
| packages: write |
| timeout-minutes: 80 |
| name: > |
| Build PROD images |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} |
| runs-on: ${{ needs.build-info.outputs.runs-on }} |
| needs: [build-info, build-ci-images] |
| if: | |
| needs.build-info.outputs.image-build == 'true' && |
| github.event.pull_request.head.repo.full_name != 'apache/airflow' |
| env: |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} |
| RUNS_ON: ${{ needs.build-info.outputs.runs-on }} |
| BACKEND: sqlite |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - uses: actions/checkout@v3 |
| with: |
| ref: ${{ needs.build-info.outputs.target-commit-sha }} |
| persist-credentials: false |
| submodules: recursive |
| - name: "Get target branch build scripts" |
| uses: ./.github/actions/get-target-branch-build-scripts |
| - name: > |
| Build PROD Images |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} |
| uses: ./.github/actions/build-prod-images |
| with: |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} |
| env: |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} |
| |
| build-ci-images-arm: |
| timeout-minutes: 50 |
| name: "Build ARM CI images ${{needs.build-info.outputs.all-python-versions-list-as-string}}" |
| runs-on: ${{ needs.build-info.outputs.runs-on }} |
| needs: [build-info, build-prod-images] |
| if: | |
| needs.build-info.outputs.image-build == 'true' && |
| needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' && |
| github.event.pull_request.head.repo.full_name != 'apache/airflow' && |
| needs.build-info.outputs.runs-on == 'self-hosted' |
| env: |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} |
| RUNS_ON: ${{ needs.build-info.outputs.runs-on }} |
| BACKEND: sqlite |
| outputs: ${{toJSON(needs.build-info.outputs) }} |
| steps: |
| - name: Cleanup repo |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - uses: actions/checkout@v3 |
| with: |
| ref: ${{ needs.build-info.outputs.target-commit-sha }} |
| persist-credentials: false |
| submodules: recursive |
| - name: "Get target branch build scripts" |
| uses: ./.github/actions/get-target-branch-build-scripts |
| - name: "Start ARM instance" |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| - name: > |
| Build ARM CI images ${{ env.IMAGE_TAG }} |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} |
| run: > |
| breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64" |
| env: |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} |
| - name: "Stop ARM instance" |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh |
| if: always() |
| - name: "Fix ownership" |
| run: breeze ci fix-ownership |
| if: always() |
