| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| --- |
| name: Push image cache |
| on: # yamllint disable-line rule:truthy |
| workflow_call: |
| inputs: |
| runs-on-as-json-public: |
| description: "The array of labels (in json form) determining public runners." |
| required: true |
| type: string |
| runs-on-as-json-self-hosted: |
| description: "The array of labels (in json form) determining self-hosted runners." |
| required: true |
| type: string |
| cache-type: |
| description: "Type of cache to push (Early / Regular)." |
| required: true |
| type: string |
| include-prod-images: |
| description: "Whether to build PROD image cache additionally to CI image cache (true/false)." |
| required: true |
| type: string |
| push-latest-images: |
| description: "Whether to also push latest images (true/false)." |
| required: true |
| type: string |
| debian-version: |
| description: "Base Debian distribution to use for the build (bookworm/bullseye)" |
| type: string |
| default: "bookworm" |
| install-mysql-client-type: |
| description: "MySQL client type to use during build (mariadb/mysql)" |
| type: string |
| default: "mariadb" |
| platform: |
| description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" |
| required: true |
| type: string |
| python-versions: |
| description: "JSON-formatted array of Python versions to build images from" |
| required: true |
| type: string |
| branch: |
| description: "Branch used to run the CI jobs in (main/v2_*_test)." |
| required: true |
| type: string |
| constraints-branch: |
| description: "Branch used to construct constraints URL from." |
| required: true |
| type: string |
| use-uv: |
| description: "Whether to use uv to build the image (true/false)" |
| required: true |
| type: string |
| include-success-outputs: |
| description: "Whether to include success outputs (true/false)." |
| required: true |
| type: string |
| docker-cache: |
| description: "Docker cache specification to build the image (registry, local, disabled)." |
| required: true |
| type: string |
| jobs: |
| push-ci-image-cache: |
| name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache " |
| # NOTE!!!!! This has to be put in one line for runs-on to recognize the "fromJSON" properly !!!! |
| # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" |
| # instead of an array of strings. |
| # yamllint disable-line rule:line-length |
| runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} |
| strategy: |
| fail-fast: false |
| matrix: |
| python: ${{ fromJSON(inputs.python-versions) }} |
| env: |
| COMMIT_SHA: ${{ github.sha }} |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the |
| # Airflow one is going to be used |
| CONSTRAINTS_GITHUB_REPOSITORY: >- |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} |
| # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one |
| DEBIAN_VERSION: ${{ inputs.debian-version }} |
| DEFAULT_BRANCH: ${{ inputs.branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} |
| DOCKER_CACHE: ${{ inputs.docker-cache }} |
| GITHUB_REPOSITORY: ${{ github.repository }} |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| GITHUB_USERNAME: ${{ github.actor }} |
| INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}" |
| INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} |
| USE_UV: ${{ inputs.use-uv }} |
| UPGRADE_TO_NEWER_DEPENDENCIES: "false" |
| VERBOSE: "true" |
| VERSION_SUFFIX_FOR_PYPI: "dev0" |
| steps: |
| - name: "Cleanup repo" |
| shell: bash |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v4 |
| with: |
| persist-credentials: false |
| - name: "Cleanup docker" |
| run: ./scripts/ci/cleanup_docker.sh |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| - name: "Start ARM instance" |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh |
| if: inputs.platform == 'linux/arm64' |
| - name: Login to ghcr.io |
| run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin |
| - name: "Push CI ${{ inputs.cache-type }} cache: ${{ matrix.python }} ${{ inputs.platform }}" |
| run: > |
| breeze ci-image build --builder airflow_cache --prepare-buildx-cache |
| --platform "${{ inputs.platform }}" --python ${{ matrix.python }} |
| - name: "Stop ARM instance" |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh |
| if: always() && inputs.platform == 'linux/arm64' |
| - name: "Push CI latest images: ${{ matrix.python }} (linux/amd64 only)" |
| run: > |
| breeze ci-image build --builder airflow_cache --push |
| --python "${{ matrix.python }}" --platform "${{ inputs.platform }}" |
| if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64' |
| |
| push-prod-image-cache: |
| name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache" |
| # NOTE!!!!! This has to be put in one line for runs-on to recognize the "fromJSON" properly !!!! |
| # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" |
| # instead of an array of strings. |
| # yamllint disable-line rule:line-length |
| runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} |
| strategy: |
| fail-fast: false |
| matrix: |
| python: ${{ fromJSON(inputs.python-versions) }} |
| env: |
| COMMIT_SHA: ${{ github.sha }} |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the |
| # Airflow one is going to be used |
| CONSTRAINTS_GITHUB_REPOSITORY: >- |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} |
| # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one |
| DEBIAN_VERSION: ${{ inputs.debian-version }} |
| DEFAULT_BRANCH: ${{ inputs.branch }} |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} |
| DOCKER_CACHE: ${{ inputs.docker-cache }} |
| GITHUB_REPOSITORY: ${{ github.repository }} |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| GITHUB_USERNAME: ${{ github.actor }} |
| INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} |
| UPGRADE_TO_NEWER_DEPENDENCIES: "false" |
| USE_UV: ${{ inputs.branch == 'main' && inputs.use-uv || 'false' }} |
| VERBOSE: "true" |
| VERSION_SUFFIX_FOR_PYPI: "dev0" |
| if: inputs.include-prod-images == 'true' |
| steps: |
| - name: "Cleanup repo" |
| shell: bash |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v4 |
| with: |
| persist-credentials: false |
| - name: "Cleanup docker" |
| run: ./scripts/ci/cleanup_docker.sh |
| - name: "Install Breeze" |
| uses: ./.github/actions/breeze |
| - name: "Cleanup dist and context file" |
| run: rm -fv ./dist/* ./docker-context-files/* |
| - name: "Download packages prepared as artifacts" |
| uses: actions/download-artifact@v4 |
| with: |
| name: prod-packages |
| path: ./docker-context-files |
| - name: "Start ARM instance" |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh |
| if: inputs.platform == 'linux/arm64' |
| - name: Login to ghcr.io |
| run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin |
| - name: "Push PROD ${{ inputs.cache-type }} cache: ${{ matrix.python-version }} ${{ inputs.platform }}" |
| run: > |
| breeze prod-image build --builder airflow_cache |
| --prepare-buildx-cache --platform "${{ inputs.platform }}" |
| --install-packages-from-context --airflow-constraints-mode constraints-source-providers |
| --python ${{ matrix.python }} |
| - name: "Stop ARM instance" |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh |
| if: always() && inputs.platform == 'linux/arm64' |
| # We only push "AMD" images as it is really only needed for any kind of automated builds in CI |
| # and currently there is not an easy way to make multi-platform image from two separate builds |
| # and we can do it after we stopped the ARM instance as it is not needed anymore |
| - name: "Push PROD latest image: ${{ matrix.python }} (linux/amd64 ONLY)" |
| run: > |
| breeze prod-image build --builder airflow_cache --install-packages-from-context |
| --push --platform "${{ inputs.platform }}" |
| if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64' |
