| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| # To learn more about GitHub Actions in Apache Beam check the CI.md |
| |
| name: Build python source distribution and wheels |
| |
| on: |
| schedule: |
| - cron: '10 2 * * *' |
| push: |
| branches: ['master', 'release-*'] |
| tags: 'v*' |
| pull_request: |
| branches: ['master', 'release-*'] |
| tags: 'v*' |
| paths: ['sdks/python/**', 'model/**', 'release/**'] |
| workflow_dispatch: |
| |
| env: |
| GCP_PATH: "gs://${{ secrets.GCP_PYTHON_WHEELS_BUCKET }}/${GITHUB_REF##*/}/${GITHUB_SHA}-${GITHUB_RUN_ID}/" |
| |
| |
| jobs: |
| |
| check_gcp_variables: |
| timeout-minutes: 5 |
| name: "Check GCP variables" |
| runs-on: ubuntu-latest |
| outputs: |
| gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }} |
| steps: |
| - uses: actions/checkout@v2 |
| - name: "Check are GCP variables set" |
| run: "./scripts/ci/ci_check_are_gcp_variables_set.sh" |
| id: check_gcp_variables |
| env: |
| GCP_SA_EMAIL: ${{ secrets.GCP_SA_EMAIL }} |
| GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} |
| GCP_PYTHON_WHEELS_BUCKET: ${{ secrets.GCP_PYTHON_WHEELS_BUCKET }} |
| GCP_PROJECT_ID: "not-needed-here" |
| GCP_REGION: "not-needed-here" |
| GCP_TESTING_BUCKET: "not-needed-here" |
| |
| build_source: |
| runs-on: ubuntu-latest |
| name: Build python source distribution |
| outputs: |
| is_rc: ${{ steps.is_rc.outputs.is_rc }} |
| rc_num: ${{ steps.get_rc_version.outputs.RC_NUM }} |
| steps: |
| - name: Checkout code |
| uses: actions/checkout@v2 |
| - name: Install python |
| uses: actions/setup-python@v2 |
| with: |
| python-version: 3.7 |
| - name: Get build dependencies |
| working-directory: ./sdks/python |
| run: python -m pip install -r build-requirements.txt |
| - name: Install wheels |
| run: python -m pip install wheel |
| - name: Get tag |
| id: get_tag |
| run: | |
| echo ::set-output name=TAG::${GITHUB_REF#refs/*/} |
| - name: Check whether an -RC tag was applied to the commit. |
| id: is_rc |
| run: | |
| echo ${{ steps.get_tag.outputs.TAG }} > temp |
| OUTPUT=$( if grep -e '-RC.' -q temp; then echo 1; else echo 0; fi) |
| echo "::set-output name=is_rc::$OUTPUT" |
| - name: Get RELEASE_VERSION and RC_NUM |
| if: steps.is_rc.outputs.is_rc == 1 |
| id: get_rc_version |
| run: | |
| RC_NUM=$(sed -n "s/^.*-RC\([0-9]*\)/\1/p" temp) |
| RELEASE_VERSION=$(sed -n "s/^v\(.*\)-RC[0-9]/\1/p" temp) |
| echo "::set-output name=RC_NUM::$RC_NUM" |
| echo "::set-output name=RELEASE_VERSION::$RELEASE_VERSION" |
| - name: Build source |
| working-directory: ./sdks/python |
| run: python setup.py sdist --formats=zip |
| - name: Add checksums |
| working-directory: ./sdks/python/dist |
| run: | |
| file=$(ls | grep .zip | head -n 1) |
| sha512sum $file > ${file}.sha512 |
| - name: Unzip source |
| working-directory: ./sdks/python |
| run: unzip dist/$(ls dist | grep .zip | head -n 1) |
| - name: Rename source directory |
| working-directory: ./sdks/python |
| run: mv $(ls | grep apache-beam) apache-beam-source |
| - name: Upload source as artifact |
| uses: actions/upload-artifact@v2 |
| with: |
| name: source |
| path: sdks/python/apache-beam-source |
| - name: Upload compressed sources as artifacts |
| uses: actions/upload-artifact@v2 |
| with: |
| name: source_zip |
| path: sdks/python/dist |
| - name: Clear dist |
| if: steps.is_rc.outputs.is_rc == 1 |
| working-directory: ./sdks/python |
| run: | |
| rm -r ./dist |
| rm -rd apache-beam-source |
| - name: Rewrite SDK version to include RC number |
| if: steps.is_rc.outputs.is_rc == 1 |
| working-directory: ./sdks/python |
| run: | |
| RELEASE_VERSION=${{ steps.get_rc_version.outputs.RELEASE_VERSION }} |
| RC_NUM=${{ steps.get_rc_version.outputs.RC_NUM }} |
| sed -i -e "s/${RELEASE_VERSION}/${RELEASE_VERSION}rc${RC_NUM}/g" apache_beam/version.py |
| - name: Build RC source |
| if: steps.is_rc.outputs.is_rc == 1 |
| working-directory: ./sdks/python |
| run: python setup.py sdist --formats=zip |
| - name: Add RC checksums |
| if: steps.is_rc.outputs.is_rc == 1 |
| working-directory: ./sdks/python/dist |
| run: | |
| file=$(ls | grep .zip | head -n 1) |
| sha512sum $file > ${file}.sha512 |
| - name: Unzip RC source |
| if: steps.is_rc.outputs.is_rc == 1 |
| working-directory: ./sdks/python |
| run: unzip dist/$(ls dist | grep .zip | head -n 1) |
| - name: Rename RC source directory |
| if: steps.is_rc.outputs.is_rc == 1 |
| working-directory: ./sdks/python |
| run: mv $(ls | grep apache-beam) apache-beam-source-rc |
| - name: Upload RC source as artifact |
| if: steps.is_rc.outputs.is_rc == 1 |
| uses: actions/upload-artifact@v2 |
| with: |
| name: source_rc${{ steps.get_rc_version.outputs.RC_NUM }} |
| path: sdks/python/apache-beam-source-rc |
| - name: Upload compressed RC sources as artifacts |
| if: steps.is_rc.outputs.is_rc == 1 |
| uses: actions/upload-artifact@v2 |
| with: |
| name: source_zip_rc${{ steps.get_rc_version.outputs.RC_NUM }} |
| path: sdks/python/dist |
| |
| |
| prepare_gcs: |
| name: Prepare GCS |
| needs: |
| - build_source |
| - check_gcp_variables |
| runs-on: ubuntu-latest |
| if: needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request' |
| steps: |
| - name: Authenticate on GCP |
| uses: google-github-actions/setup-gcloud@master |
| with: |
| service_account_email: ${{ secrets.GCP_SA_EMAIL }} |
| service_account_key: ${{ secrets.GCP_SA_KEY }} |
| - name: Remove existing files on GCS bucket |
| run: gsutil rm -r ${{ env.GCP_PATH }} || true |
| |
| upload_source_to_gcs: |
| name: Upload python source distribution to GCS bucket |
| needs: |
| - prepare_gcs |
| - check_gcp_variables |
| runs-on: ubuntu-latest |
| if: needs.check_gcp_variables.outputs.gcp-variables-set == 'true' |
| steps: |
| - name: Download compressed sources from artifacts |
| uses: actions/download-artifact@v2 |
| with: |
| name: source_zip |
| path: source/ |
| - name: Authenticate on GCP |
| uses: google-github-actions/setup-gcloud@master |
| with: |
| service_account_email: ${{ secrets.GCP_SA_EMAIL }} |
| service_account_key: ${{ secrets.GCP_SA_KEY }} |
| - name: Copy sources to GCS bucket |
| run: gsutil cp -r -a public-read source/* ${{ env.GCP_PATH }} |
| |
| build_wheels: |
| name: Build python wheels on ${{matrix.arch}} for ${{ matrix.os_python.os }} |
| needs: build_source |
| env: |
| CIBW_ARCHS_LINUX: ${{matrix.arch}} |
| runs-on: ${{ matrix.os_python.os }} |
| strategy: |
| matrix: |
| os_python: [ |
| {"os": "ubuntu-latest", "python": "cp36-* cp37-* cp38-*"}, |
| {"os": "macos-latest", "python": "cp36-* cp37-* cp38-*"}, |
| {"os": "windows-latest", "python": "cp36-* cp37-* cp38-*"}, |
| ] |
| arch: [auto] |
| include: |
| - os_python: {"os": "ubuntu-latest", "python": "cp36-* cp37-* cp38-*"} |
| arch: aarch64 |
| steps: |
| - name: Download python source distribution from artifacts |
| uses: actions/download-artifact@v2 |
| with: |
| name: source |
| path: apache-beam-source |
| - name: Download Python SDK RC source distribution from artifacts |
| if: ${{ needs.build_source.outputs.is_rc == 1 }} |
| uses: actions/download-artifact@v2 |
| with: |
| name: source_rc${{ needs.build_source.outputs.rc_num }} |
| path: apache-beam-source-rc |
| - name: Install Python |
| uses: actions/setup-python@v2 |
| with: |
| python-version: 3.7 |
| - uses: docker/setup-qemu-action@v1 |
| if: ${{matrix.arch == 'aarch64'}} |
| name: Set up QEMU |
| - name: Install cibuildwheel |
| run: pip install cibuildwheel==1.11.0 |
| - name: Build wheel |
| working-directory: apache-beam-source |
| env: |
| CIBW_BUILD: ${{ matrix.os_python.python }} |
| CIBW_BEFORE_BUILD: pip install cython |
| run: cibuildwheel --print-build-identifiers && cibuildwheel --output-dir wheelhouse |
| shell: bash |
| - name: install sha512sum on MacOS |
| if: startsWith(matrix.os_python.os, 'macos') |
| run: brew install coreutils |
| - name: Add checksums |
| working-directory: apache-beam-source/wheelhouse/ |
| run: | |
| for file in *.whl; do |
| sha512sum $file > ${file}.sha512 |
| done |
| shell: bash |
| - name: Upload wheels as artifacts |
| uses: actions/upload-artifact@v2 |
| with: |
| name: wheelhouse-${{ matrix.os_python.os }} |
| path: apache-beam-source/wheelhouse/ |
| - name: Build RC wheels |
| if: ${{ needs.build_source.outputs.is_rc == 1 }} |
| working-directory: apache-beam-source-rc |
| env: |
| CIBW_BUILD: ${{ matrix.os_python.python }} |
| CIBW_BEFORE_BUILD: pip install cython |
| run: cibuildwheel --print-build-identifiers && cibuildwheel --output-dir wheelhouse |
| shell: bash |
| - name: Add RC checksums |
| if: ${{ needs.build_source.outputs.is_rc == 1 }} |
| working-directory: apache-beam-source-rc/wheelhouse/ |
| run: | |
| for file in *.whl; do |
| sha512sum $file > ${file}.sha512 |
| done |
| shell: bash |
| - name: Upload RC wheels as artifacts |
| if: ${{ needs.build_source.outputs.is_rc == 1 }} |
| uses: actions/upload-artifact@v2 |
| with: |
| name: wheelhouse-rc${{ needs.build_source.outputs.rc_num }}-${{ matrix.os_python.os }} |
| path: apache-beam-source-rc/wheelhouse/ |
| |
| upload_wheels_to_gcs: |
| name: Upload wheels to GCS bucket |
| needs: |
| - build_wheels |
| - check_gcp_variables |
| runs-on: ubuntu-latest |
| if: needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request' |
| strategy: |
| matrix: |
| os : [ubuntu-latest, macos-latest, windows-latest] |
| steps: |
| - name: Download wheels from artifacts |
| uses: actions/download-artifact@v2 |
| with: |
| name: wheelhouse-${{ matrix.os }} |
| path: wheelhouse/ |
| - name: Authenticate on GCP |
| uses: google-github-actions/setup-gcloud@master |
| with: |
| service_account_email: ${{ secrets.GCP_SA_EMAIL }} |
| service_account_key: ${{ secrets.GCP_SA_KEY }} |
| - name: Copy wheels to GCS bucket |
| run: gsutil cp -r -a public-read wheelhouse/* ${{ env.GCP_PATH }} |
| - name: Create github action information file on GCS bucket |
| run: | |
| cat > github_action_info <<EOF |
| GITHUB_WORKFLOW=$GITHUB_WORKFLOW |
| GITHUB_RUN_ID=$GITHUB_RUN_ID |
| GITHUB_RUN_NUMBER=$GITHUB_RUN_NUMBER |
| GITHUB_ACTION=$GITHUB_ACTION |
| GITHUB_ACTOR=$GITHUB_ACTOR |
| GITHUB_REPOSITORY=$GITHUB_REPOSITORY |
| GITHUB_EVENT_NAME=$GITHUB_EVENT_NAME |
| GITHUB_SHA=$GITHUB_SHA |
| GITHUB_REF=$GITHUB_REF |
| # only for forked repositiories |
| GITHUB_HEAD_REF=$GITHUB_HEAD_REF |
| GITHUB_BASE_REF=$GITHUB_BASE_REF |
| EOF |
| echo $(cat github_action_info) |
| gsutil cp -a public-read github_action_info ${{ env.GCP_PATH }} |
| - name: Upload GitHub event file to GCS bucket |
| run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{ env.GCP_PATH }} |
| |
| list_files_on_gcs: |
| name: List files on Google Cloud Storage Bucket |
| needs: |
| - upload_wheels_to_gcs |
| - check_gcp_variables |
| runs-on: ubuntu-latest |
| if: needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request' |
| steps: |
| - name: Authenticate on GCP |
| uses: google-github-actions/setup-gcloud@master |
| with: |
| service_account_email: ${{ secrets.GCP_SA_EMAIL }} |
| service_account_key: ${{ secrets.GCP_SA_KEY }} |
| - name: List file on Google Cloud Storage Bucket |
| run: gsutil ls "${{ env.GCP_PATH }}*" |
| |
| tag_repo_nightly: |
| name: Tag repo nightly |
| needs: |
| - build_source |
| - build_wheels |
| runs-on: ubuntu-latest |
| timeout-minutes: 60 |
| if: github.repository_owner == 'apache' && github.event_name == 'schedule' |
| steps: |
| - name: Checkout code on master branch |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| submodules: recursive |
| - name: Tag commit |
| run: | |
| BRANCH_NAME=${GITHUB_REF##*/} |
| echo "Tagging ${BRANCH_NAME}" |
| git tag -f nightly-${BRANCH_NAME} HEAD |
| - name: Push tags |
| uses: ./.github/actions/github-push-action |
| with: |
| github_token: ${{ secrets.GITHUB_TOKEN }} |
| tags: true |
| force: true |
| branch: master |