| name: build_release_candidate |
| |
| # Workflow added after https://github.com/apache/beam/commit/4183e747becebd18becee5fff547af365910fc9c |
| # If help is needed debugging issues, you can view the release guide at that commit for guidance on how to do this manually. |
| # (https://github.com/apache/beam/blob/4183e747becebd18becee5fff547af365910fc9c/website/www/site/content/en/contribute/release-guide.md) |
| on: |
| workflow_dispatch: |
| inputs: |
| RELEASE: |
| description: Beam version of current release (e.g. 2.XX.0) |
| required: true |
| default: '2.XX.0' |
| RC: |
| description: Integer RC version for the release (e.g. 3 for RC3) |
| required: true |
| APACHE_ID: |
| description: Your Apache id. Required if you want to stage artifacts into https://dist.apache.org/repos/dist/dev/beam/ |
| required: false |
| APACHE_PASSWORD: |
| description: Your Apache password. Required if you want to stage artifacts into https://dist.apache.org/repos/dist/dev/beam/ |
| required: false |
| REPO_TOKEN: |
| description: | |
| Github Personal Access Token with repo permissions if you want to create the beam-site docs PR, |
| create a draft release, or upload prism artifacts to that release. |
| See https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens. |
| default: '' |
| STAGE: |
| description: | |
| Configure which actions this workflow should perform, by setting the desired fields to "yes". |
| -- |
| java_artifacts: publish java artifacts to https://repository.apache.org/#stagingRepositories |
| -- |
| java_source: java source into https://dist.apache.org/repos/dist/dev/beam/ |
| -- |
| docker_artifacts: stage SDK docker images to docker hub Apache organization |
| -- |
| python_artifacts: stage the python artifacts into https://dist.apache.org/repos/dist/dev/beam/ |
| -- |
| beam_site_pr: create the documentation update PR against apache/beam-site. |
| -- |
| prism: build and upload the artifacts to the release for this tag |
| -- |
| managed_io_docs_pr: create the managed-io.md update PR against apache/beam. |
| required: true |
| default: | |
| {java_artifacts: "no", |
| java_source: "no", |
| docker_artifacts: "no", |
| python_artifacts: "no", |
| beam_site_pr: "no", |
| prism: "no", |
| managed_io_docs_pr: "no"} |
| |
| env: |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| |
| jobs: |
| publish_java_artifacts: |
| if: ${{ fromJson(github.event.inputs.STAGE).java_artifacts == 'yes'}} |
| runs-on: [self-hosted, ubuntu-20.04, main] |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| with: |
| ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" |
| repository: apache/beam |
| - name: Install Java 11 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: | |
| 21 |
| 11 |
| - name: Import GPG key |
| id: import_gpg |
| uses: crazy-max/ghaction-import-gpg@e89d40939c28e39f97cf32126055eeae86ba74ec |
| with: |
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} |
| - name: Auth for nexus |
| run: | |
| mkdir -p ${HOME}/.m2 |
| echo "<settings> |
| <servers> |
| <server> |
| <id>apache.releases.https</id> |
| <username>${{ secrets.NEXUS_STAGE_DEPLOYER_USER }}</username> |
| <password>${{ secrets.NEXUS_STAGE_DEPLOYER_PW }}</password> |
| </server> |
| <server> |
| <id>apache.snapshots.https</id> |
| <username>${{ secrets.NEXUS_USER }}</username> |
| <password>${{ secrets.NEXUS_PW }}</password> |
| </server> |
| </servers> |
| </settings>" > ${HOME}/.m2/settings.xml |
| - name: Configure git |
| run: git config credential.helper store |
| - name: Stage Java Artifacts into Maven |
| run: ./gradlew publish -Psigning.gnupg.keyName=${{steps.import_gpg.outputs.fingerprint}} -PisRelease -Pjava21Home=$JAVA_HOME_21_X64 --no-daemon --no-parallel |
| |
| |
| stage_java_source: |
| if: ${{ fromJson(github.event.inputs.STAGE).java_source == 'yes'}} |
| runs-on: ubuntu-22.04 |
| steps: |
| - name: Mask Apache Password |
| run: | |
| # Workaround for Actions bug - https://github.com/actions/runner/issues/643 |
| APACHE_PASS=$(jq -r '.inputs.APACHE_PASSWORD' $GITHUB_EVENT_PATH) |
| echo "::add-mask::$APACHE_PASS" |
| - name: Validate apache id/password |
| run: | |
| if [ "${{ github.event.inputs.APACHE_ID }}" == "" ] |
| then |
| echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" |
| fi |
| if [ "${{ github.event.inputs.APACHE_PASSWORD }}" == "" ] |
| then |
| echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" |
| fi |
| - name: Install Java 11 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: '11' |
| - name: Import GPG key |
| id: import_gpg |
| uses: crazy-max/ghaction-import-gpg@e89d40939c28e39f97cf32126055eeae86ba74ec |
| with: |
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} |
| - name: stage source |
| run: | |
| svn co https://dist.apache.org/repos/dist/dev/beam |
| mkdir -p beam/${{ github.event.inputs.RELEASE }} |
| cd beam/${{ github.event.inputs.RELEASE }} |
| RC_DIR="beam-${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" |
| RC_ZIP="${RC_DIR}.zip" |
| RELEASE_DIR="beam-${{ github.event.inputs.RELEASE }}" |
| RC_TAG="v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" |
| SOURCE_RELEASE_ZIP="apache-beam-${{ github.event.inputs.RELEASE }}-source-release.zip" |
| # Check whether there is an existing dist dir |
| if (svn ls "${SOURCE_RELEASE_ZIP}"); then |
| echo "Removing existing ${SOURCE_RELEASE_ZIP}." |
| svn delete "${SOURCE_RELEASE_ZIP}" |
| fi |
| |
| echo "Downloading: https://github.com/apache/beam/archive/${RC_TAG}.zip" |
| wget https://github.com/apache/beam/archive/${RC_TAG}.zip -O "${RC_ZIP}" |
| |
| unzip "$RC_ZIP" |
| rm "$RC_ZIP" |
| mv "$RC_DIR" "$RELEASE_DIR" |
| zip -r "${SOURCE_RELEASE_ZIP}" "$RELEASE_DIR" |
| rm -r "$RELEASE_DIR" |
| |
| echo "----Signing Source Release ${SOURCE_RELEASE_ZIP}-----" |
| gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig "${SOURCE_RELEASE_ZIP}" |
| |
| echo "----Creating Hash Value for ${SOURCE_RELEASE_ZIP}----" |
| sha512sum ${SOURCE_RELEASE_ZIP} > ${SOURCE_RELEASE_ZIP}.sha512 |
| |
| svn add --force . |
| svn status |
| svn commit -m "Staging Java artifacts for Apache Beam ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}" |
| |
| stage_python_artifacts: |
| if: ${{ fromJson(github.event.inputs.STAGE).python_artifacts == 'yes'}} |
| needs: [stage_java_source, build_and_stage_prism] # Enforce ordering to avoid svn conflicts |
| runs-on: ubuntu-22.04 |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| - name: Mask Apache Password |
| run: | |
| # Workaround for Actions bug - https://github.com/actions/runner/issues/643 |
| APACHE_PASS=$(jq -r '.inputs.APACHE_PASSWORD' $GITHUB_EVENT_PATH) |
| echo "::add-mask::$APACHE_PASS" |
| - name: Mask apache id/password |
| run: | |
| if [ "${{ github.event.inputs.APACHE_ID }}" == "" ] |
| then |
| echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" |
| fi |
| if [ "${{ github.event.inputs.APACHE_PASSWORD }}" == "" ] |
| then |
| echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" |
| fi |
| - name: Setup environment |
| uses: ./.github/actions/setup-environment-action |
| with: |
| python-version: default |
| disable-cache: true |
| - name: Import GPG key |
| id: import_gpg |
| uses: crazy-max/ghaction-import-gpg@e89d40939c28e39f97cf32126055eeae86ba74ec |
| with: |
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} |
| - name: Install dependencies |
| run: | |
| pip install python-dateutil |
| pip install requests |
| - name: stage python artifacts |
| env: |
| RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" |
| GIT_REPO_BASE_URL: https://github.com/apache/beam |
| RELEASE_DIR: "beam/${{ github.event.inputs.RELEASE }}" |
| RELEASE: "${{ github.event.inputs.RELEASE }}" |
| SCRIPT_DIR: release/src/main/scripts |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| SVN_ARTIFACTS_DIR: "beam/${{ github.event.inputs.RELEASE }}/python" |
| run: | |
| svn co https://dist.apache.org/repos/dist/dev/beam |
| mkdir -p "${SVN_ARTIFACTS_DIR}" |
| |
| git fetch --all --tags --prune |
| RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}") |
| |
| python "${SCRIPT_DIR}/download_github_actions_artifacts.py" \ |
| --github-token-var GITHUB_TOKEN \ |
| --repo-url "apache/beam" \ |
| --rc-tag "${RC_TAG}" \ |
| --release-commit "${RELEASE_COMMIT}" \ |
| --artifacts_dir "${RELEASE_DIR}/python" \ |
| --yes True |
| |
| cd "${RELEASE_DIR}"/python |
| ls |
| |
| echo "------Checking Hash Value for apache_beam-${RELEASE}.tar.gz-----" |
| sha512sum -c "apache_beam-${RELEASE}.tar.gz.sha512" |
| |
| echo "------Signing Source Release apache_beam-${RELEASE}.tar.gz------" |
| gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --detach-sig "apache_beam-${RELEASE}.tar.gz" |
| |
| for artifact in *.whl; do |
| echo "----------Checking Hash Value for ${artifact} wheel-----------" |
| sha512sum -c "${artifact}.sha512" |
| done |
| |
| for artifact in *.whl; do |
| echo "------------------Signing ${artifact} wheel-------------------" |
| gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig "${artifact}" |
| done |
| |
| cd .. |
| svn add --force python |
| svn status |
| svn commit -m "Staging Python artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}" |
| |
| |
| stage_docker: |
| if: ${{ fromJson(github.event.inputs.STAGE).docker_artifacts == 'yes'}} |
| runs-on: ubuntu-22.04 |
| strategy: |
| fail-fast: false |
| matrix: |
| # Split up images to publish so that longer ones are able to run independently/finish faster, otherwise this takes >6 hours |
| # Any task which is skipped from a broader task must be explicitly included in this list to avoid accidentally missing new |
| # tasks as they are added. |
| images_to_publish: [ |
| {"gradle_task": ":pushAllRunnersDockerImages", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, |
| {"gradle_task": ":sdks:python:container:push310", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, |
| {"gradle_task": ":sdks:python:container:push311", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, |
| {"gradle_task": ":sdks:python:container:push312", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, |
| {"gradle_task": ":sdks:python:container:pushAll", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless -Pskip-python-39-images -Pskip-python-310-images -Pskip-python-311-images -Pskip-python-312-images"}, |
| {"gradle_task": ":pushAllSdkDockerImages", "include_skip_flags": "-Pskip-python-images"}, |
| {"gradle_task": ":pushAllDockerImages", "include_skip_flags": "-Pskip-runner-images -Pskip-sdk-images"} |
| ] |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| with: |
| ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" |
| repository: apache/beam |
| - name: Free Disk Space (Ubuntu) |
| uses: jlumbroso/free-disk-space@v1.3.1 |
| - name: Install Java 11 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: '11' |
| - name: Install Python 3.10 |
| uses: actions/setup-python@v5 |
| with: |
| python-version: '3.10' |
| - name: Set up Docker Buildx |
| uses: docker/setup-buildx-action@v3 |
| - name: Remove default github maven configuration |
| # This step is a workaround to avoid a decryption issue of Beam's |
| # net.linguica.gradle.maven.settings plugin and github's provided maven |
| # settings.xml file |
| run: rm ~/.m2/settings.xml || true |
| - name: Login to Docker Hub |
| uses: docker/login-action@v3 |
| with: |
| username: ${{ secrets.DOCKERHUB_USER }} |
| password: ${{ secrets.DOCKERHUB_TOKEN }} |
| - name: Push docker images |
| run: ./gradlew ${{ matrix.images_to_publish.gradle_task }} -PisRelease -Pdocker-pull-licenses -Pprune-images ${{ matrix.images_to_publish.include_skip_flags }} -Pdocker-tag=${{ github.event.inputs.RELEASE }}rc${{ github.event.inputs.RC }} --no-daemon --no-parallel |
| |
| beam_site_pr: |
| if: ${{ fromJson(github.event.inputs.STAGE).beam_site_pr == 'yes'}} |
| # Note: if this ever changes to self-hosted, remove the "Remove default github maven configuration" step |
| runs-on: ubuntu-22.04 |
| env: |
| RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" |
| BRANCH_NAME: updates_release_${{ github.event.inputs.RELEASE }} |
| BEAM_ROOT_DIR: ${{ github.workspace }}/beam |
| SITE_ROOT_DIR: ${{ github.workspace }}/beam-site |
| steps: |
| - name: Free Disk Space (Ubuntu) |
| uses: jlumbroso/free-disk-space@v1.3.1 |
| with: |
| docker-images: false |
| - name: Checkout Beam Repo |
| uses: actions/checkout@v4 |
| with: |
| ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" |
| repository: apache/beam |
| path: beam |
| - name: Checkout Beam Site Repo |
| uses: actions/checkout@v4 |
| with: |
| repository: apache/beam-site |
| path: beam-site |
| token: ${{ github.event.inputs.REPO_TOKEN }} |
| ref: release-docs |
| - name: Install Python 3.10 |
| uses: actions/setup-python@v5 |
| with: |
| python-version: '3.10' |
| - name: Install node |
| uses: actions/setup-node@v6 |
| with: |
| node-version: '16' |
| - name: Install Java 21 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: '21' |
| - name: Remove default github maven configuration |
| # This step is a workaround to avoid a decryption issue of Beam's |
| # net.linguica.gradle.maven.settings plugin and github's provided maven |
| # settings.xml file |
| run: rm ~/.m2/settings.xml || true |
| - name: Create documentation commit for website |
| run: | |
| echo "OK!" |
| - name: Build Python Docs |
| working-directory: beam/sdks/python |
| run: | |
| pip install --upgrade pip setuptools wheel |
| pip install -U pip |
| pip install tox |
| # TODO(https://github.com/apache/beam/issues/20209): Don't hardcode py version in this file. |
| tox -e docs |
| rm -rf target/docs/_build/.doctrees |
| - name: Build Typescript Docs |
| working-directory: beam/sdks/typescript |
| run: | |
| npm ci && npm run docs |
| - name: Build Java Docs |
| working-directory: beam |
| run: | |
| ./gradlew :sdks:java:javadoc:aggregateJavadoc -PisRelease -PdisableSpotlessCheck=true --no-daemon --no-parallel |
| - name: Build YAML Docs |
| working-directory: beam |
| run: | |
| ./gradlew :sdks:python:generateYamlDocs -PisRelease |
| - name: Add canonical link into javadocs |
| uses: cicirello/javadoc-cleanup@v1 |
| with: |
| path-to-root: beam/sdks/java/javadoc/build/docs/javadoc |
| base-url-path: https://beam.apache.org/releases/javadoc/current/ |
| - name: Consolidate Release Docs to beam-site branch with symlinks |
| working-directory: beam-site |
| run: | |
| git checkout -b $BRANCH_NAME release-docs |
| |
| echo "..........Copying generated javadoc into beam-site.........." |
| cp -r ${BEAM_ROOT_DIR}/sdks/java/javadoc/build/docs/javadoc/ javadoc/${{ github.event.inputs.RELEASE }} |
| # Update current symlink to point to the latest release |
| unlink javadoc/current |
| ln -s ${{ github.event.inputs.RELEASE }} javadoc/current |
| |
| echo "............Copying generated pydoc into beam-site.........." |
| cp -r ${BEAM_ROOT_DIR}/sdks/python/target/docs/_build pydoc/${{ github.event.inputs.RELEASE }} |
| # Update current symlink to point to the latest release |
| unlink pydoc/current |
| ln -s ${{ github.event.inputs.RELEASE }} pydoc/current |
| |
| echo "............Copying generated typedoc into beam-site.........." |
| mkdir -p typedoc |
| cp -r ${BEAM_ROOT_DIR}/sdks/typescript/docs typedoc/${{ github.event.inputs.RELEASE }} |
| # Update current symlink to point to the latest release |
| unlink typedoc/current | true |
| ln -s ${{ github.event.inputs.RELEASE }} typedoc/current |
| |
| echo "............Copying generated yamldoc into beam-site.........." |
| mkdir -p yamldoc/${{ github.event.inputs.RELEASE }} |
| cp -r ${BEAM_ROOT_DIR}/sdks/python/build/yaml-ref.html yamldoc/${{ github.event.inputs.RELEASE }}/index.html |
| cp -r ${BEAM_ROOT_DIR}/sdks/python/build/yaml-examples.html yamldoc/${{ github.event.inputs.RELEASE }}/examples.html |
| # Update current symlink to point to the latest release |
| unlink yamldoc/current | true |
| ln -s ${{ github.event.inputs.RELEASE }} yamldoc/current |
| - name: Create commit on beam-site branch |
| working-directory: beam |
| run: | |
| # Get the commit from the beam repo, not the beam-site repo. |
| RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}") |
| cd ${SITE_ROOT_DIR} |
| git config user.name $GITHUB_ACTOR |
| git config user.email actions@"$RUNNER_NAME".local |
| git add -A |
| git commit -m "Update beam-site for release ${{ github.event.inputs.RELEASE }}." -m "Content generated from commit ${RELEASE_COMMIT}." |
| git push -f --set-upstream origin $BRANCH_NAME |
| - name: Create beam-site PR |
| working-directory: beam-site |
| env: |
| GH_TOKEN: ${{ github.event.inputs.REPO_TOKEN }} |
| PR_TITLE: "Publish docs for ${{ github.event.inputs.RELEASE }} release" |
| PR_BODY: "Content generated from https://github.com/apache/beam/tree/${{ env.RC_TAG }}." |
| run: | |
| gh pr create -t "$PR_TITLE" -b "$PR_BODY" --base release-docs |
| |
| build_and_stage_prism: |
| if: ${{ fromJson(github.event.inputs.STAGE).prism == 'yes'}} |
| needs: [stage_java_source] # Enforce ordering to avoid svn conflicts |
| runs-on: ubuntu-22.04 |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| with: |
| ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" |
| repository: apache/beam |
| - name: Mask Apache Password |
| run: | |
| # Workaround for Actions bug - https://github.com/actions/runner/issues/643 |
| APACHE_PASS=$(jq -r '.inputs.APACHE_PASSWORD' $GITHUB_EVENT_PATH) |
| echo "::add-mask::$APACHE_PASS" |
| - name: Mask apache id/password |
| run: | |
| if [ "${{ github.event.inputs.APACHE_ID }}" == "" ] |
| then |
| echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" |
| fi |
| if [ "${{ github.event.inputs.APACHE_PASSWORD }}" == "" ] |
| then |
| echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" |
| fi |
| - uses: actions/setup-go@v6 |
| with: |
| go-version: '1.25' |
| - name: Import GPG key |
| id: import_gpg |
| uses: crazy-max/ghaction-import-gpg@e89d40939c28e39f97cf32126055eeae86ba74ec |
| with: |
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} |
| - name: Build prism artifacts |
| env: |
| RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" |
| GIT_REPO_BASE_URL: https://github.com/apache/beam |
| RELEASE_DIR: "beam/${{ github.event.inputs.RELEASE }}" |
| RELEASE: "${{ github.event.inputs.RELEASE }}" |
| GH_TOKEN: ${{ github.event.inputs.REPO_TOKEN }} |
| SVN_ARTIFACTS_DIR: "beam/${{ github.event.inputs.RELEASE }}/prism" |
| run: | |
| # TODO stop the action if a non-prerelease release exists |
| # TODO create the draft release automatically if none exist. |
| # TODO update the draft release's RC tag to current. |
| |
| export GH_RELEASE_ID=`gh api \ |
| -H "Accept: application/vnd.github+json" \ |
| -H "X-GitHub-Api-Version: 2022-11-28" \ |
| /repos/apache/beam/releases | jq '.[] | select(.tag_name | contains(env.RC_TAG)) .id'` |
| |
| echo "with gh release id $GH_RELEASE_ID for tag $RC_TAG" |
| |
| # Store a reference so we can get back to the start. |
| ROOT_DIR=`pwd` |
| echo "ROOT_DIR=$ROOT_DIR" |
| echo "............Checking out Apache Dev Repo.........." |
| |
| svn co https://dist.apache.org/repos/dist/dev/beam |
| mkdir -p "${SVN_ARTIFACTS_DIR}" |
| |
| OUTPUT_DIR=$ROOT_DIR/$SVN_ARTIFACTS_DIR |
| mkdir -p $OUTPUT_DIR |
| echo "OUTPUT_DIR=$OUTPUT_DIR" |
| |
| # Remove existing prism contents from the svn repo. |
| cd $OUTPUT_DIR |
| rm -rf * |
| cd $ROOT_DIR |
| |
| # Navigate to the prism command directory. |
| cd sdks/go/cmd/prism |
| |
| # Store a reference to where we're building from. |
| BUILD_DIR=`pwd` |
| |
| echo "............Build and stage prism artifacts in the Github Release $GH_RELEASE_ID for tag $RC_TAG.........." |
| |
| |
| # Loop through and build desired set from allowed types. |
| for OS in linux windows darwin; do |
| for ARCH in amd64 arm64; do |
| TARGET_DIR="$OUTPUT_DIR/$OS/$ARCH" |
| TARGET_NAME="apache_beam-v$RELEASE-prism-$OS-$ARCH" |
| |
| mkdir -p "$TARGET_DIR" |
| OUTPUT_FILE="$TARGET_DIR/$TARGET_NAME" |
| if GOOS=$OS GOARCH=$ARCH CGO_ENABLED=0 go build -trimpath -buildvcs=false -o "$OUTPUT_FILE" . > /dev/null 2>&1; then |
| cd $TARGET_DIR |
| # Extract real output name. Windows builds automatically have .exe added. |
| ARTIFACT=`ls` |
| echo "target built - $ARTIFACT" |
| |
| ZIP_NAME="$ARTIFACT.zip" |
| zip -r $ZIP_NAME $ARTIFACT |
| |
| # Sign and hash |
| gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig $ZIP_NAME |
| sha512sum $ZIP_NAME > "${ZIP_NAME}.sha512" |
| |
| # Upload to the release. |
| gh release upload $RC_TAG $ZIP_NAME ${ZIP_NAME}.sha512 ${ZIP_NAME}.asc --clobber |
| |
| # Remove the binary from the release directory |
| rm $ARTIFACT |
| |
| # Return to our root build dir. |
| cd $BUILD_DIR |
| else |
| echo "target failed for $OS - $ARCH" |
| rm -f "$OUTPUT_FILE" |
| exit 1 # Do not continue action |
| fi |
| done |
| done |
| echo "...........Prism builds done!..........." |
| cd $ROOT_DIR |
| cd $RELEASE_DIR |
| |
| echo "...........Adding prism artifacts to the Dev Apache SVN repo..........." |
| |
| svn add --force --parents prism |
| svn status |
| svn commit -m "Staging Prism artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}" |
| |
| managed_io_docs_pr: |
| if: ${{ fromJson(github.event.inputs.STAGE).managed_io_docs_pr == 'yes'}} |
| runs-on: ubuntu-22.04 |
| env: |
| BRANCH_NAME: updates_managed_io_docs_${{ github.event.inputs.RELEASE }}_rc${{ github.event.inputs.RC }} |
| BEAM_ROOT_DIR: ${{ github.workspace }}/beam |
| MANAGED_IO_DOCS_PATH: website/www/site/content/en/documentation/io/managed-io.md |
| steps: |
| - name: Checkout Beam Repo |
| uses: actions/checkout@v4 |
| with: |
| ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" |
| repository: apache/beam |
| path: beam |
| token: ${{ github.event.inputs.REPO_TOKEN }} |
| persist-credentials: true |
| - name: Install Python 3.10 |
| uses: actions/setup-python@v5 |
| with: |
| python-version: '3.10' |
| - name: Install Java 11 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: '11' |
| - name: Remove default github maven configuration |
| # This step is a workaround to avoid a decryption issue of Beam's |
| # net.linguica.gradle.maven.settings plugin and github's provided maven |
| # settings.xml file |
| run: rm ~/.m2/settings.xml || true |
| - name: Install SDK |
| working-directory: beam/sdks/python |
| run: | |
| pip install -e. |
| - name: Build Expansion Service Jar |
| working-directory: beam |
| run: | |
| ./gradlew sdks:java:io:expansion-service:shadowJar |
| - name: Build GCP Expansion Service Jar |
| working-directory: beam |
| run: | |
| ./gradlew sdks:java:io:google-cloud-platform:expansion-service:shadowJar |
| - name: Copy expansion service jars |
| working-directory: beam |
| run: | |
| mkdir -p ~/.apache_beam/cache/jars/ |
| cp sdks/java/io/expansion-service/build/libs/beam-sdks-java-io-expansion-service-${{ github.event.inputs.RELEASE }}.jar ~/.apache_beam/cache/jars/ |
| cp sdks/java/io/google-cloud-platform/expansion-service/build/libs/beam-sdks-java-io-google-cloud-platform-expansion-service-${{ github.event.inputs.RELEASE }}.jar ~/.apache_beam/cache/jars/ |
| - name: Generate Managed IO Docs |
| working-directory: beam/sdks/python |
| run: | |
| python gen_managed_doc.py --output_location ${{ runner.temp }}/managed-io.md |
| - name: Create commit on beam branch |
| working-directory: beam |
| run: | |
| git fetch origin master |
| git checkout -b $BRANCH_NAME origin/master |
| mv ${{ runner.temp }}/managed-io.md ${{ env.MANAGED_IO_DOCS_PATH }} |
| git config user.name $GITHUB_ACTOR |
| git config user.email actions@"$RUNNER_NAME".local |
| git add ${{ env.MANAGED_IO_DOCS_PATH }} |
| git commit --allow-empty -m "Update managed-io.md for release ${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}." |
| git push -f --set-upstream origin $BRANCH_NAME |
| - name: Create beam PR |
| working-directory: beam |
| env: |
| GH_TOKEN: ${{ github.event.inputs.REPO_TOKEN }} |
| PR_TITLE: "Update managed-io.md for release ${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" |
| PR_BODY: "Content generated from release ${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}." |
| run: | |
| gh pr create -t "$PR_TITLE" -b "$PR_BODY" --base master --repo apache/beam |