| # Licensed to the Apache Software Foundation (ASF) under one or more | |
| # contributor license agreements. See the NOTICE file distributed with | |
| # this work for additional information regarding copyright ownership. | |
| # The ASF licenses this file to You under the Apache License, Version 2.0 | |
| # (the "License"); you may not use this file except in compliance with | |
| # the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: PreCommit Flink Container | |
| on: | |
| pull_request: | |
| paths: | |
| - 'model/**' | |
| - 'sdks/python/apache_beam/runners/portability/**' | |
| - 'release/**' | |
| - 'runners/core-java/**' | |
| - 'runners/flink/**' | |
| - 'runners/java-fn-execution/**' | |
| - '.github/trigger_files/beam_PreCommit_Flink_Container.json' | |
| - 'release/trigger_all_tests.json' | |
| push: | |
| tags: [ 'v*' ] | |
| branches: [ 'master', 'release-*' ] | |
| paths: | |
| - 'model/**' | |
| - 'sdks/python/apache_beam/runners/portability/**' | |
| - 'release/**' | |
| - 'runners/core-java/**' | |
| - 'runners/flink/**' | |
| - 'runners/java-fn-execution/**' | |
| schedule: | |
| - cron: '0 */6 * * *' | |
| workflow_dispatch: | |
| # Setting explicit permissions for the action to avoid the default permissions which are `write-all` | |
| permissions: | |
| actions: write | |
| pull-requests: read | |
| checks: read | |
| contents: read | |
| deployments: read | |
| id-token: none | |
| issues: read | |
| discussions: read | |
| packages: read | |
| pages: read | |
| repository-projects: read | |
| security-events: read | |
| statuses: read | |
| # This allows a subsequently queued workflow run to interrupt previous runs | |
| concurrency: | |
| group: '${{ github.workflow }} @ ${{ github.event.pull_request.number || github.sha || github.head_ref || github.ref }}' | |
| cancel-in-progress: true | |
| env: | |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} | |
| GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} | |
| GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} | |
| INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} | |
| INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} | |
| GCLOUD_REGION: us-central1 | |
| CLUSTER_NAME: beam-precommit-flink-container-${{ github.run_id }} | |
| GCS_BUCKET: gs://beam-flink-cluster | |
| FLINK_TASKMANAGER_SLOTS: 1 | |
| DETACHED_MODE: true | |
| HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest | |
| JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink_job_server:latest-flink2.2 | |
| ARTIFACTS_DIR: gs://beam-flink-cluster/beam-precommit-flink-container-${{ github.run_id }} | |
| DOCKER_REGISTRY: gcr.io | |
| DOCKER_REPOSITORY_ROOT: ${{ github.event_name == 'pull_request' && 'gcr.io/apache-beam-testing/beam-sdk-pr' || 'gcr.io/apache-beam-testing/beam-sdk' }} | |
| PYTHON_VERSION: '3.11' | |
| PYTHON_SDK_IMAGE_TAG: latest | |
| jobs: | |
| beam_PreCommit_Flink_Container: | |
| if: | | |
| github.event_name == 'workflow_dispatch' || | |
| github.event_name == 'push' || | |
| github.event_name == 'schedule' || | |
| github.event_name == 'pull_request' | |
| runs-on: [self-hosted, ubuntu-24.04, main] | |
| timeout-minutes: 90 | |
| name: ${{ matrix.job_name }} | |
| strategy: | |
| matrix: | |
| job_name: ["beam_PreCommit_Flink_Container"] | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| persist-credentials: false | |
| - name: Setup repository | |
| uses: ./.github/actions/setup-action | |
| - name: Setup environment | |
| uses: ./.github/actions/setup-environment-action | |
| with: | |
| python-version: default | |
| - name: GCloud Docker credential helper | |
| if: ${{ github.event_name == 'pull_request' }} | |
| run: | | |
| gcloud auth configure-docker ${{ env.DOCKER_REGISTRY }} | |
| - name: Set PYTHON_SDK_IMAGE_TAG unique variable based on timestamp | |
| if: ${{ github.event_name == 'pull_request' }} | |
| run: echo "PYTHON_SDK_IMAGE_TAG=$(date +'%Y%m%d-%H%M%S%N')" >> $GITHUB_ENV | |
| - name: Build and push to registry | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/gradle-command-self-hosted-action | |
| with: | |
| gradle-command: :sdks:python:container:py311:docker | |
| arguments: | | |
| -PpythonVersion=${{ env.PYTHON_VERSION }} \ | |
| -Pdocker-repository-root=${{ env.DOCKER_REPOSITORY_ROOT }} \ | |
| -Pdocker-tag=${{ env.PYTHON_SDK_IMAGE_TAG }} \ | |
| -PuseDockerBuildx \ | |
| -Ppush-containers | |
| - name: Prepare test arguments | |
| uses: ./.github/actions/test-arguments-action | |
| with: | |
| test-type: precommit | |
| test-language: go,python,java | |
| argument-file-paths: | | |
| ${{ github.workspace }}/.github/workflows/flink-tests-pipeline-options/go_Combine_Flink_Batch_small.txt | |
| ${{ github.workspace }}/.github/workflows/flink-tests-pipeline-options/python_Combine_Flink_Batch_small.txt | |
| ${{ github.workspace }}/.github/workflows/flink-tests-pipeline-options/java_Combine_Flink_Batch_small.txt | |
| - name: get current time | |
| run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV | |
| - name: Start Flink with 2 workers | |
| env: | |
| FLINK_NUM_WORKERS: 2 | |
| run: | | |
| cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create | |
| # Run a simple Go Combine load test to verify the Flink container | |
| - name: Run Flink Container Test with Go Combine | |
| timeout-minutes: 10 | |
| uses: ./.github/actions/gradle-command-self-hosted-action | |
| with: | |
| gradle-command: :sdks:go:test:load:run | |
| arguments: | | |
| -PloadTest.mainClass=combine \ | |
| -Prunner=PortableRunner \ | |
| '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_1 }} --job_name=flink-tests-go-${{env.NOW_UTC}}' | |
| # Run a Python Combine load test to verify the Flink container | |
| - name: Run Flink Container Test with Python Combine | |
| timeout-minutes: 20 | |
| uses: ./.github/actions/gradle-command-self-hosted-action | |
| with: | |
| gradle-command: :sdks:python:apache_beam:testing:load_tests:run | |
| arguments: | | |
| -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ | |
| -Prunner=FlinkRunner \ | |
| '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_2 }} --environment_config=${{ env.DOCKER_REPOSITORY_ROOT }}/beam_python${{ env.PYTHON_VERSION }}_sdk:${{ env.PYTHON_SDK_IMAGE_TAG }} --job_name=flink-tests-python-${{env.NOW_UTC}}' | |
| # Run a Java Combine load test to verify the Flink container | |
| - name: Run Flink Container Test with Java Combine | |
| timeout-minutes: 20 | |
| uses: ./.github/actions/gradle-command-self-hosted-action | |
| with: | |
| gradle-command: :sdks:java:testing:load-tests:run | |
| arguments: | | |
| -PloadTest.mainClass=org.apache.beam.sdk.loadtests.CombineLoadTest \ | |
| -Prunner=:runners:flink:2.0 \ | |
| '-PloadTest.args=${{ env.beam_PreCommit_Flink_Container_test_arguments_3 }} --jobName=flink-tests-java11-${{env.NOW_UTC}}' | |
| - name: Teardown Flink | |
| if: always() | |
| run: | | |
| ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete | |
| - name: Cleanup Python SDK Container | |
| if: ${{ always() && github.event_name == 'pull_request' }} | |
| run: | | |
| gcloud container images delete ${DOCKER_REPOSITORY_ROOT}/beam_python${{ env.PYTHON_VERSION }}_sdk:${{ env.PYTHON_SDK_IMAGE_TAG }} --force-delete-tags --quiet |