| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| name: LoadTests Python Combine Flink Streaming |
| |
| on: |
| schedule: |
| - cron: '50 15 * * *' |
| workflow_dispatch: |
| |
| # Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event |
| permissions: |
| actions: write |
| pull-requests: read |
| checks: read |
| contents: read |
| deployments: read |
| id-token: none |
| issues: read |
| discussions: read |
| packages: read |
| pages: read |
| repository-projects: read |
| security-events: read |
| statuses: read |
| |
| # This allows a subsequently queued workflow run to interrupt previous runs |
| concurrency: |
| group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' |
| cancel-in-progress: true |
| |
| env: |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} |
| GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} |
| INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }} |
| INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} |
| GCLOUD_ZONE: us-central1-a |
| CLUSTER_NAME: beam-loadtests-py-cmb-flink-streaming-${{ github.run_id }} |
| GCS_BUCKET: gs://beam-flink-cluster |
| FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.17.0/flink-1.17.0-bin-scala_2.12.tgz |
| HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar |
| FLINK_TASKMANAGER_SLOTS: 1 |
| DETACHED_MODE: true |
| HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest |
| JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.17_job_server:latest |
| ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-py-cmb-flink-streaming-${{ github.run_id }} |
| |
| jobs: |
| beam_LoadTests_Python_Combine_Flink_Streaming: |
| if: | |
| github.event_name == 'workflow_dispatch' || |
| (github.event_name == 'schedule' && github.repository == 'apache/beam') || |
| github.event.comment.body == 'Run Load Tests Python Combine Flink Streaming' |
| runs-on: [self-hosted, ubuntu-20.04, main] |
| timeout-minutes: 80 |
| name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) |
| strategy: |
| matrix: |
| job_name: ["beam_LoadTests_Python_Combine_Flink_Streaming"] |
| job_phrase: ["Run Load Tests Python Combine Flink Streaming"] |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Setup repository |
| uses: ./.github/actions/setup-action |
| with: |
| comment_phrase: ${{ matrix.job_phrase }} |
| github_token: ${{ secrets.GITHUB_TOKEN }} |
| github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) |
| - name: Setup environment |
| uses: ./.github/actions/setup-environment-action |
| with: |
| python-version: default |
| - name: Prepare test arguments |
| uses: ./.github/actions/test-arguments-action |
| with: |
| test-type: load |
| test-language: python |
| argument-file-paths: | |
| ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_1.txt |
| ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_small_Fanout_2.txt |
| # large loads do not work now |
| # ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_4.txt |
| # ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_Combine_Flink_Streaming_2GB_Fanout_8.txt |
| - name: Start Flink with parallelism 16 |
| env: |
| FLINK_NUM_WORKERS: 16 |
| HIGH_MEM_MACHINE: n1-highmem-16 |
| HIGH_MEM_FLINK_PROPS: flink:taskmanager.memory.process.size=16g,flink:taskmanager.memory.flink.size=12g,flink:taskmanager.memory.jvm-overhead.max=4g,flink:jobmanager.memory.process.size=6g,flink:jobmanager.memory.jvm-overhead.max= 2g,flink:jobmanager.memory.flink.size=4g |
| run: | |
| cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create |
| - name: get current time |
| run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV |
| # The env variables are created and populated in the test-arguments-action as "<github.job>_test_arguments_<argument_file_paths_index>" |
| - name: run Load test small Fanout 1 |
| uses: ./.github/actions/gradle-command-self-hosted-action |
| with: |
| gradle-command: :sdks:python:apache_beam:testing:load_tests:run |
| arguments: | |
| -PpythonVersion=3.9 \ |
| -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ |
| -Prunner=PortableRunner \ |
| '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_1 }} --job_name=load-tests-python-flink-streaming-combine-4-${{env.NOW_UTC}}' \ |
| - name: run Load test small Fanout 2 |
| uses: ./.github/actions/gradle-command-self-hosted-action |
| with: |
| gradle-command: :sdks:python:apache_beam:testing:load_tests:run |
| arguments: | |
| -PpythonVersion=3.9 \ |
| -PloadTest.mainClass=apache_beam.testing.load_tests.combine_test \ |
| -Prunner=PortableRunner \ |
| '-PloadTest.args=${{ env.beam_LoadTests_Python_Combine_Flink_Streaming_test_arguments_2 }} --job_name=load-tests-python-flink-streaming-combine-5-${{env.NOW_UTC}}' \ |
| - name: Teardown Flink |
| if: always() |
| run: | |
| ${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete |
| |
| # // TODO(https://github.com/apache/beam/issues/20402). Skipping some cases because they are too slow: |
| # load-tests-python-flink-streaming-combine-1' |
