| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the 'License'); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an 'AS IS' BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| name: Backend |
| |
| on: |
| workflow_call: |
| inputs: |
| TEST_IN_PR: |
| required: false |
| type: string |
| default: 'true' |
| |
| concurrency: |
| group: backend-${{ github.event.pull_request.number || github.ref }} |
| cancel-in-progress: true |
| |
| env: |
| TEST_IN_PR: ${{ inputs.TEST_IN_PR }} |
| |
| jobs: |
| license-header: |
| name: License header |
| runs-on: ubuntu-latest |
| timeout-minutes: 10 |
| steps: |
| - uses: actions/checkout@v4 |
| with: |
| submodules: true |
| - name: Check license header |
| uses: apache/skywalking-eyes@v0.5.0 |
| |
| code-style: |
| name: Code style |
| runs-on: ubuntu-latest |
| timeout-minutes: 10 |
| steps: |
| - uses: actions/checkout@v4 |
| with: |
| submodules: true |
| - name: Check code style |
| run: ./mvnw --batch-mode --quiet --no-snapshot-updates clean spotless:check |
| - name: Check code specification |
| run: ./mvnw -B -T 1 clean test -D"license.skipAddThirdParty"=true -pl seatunnel-ci-tools -am --no-snapshot-updates |
| env: |
| MAVEN_OPTS: -Xmx512m |
| - name: Check for .class files in git |
| run: | |
| echo "Checking for .class files tracked by git..." |
| |
| # Find all .class files tracked by git |
| CLASS_FILES=$(git ls-files '*.class') |
| |
| if [ -n "$CLASS_FILES" ]; then |
| echo "ERROR: The following .class files are tracked by git:" |
| echo "$CLASS_FILES" |
| echo "" |
| echo "Please remove .class files from the repository." |
| echo "These files should not be committed. You can remove them using:" |
| echo " git rm --cached <file>.class" |
| echo " git commit -m 'Remove .class files'" |
| echo "" |
| echo "Also, consider adding '*.class' to .gitignore if not already present." |
| exit 1 |
| else |
| echo "No .class files found in git repository." |
| fi |
| - name: Check connector registration completeness |
| run: | |
| if [ -n "$GITHUB_BASE_REF" ]; then |
| python3 tools/check_connector_registration.py --base-ref "$GITHUB_BASE_REF" |
| else |
| echo "GITHUB_BASE_REF is not set (push event), skipping connector registration check." |
| fi |
| |
| helm-chart-check: |
| name: Check Helm Chart Syntax |
| needs: [ license-header, code-style] |
| runs-on: ubuntu-latest |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Setup Helm |
| run: | |
| if ! command -v helm >/dev/null 2>&1; then |
| curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash |
| fi |
| helm version |
| - name: Lint Chart |
| run: helm lint deploy/kubernetes/seatunnel |
| |
| dead-link: |
| name: Dead links |
| runs-on: ubuntu-latest |
| timeout-minutes: 150 |
| # Temporarily ignore this job to avoid blocking PRs |
| continue-on-error: true |
| steps: |
| - uses: actions/checkout@v4 |
| - run: sudo npm install -g markdown-link-check@3.8.7 |
| - run: | |
| for file in $(find . -name "*.md"); do |
| markdown-link-check -c .dlc.json -q "$file" |
| done |
| |
| sanity-check: |
| name: Sanity check results |
| needs: [ license-header, code-style ] |
| runs-on: ubuntu-latest |
| timeout-minutes: 10 |
| steps: |
| - name: Check results |
| run: | |
| [[ ${{ needs.license-header.result }} == 'success' ]] || exit 1; |
| [[ ${{ needs.code-style.result }} == 'success' ]] || exit 1; |
| |
| changes: |
| runs-on: ubuntu-latest |
| timeout-minutes: 20 |
| outputs: |
| api: ${{ steps.filter.outputs.api }} |
| engine: ${{ steps.filter.outputs.engine }} |
| engine-e2e: ${{ steps.filter.outputs.engine-e2e }} |
| docs: ${{ steps.filter.outputs.docs }} |
| ut-modules: ${{ steps.ut-modules.outputs.modules }} |
| it-modules: ${{ steps.it-modules.outputs.modules }} |
| steps: |
| - uses: actions/checkout@v4 |
| with: |
| fetch-depth: '2000' |
| - name: checkout apache seatunnel dev branch |
| id: git_init |
| run: | |
| /usr/bin/git remote add apache https://github.com/apache/seatunnel |
| /usr/bin/git -c protocol.version=2 fetch --no-tags --prune --no-recurse-submodules --depth=2000 apache +refs/heads/dev*:refs/remotes/apache/dev* +refs/tags/dev*:refs/tags/dev* |
| /usr/bin/git checkout apache/dev |
| /usr/bin/git checkout '${{ github.ref }}' |
| echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT |
| - uses: actions/setup-python@v5 |
| with: |
| python-version: '3.11.0' |
| - name: Check for file changes by python |
| id: filter |
| run: | |
| current_branch='${{ steps.git_init.outputs.branch }}' |
| pip install GitPython |
| workspace="${GITHUB_WORKSPACE}" |
| repository_owner="${GITHUB_REPOSITORY_OWNER}" |
| cv2_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-connectors-v2/**"` |
| true_or_false=${cv2_files%%$'\n'*} |
| file_list=${cv2_files#*$'\n'} |
| echo "cv2=$true_or_false" >> $GITHUB_OUTPUT |
| echo "cv2_files=$file_list" >> $GITHUB_OUTPUT |
| |
| cv2_e2e_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-e2e/seatunnel-connector-v2-e2e/**"` |
| true_or_false=${cv2_e2e_files%%$'\n'*} |
| file_list=${cv2_e2e_files#*$'\n'} |
| echo "cv2-e2e=$true_or_false" >> $GITHUB_OUTPUT |
| echo "cv2-e2e_files=$file_list" >> $GITHUB_OUTPUT |
| |
| engine_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-engine/**"` |
| true_or_false=${engine_files%%$'\n'*} |
| file_list=${engine_files#*$'\n'} |
| echo "engine=$true_or_false" >> $GITHUB_OUTPUT |
| echo "engine_files=$file_list" >> $GITHUB_OUTPUT |
| |
| deleted_poms_files=`python tools/update_modules_check/check_file_updates.py d $workspace apache/dev origin/$current_branch "**/pom.xml"` |
| true_or_false=${deleted_poms_files%%$'\n'*} |
| file_list=${deleted_poms_files#*$'\n'} |
| echo "deleted-poms=$true_or_false" >> $GITHUB_OUTPUT |
| echo "deleted-poms_files=$file_list" >> $GITHUB_OUTPUT |
| |
| doc_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "docs/**"` |
| true_or_false=${doc_files%%$'\n'*} |
| file_list=${doc_files#*$'\n'} |
| echo "docs=$true_or_false" >> $GITHUB_OUTPUT |
| echo "docs_files=$file_list" >> $GITHUB_OUTPUT |
| |
| engine_e2e_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-e2e/seatunnel-engine-e2e/**"` |
| true_or_false=${engine_e2e_files%%$'\n'*} |
| file_list=${engine_e2e_files#*$'\n'} |
| echo "engine-e2e=$true_or_false" >> $GITHUB_OUTPUT |
| echo "engine-e2e_files=$file_list" >> $GITHUB_OUTPUT |
| |
| api_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-api/**" "seatunnel-common/**" "seatunnel-config/**" "seatunnel-core/**" "seatunnel-e2e/seatunnel-e2e-common/**" "seatunnel-formats/**" "seatunnel-plugin-discovery/**" "seatunnel-transforms-v2/**" "seatunnel-translation/**" "seatunnel-e2e/seatunnel-transforms-v2-e2e/**" "pom.xml" "**/workflows/**" "tools/**" "seatunnel-dist/**"` |
| true_or_false=${api_files%%$'\n'*} |
| file_list=${api_files#*$'\n'} |
| if [[ $repository_owner == 'apache' ]];then |
| true_or_false='true' |
| fi |
| echo "api=$true_or_false" >> $GITHUB_OUTPUT |
| echo "api_files=$file_list" >> $GITHUB_OUTPUT |
| |
| - name: Check Connector V2 Update |
| id: cv2-modules |
| if: ${{ steps.filter.outputs.cv2 == 'true' }} |
| run: | |
| update_files='${{ steps.filter.outputs.cv2_files }}' |
| modules=`python tools/update_modules_check/update_modules_check.py cv2 "$update_files"` |
| echo $modules |
| echo "modules=$modules" >> $GITHUB_OUTPUT |
| |
| - name: Check Connector V2 E2E Update |
| id: cv2-e2e-modules |
| if: ${{ steps.filter.outputs.cv2-e2e == 'true' }} |
| run: | |
| update_files='${{ steps.filter.outputs.cv2-e2e_files }}' |
| modules=`python tools/update_modules_check/update_modules_check.py cv2-e2e "$update_files"` |
| echo $modules |
| echo "modules=$modules" >> $GITHUB_OUTPUT |
| |
| - name: Check Engine Update |
| id: engine-modules |
| if: ${{ steps.filter.outputs.engine == 'true' }} |
| run: | |
| update_files='${{ steps.filter.outputs.engine_files }}' |
| modules=`python tools/update_modules_check/update_modules_check.py engine "$update_files"` |
| echo $modules |
| echo "modules=$modules" >> $GITHUB_OUTPUT |
| |
| - name: Check Engine E2E Update |
| id: engine-e2e-modules |
| if: ${{ steps.filter.outputs.engine-e2e == 'true' }} |
| run: | |
| update_files='${{ steps.filter.outputs.engine-e2e_files }}' |
| modules=`python tools/update_modules_check/update_modules_check.py engine-e2e "$update_files"` |
| echo $modules |
| echo "modules=$modules" >> $GITHUB_OUTPUT |
| |
| - name: Check Deleted Modules |
| id: deleted-modules |
| if: ${{ steps.filter.outputs.deleted-poms == 'true' }} |
| run: | |
| update_files='${{ steps.filter.outputs.deleted-poms_files }}' |
| modules=`python tools/update_modules_check/update_modules_check.py delete "$update_files"` |
| echo $modules |
| echo "modules=$modules" >> $GITHUB_OUTPUT |
| |
| - name: Make unit test modules |
| id: ut-modules |
| timeout-minutes: 60 |
| if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '') }} |
| run: | |
| modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}' |
| modules=${modules: 1} |
| pl_modules=`python tools/update_modules_check/update_modules_check.py replace "$modules"` |
| # remove deleted modules |
| delete_modules='${{ steps.deleted-modules.outputs.modules }}' |
| if [[ "zz"$delete_modules != "zz" ]];then |
| pl_modules=`python tools/update_modules_check/update_modules_check.py rm "$pl_modules" "$delete_modules"` |
| fi |
| |
| if [[ "zz"$pl_modules == "zz" ]];then |
| exit 0 |
| fi |
| |
| engine_modules='${{ steps.engine-modules.outputs.modules }}' |
| connector_modules='${{ steps.cv2-modules.outputs.modules }}' |
| if [[ "zz"$connector_modules == "zz" && "zz"$engine_modules != "zz" ]];then |
| # Engine changes already route the full downstream engine test set through engine=true. |
| # Keep the fixed smoke modules as the output without resolving the Maven dependency tree. |
| echo $pl_modules |
| echo "modules=$pl_modules" >> $GITHUB_OUTPUT |
| exit 0 |
| fi |
| |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl $pl_modules > /tmp/sub_module.txt |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` |
| tree_modules="$modules$sub_modules" |
| includes=`python tools/update_modules_check/update_modules_check.py tree "$tree_modules"` |
| ./mvnw -Pci -D"e2e.dependency.skip"=false dependency:tree $includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt |
| build_modules=`python tools/update_modules_check/update_modules_check.py final_ut /tmp/tree_out.txt` |
| if [[ "zz"$build_modules == "zz" ]];then |
| build_modules=$pl_modules |
| fi |
| echo $build_modules |
| echo "modules=$build_modules" >> $GITHUB_OUTPUT |
| |
| - name: Make integration test modules |
| id: it-modules |
| timeout-minutes: 60 |
| if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '' || steps.cv2-e2e-modules.outputs.modules != '' || steps.engine-e2e-modules.outputs.modules != '') }} |
| run: | |
| modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}' |
| modules=${modules: 1} |
| pl_modules=`python tools/update_modules_check/update_modules_check.py replace "$modules"` |
| # remove deleted modules |
| delete_modules='${{ steps.deleted-modules.outputs.modules }}' |
| if [[ "zz"$delete_modules != "zz" ]];then |
| pl_modules=`python tools/update_modules_check/update_modules_check.py rm "$pl_modules" "$delete_modules"` |
| fi |
| |
| if [[ "zz"$pl_modules == "zz" ]];then |
| exit 0 |
| fi |
| |
| engine_modules='${{ steps.engine-modules.outputs.modules }}' |
| connector_modules='${{ steps.cv2-modules.outputs.modules }}' |
| connector_modules="$connector_modules"'${{ steps.cv2-e2e-modules.outputs.modules }}' |
| engine_e2e_modules='${{ steps.engine-e2e-modules.outputs.modules }}' |
| if [[ "zz${connector_modules}${engine_e2e_modules}" == "zz" && "zz"$engine_modules != "zz" ]];then |
| # Engine changes already trigger the broad downstream integration jobs through engine=true. |
| # Avoid a second Maven dependency tree pass when no connector or engine-e2e module changed. |
| echo $pl_modules |
| echo "modules=$pl_modules" >> $GITHUB_OUTPUT |
| exit 0 |
| fi |
| |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl $pl_modules > /tmp/sub_module.txt |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` |
| tree_modules="$modules$sub_modules" |
| includes=`python tools/update_modules_check/update_modules_check.py tree "$tree_modules"` |
| ./mvnw -Pci -D"e2e.dependency.skip"=false dependency:tree $includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt |
| build_modules=`python tools/update_modules_check/update_modules_check.py final_it /tmp/tree_out.txt` |
| echo $build_modules |
| echo "modules=$build_modules" >> $GITHUB_OUTPUT |
| |
| dependency-license: |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| name: Dependency licenses |
| needs: [ changes, sanity-check ] |
| runs-on: ubuntu-latest |
| timeout-minutes: 60 |
| steps: |
| - uses: actions/checkout@v4 |
| with: |
| submodules: true |
| - uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: '8' |
| cache: 'maven' |
| - name: Install |
| uses: nick-fields/retry@v3 |
| with: |
| timeout_minutes: 40 |
| max_attempts: 3 |
| retry_on: error |
| command: | |
| ./mvnw -B install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"license.skipAddThirdParty" -D"skip.ui"=true |
| - name: Check Dependencies Licenses |
| run: tools/dependencies/checkLicense.sh |
| |
| document: |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.docs == 'true' |
| needs: [ changes, sanity-check ] |
| name: Build website |
| runs-on: ubuntu-latest |
| timeout-minutes: 90 |
| steps: |
| - name: Checkout PR |
| uses: actions/checkout@v4 |
| with: |
| path: seatunnel-pr |
| - name: Checkout website repo |
| uses: actions/checkout@v4 |
| with: |
| repository: apache/seatunnel-website |
| path: seatunnel-website |
| - name: Sync PR changes to website |
| run: | |
| bash seatunnel-pr/tools/documents/sync.sh seatunnel-pr seatunnel-website |
| - uses: actions/setup-node@v4 |
| with: |
| node-version: 18.20.7 |
| - name: Run docusaurus build |
| env: |
| GITHUB_TOKEN: ${{ github.token }} |
| GH_TOKEN: ${{ github.token }} |
| run: | |
| cd seatunnel-website |
| npm set strict-ssl false |
| npm install |
| npm run build |
| |
| seatunnel-ui: |
| if: needs.changes.outputs.api == 'true' |
| needs: [ changes, sanity-check ] |
| name: Build SeaTunnel UI |
| runs-on: ubuntu-latest |
| timeout-minutes: 60 |
| steps: |
| - name: Checkout PR |
| uses: actions/checkout@v4 |
| - uses: actions/setup-node@v4 |
| with: |
| node-version: 20.x |
| - name: Install Dependencies and Check Code Style |
| run: | |
| cd seatunnel-engine/seatunnel-engine-ui/ |
| npm install |
| npm run lint |
| - name: Run unit tests |
| run: | |
| cd seatunnel-engine/seatunnel-engine-ui/ |
| npm run test:unit |
| - name: Build SeaTunnel UI |
| run: | |
| cd seatunnel-engine/seatunnel-engine-ui/ |
| npm run build |
| |
| unit-test: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api == 'false' && needs.changes.outputs.ut-modules != '') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest', 'windows-latest' ] |
| timeout-minutes: 90 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: run all modules unit test |
| run: | |
| ./mvnw -B -T 1 clean verify -DskipUT=false -DskipIT=true -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| updated-modules-integration-test-part-1: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '' |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 180 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run updated modules integration test (part-1) |
| run: | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 0` |
| if [ ! -z $sub_modules ]; then |
| echo $sub_modules |
| ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci |
| else |
| echo "sub modules is empty, skipping" |
| fi |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| updated-modules-integration-test-part-2: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '' |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 180 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run updated modules integration test (part-2) |
| run: | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 1` |
| if [ ! -z $sub_modules ]; then |
| echo $sub_modules |
| ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci |
| else |
| echo "sub modules is empty, skipping" |
| fi |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| updated-modules-integration-test-part-3: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '' |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 210 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run updated modules integration test (part-3) |
| run: | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 2` |
| if [ ! -z $sub_modules ]; then |
| echo $sub_modules |
| ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci |
| else |
| echo "sub modules is empty, skipping" |
| fi |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| updated-modules-integration-test-part-4: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '' |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 200 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run updated modules integration test (part-4) |
| run: | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 3` |
| if [ ! -z $sub_modules ]; then |
| echo $sub_modules |
| ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci |
| else |
| echo "sub modules is empty, skipping" |
| fi |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| updated-modules-integration-test-part-5: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '' |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 180 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run updated modules integration test (part-5) |
| run: | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 4` |
| if [ ! -z $sub_modules ]; then |
| echo $sub_modules |
| ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci |
| else |
| echo "sub modules is empty, skipping" |
| fi |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| updated-modules-integration-test-part-6: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '' |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 210 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run updated modules integration test (part-6) |
| run: | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 5` |
| if [ ! -z $sub_modules ]; then |
| echo $sub_modules |
| ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci |
| else |
| echo "sub modules is empty, skipping" |
| fi |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| updated-modules-integration-test-part-7: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '' |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run updated modules integration test (part-7) |
| run: | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 6` |
| if [ ! -z $sub_modules ]; then |
| echo $sub_modules |
| ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci |
| else |
| echo "sub modules is empty, skipping" |
| fi |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| updated-modules-integration-test-part-8: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != '' |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run updated modules integration test (part-8) |
| run: | |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 7` |
| if [ ! -z $sub_modules ]; then |
| echo $sub_modules |
| ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci |
| else |
| echo "sub modules is empty, skipping" |
| fi |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| engine-v2-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || needs.changes.outputs.engine-e2e == 'true' |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 150 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run seatunnel zeta integration test |
| run: | |
| ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-seatunnel-e2e-base,:connector-console-seatunnel-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| engine-k8s-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || contains(needs.changes.outputs.it-modules, 'seatunnel-engine-k8s-e2e') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 45 |
| steps: |
| - name: install k8s |
| run: | |
| curl -sfL https://get.k3s.io | K3S_KUBECONFIG_MODE=777 sh -s - --docker |
| cat /etc/rancher/k3s/k3s.yaml |
| mkdir -p ~/.kube |
| cp /etc/rancher/k3s/k3s.yaml ~/.kube/config |
| env: |
| KUBECONFIG: /etc/rancher/k3s/k3s.yaml |
| - uses: actions/checkout@v4 |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: run seatunnel zeta on k8s test |
| run: | |
| ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :seatunnel-engine-k8s-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| KUBECONFIG: /etc/rancher/k3s/k3s.yaml |
| |
| transform-v2-it-part-1: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run transform-v2 integration test (part-1) |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-1 -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| transform-v2-it-part-2: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 150 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run transform-v2 integration test (part-2) |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-2 -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| all-connectors-it-1: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 210 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run connector-v2 integration test (part-1) |
| run: | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 0` |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| all-connectors-it-2: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 180 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run connector-v2 integration test (part-2) |
| run: | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 1` |
| # Keep the longest Iceberg and HBase suites in a dedicated shard so this hosted runner |
| # does not lose heartbeat before the rest of the part-2 modules finish. |
| # Handle both first-position (no leading comma) and mid/last-position (leading comma). |
| run_it_modules=${run_it_modules//:connector-iceberg-e2e,/} |
| run_it_modules=${run_it_modules//,:connector-iceberg-e2e/} |
| run_it_modules=${run_it_modules//:connector-hbase-e2e,/} |
| run_it_modules=${run_it_modules//,:connector-hbase-e2e/} |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| all-connectors-it-3: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 210 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run connector-v2 integration test (part-3) |
| run: | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 2` |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| all-connectors-it-4: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 270 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run connector-v2 integration test (part-4) |
| run: | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 3` |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| all-connectors-it-5: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 270 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run connector-v2 integration test (part-5) |
| run: | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 4` |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| all-connectors-it-6: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 210 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run connector-v2 integration test (part-6) |
| run: | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 5` |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| all-connectors-it-7: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 210 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run connector-v2 integration test (part-7) |
| run: | |
| ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt |
| sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt` |
| run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 6` |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| all-connectors-it-8: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 210 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run connector-v2 integration test (part-8) |
| run: | |
| # These two suites dominated part-2 and triggered hosted-runner heartbeat loss. |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-iceberg-e2e,:connector-hbase-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| jdbc-connectors-it-part-1: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run jdbc connectors integration test (part-1) |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-1 -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| jdbc-connectors-it-part-2: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run jdbc connectors integration test (part-2) |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-2 -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| jdbc-connectors-it-part-3: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run jdbc connectors integration test (part-3) |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-3 -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| jdbc-connectors-it-part-4: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run jdbc connectors integration test (part-4) |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-4 -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| jdbc-connectors-it-part-5: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run jdbc connectors integration test (part-5) |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-5 -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| jdbc-connectors-it-part-6: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run jdbc connectors integration test (part-6) |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-6 -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| jdbc-connectors-it-part-7: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run jdbc connectors integration test (part-7) |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-7 -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| jdbc-connectors-it-ddl: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' |
| runs-on: ${{ matrix.os }} |
| env: |
| RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }} |
| RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 180 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run jdbc connectors integration test (sink ddl) |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-ddl -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| kudu-connector-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-kudu-e2e') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| # Kudu E2E expands each @TestTemplate case across several PR test containers. |
| timeout-minutes: 90 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run kudu connector integration test |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-kudu-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| amazonSqs-connector-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-amazonsqs-e2e') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run amazonsqs connector integration test |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-amazonsqs-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| kafka-connector-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-kafka-e2e') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 210 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run kafka connector integration test |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-kafka-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| rocketmq-connector-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-rocketmq-e2e') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run rocket connector integration test |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-rocketmq-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| |
| doris-connector-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-doris-e2e') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 180 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run doris connector integration test |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-doris-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| paimon-connector-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-paimon-e2e') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 180 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run paimon connector integration test |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-paimon-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| oracle-cdc-connector-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-cdc-oracle-e2e') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 210 |
| steps: |
| - name: Checkout repository |
| uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run oracle cdc connector integration test |
| uses: nick-fields/retry@v3 |
| with: |
| timeout_seconds: 9000 |
| max_attempts: 3 |
| retry_on: error |
| command: | |
| echo 'running oracle cdc connector integration test...' && \ |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-cdc-oracle-e2e -am -Pci |
| |
| connector-file-local-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-file-local-e2e') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run file local connector integration test |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-file-local-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| connector-file-sftp-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-file-sftp-e2e') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 120 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run file sftp connector integration test |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-file-sftp-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| connector-redis-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-redis-e2e') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 210 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run redis connector integration test |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-redis-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
| |
| connector-sensorsdata-it: |
| needs: [ changes, sanity-check ] |
| if: needs.changes.outputs.api == 'true' || contains(needs.changes.outputs.it-modules, 'connector-sensorsdata-e2e') |
| runs-on: ${{ matrix.os }} |
| strategy: |
| matrix: |
| java: [ '8', '11' ] |
| os: [ 'ubuntu-latest' ] |
| timeout-minutes: 180 |
| steps: |
| - uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| java-version: ${{ matrix.java }} |
| distribution: 'temurin' |
| cache: 'maven' |
| - name: free disk space |
| run: tools/github/free_disk_space.sh |
| - name: run sensorsdata connector integration test |
| run: | |
| ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-sensorsdata-e2e -am -Pci |
| env: |
| MAVEN_OPTS: -Xmx4096m |
