| name: Multi-Cluster IT | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - "rel/*" | |
| - "rc/*" | |
| paths-ignore: | |
| - "docs/**" | |
| - "site/**" | |
| - "iotdb-client/client-cpp/**" | |
| - ".github/workflows/client-cpp-package.yml" | |
| - ".github/scripts/package-client-cpp-*.sh" | |
| - ".github/workflows/multi-language-client.yml" | |
| - "iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**" #queryengine | |
| pull_request: | |
| branches: | |
| - master | |
| - "rel/*" | |
| - "rc/*" | |
| - "force_ci/**" | |
| paths-ignore: | |
| - "docs/**" | |
| - "site/**" | |
| - "iotdb-client/client-cpp/**" | |
| - ".github/workflows/client-cpp-package.yml" | |
| - ".github/scripts/package-client-cpp-*.sh" | |
| - ".github/workflows/multi-language-client.yml" | |
| - "iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**" #queryengine | |
| # allow manually run the action: | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 | |
| MAVEN_ARGS: --batch-mode --no-transfer-progress | |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} | |
| jobs: | |
| single: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster1: [HighPerformanceMode] | |
| cluster2: [HighPerformanceMode] | |
| os: [ubuntu-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.clusterReadyRetryCount=90 \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT1 \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| # 12 IT classes split across 3 parallel shards to cut the historical ~42 min | |
| # wall clock to ~14 min. See cluster-it-1c1d.yml for the shard pattern. | |
| dual-tree-auto-basic: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster: [HighPerformanceMode] | |
| os: [ubuntu-latest] | |
| shard: [0, 1, 2] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: Build IT shard list | |
| shell: bash | |
| # See cluster-it-1c1d.yml for the shard-list pattern. Write under | |
| # $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file. | |
| run: | | |
| set -euo pipefail | |
| SHARD=${{ matrix.shard }} | |
| TOTAL=3 | |
| grep -rlE --include='*IT.java' '\bMultiClusterIT2DualTreeAutoBasic\b' integration-test/src/test/java \ | |
| | awk -F'/' '{print $NF}' | sed 's/\.java$//' \ | |
| | sort \ | |
| | awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \ | |
| > "$RUNNER_TEMP/it-shard.txt" | |
| echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes" | |
| head -5 "$RUNNER_TEMP/it-shard.txt" | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.clusterReadyRetryCount=90 \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ | |
| -Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \ | |
| -DfailIfNoTests=false \ | |
| -Dfailsafe.failIfNoSpecifiedTests=false \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2DualTreeAutoBasic \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: cluster-log-dual-tree-auto-basic-shard${{ matrix.shard }}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| # 9 IT classes split across 3 parallel shards to cut the historical ~51 min | |
| # wall clock to ~17 min. See cluster-it-1c1d.yml for the shard pattern. | |
| dual-tree-auto-enhanced: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster1: [HighPerformanceMode] | |
| cluster2: [HighPerformanceMode] | |
| os: [ubuntu-latest] | |
| shard: [0, 1, 2] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: Build IT shard list | |
| shell: bash | |
| # See cluster-it-1c1d.yml for the shard-list pattern. Write under | |
| # $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file. | |
| run: | | |
| set -euo pipefail | |
| SHARD=${{ matrix.shard }} | |
| TOTAL=3 | |
| grep -rlE --include='*IT.java' '\bMultiClusterIT2DualTreeAutoEnhanced\b' integration-test/src/test/java \ | |
| | awk -F'/' '{print $NF}' | sed 's/\.java$//' \ | |
| | sort \ | |
| | awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \ | |
| > "$RUNNER_TEMP/it-shard.txt" | |
| echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes" | |
| head -5 "$RUNNER_TEMP/it-shard.txt" | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.clusterReadyRetryCount=90 \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ | |
| -Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \ | |
| -DfailIfNoTests=false \ | |
| -Dfailsafe.failIfNoSpecifiedTests=false \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2DualTreeAutoEnhanced \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: cluster-log-dual-tree-auto-enhanced-shard${{ matrix.shard }}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| # 11 IT classes split across 3 parallel shards to cut the historical ~27 min | |
| # wall clock to ~9 min. See cluster-it-1c1d.yml for the shard pattern. | |
| dual-tree-manual: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster1: [HighPerformanceMode] | |
| cluster2: [HighPerformanceMode] | |
| os: [ubuntu-latest] | |
| shard: [0, 1, 2] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: Build IT shard list | |
| shell: bash | |
| # See cluster-it-1c1d.yml for the shard-list pattern. Write under | |
| # $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file. | |
| run: | | |
| set -euo pipefail | |
| SHARD=${{ matrix.shard }} | |
| TOTAL=3 | |
| grep -rlE --include='*IT.java' '\bMultiClusterIT2DualTreeManual\b' integration-test/src/test/java \ | |
| | awk -F'/' '{print $NF}' | sed 's/\.java$//' \ | |
| | sort \ | |
| | awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \ | |
| > "$RUNNER_TEMP/it-shard.txt" | |
| echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes" | |
| head -5 "$RUNNER_TEMP/it-shard.txt" | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.clusterReadyRetryCount=90 \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ | |
| -Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \ | |
| -DfailIfNoTests=false \ | |
| -Dfailsafe.failIfNoSpecifiedTests=false \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2DualTreeManual \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: cluster-log-dual-tree-manual-shard${{ matrix.shard }}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| dual-table-manual-basic: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster: [HighPerformanceMode] | |
| os: [ubuntu-latest] | |
| shard: [0, 1, 2] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: Build IT shard list | |
| shell: bash | |
| # See cluster-it-1c1d.yml for the shard-list pattern. Write under | |
| # $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file. | |
| run: | | |
| set -euo pipefail | |
| SHARD=${{ matrix.shard }} | |
| TOTAL=3 | |
| grep -rlE --include='*IT.java' '\bMultiClusterIT2DualTableManualBasic\b' integration-test/src/test/java \ | |
| | awk -F'/' '{print $NF}' | sed 's/\.java$//' \ | |
| | sort \ | |
| | awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \ | |
| > "$RUNNER_TEMP/it-shard.txt" | |
| echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes" | |
| head -5 "$RUNNER_TEMP/it-shard.txt" | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.clusterReadyRetryCount=90 \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ | |
| -Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \ | |
| -DfailIfNoTests=false \ | |
| -Dfailsafe.failIfNoSpecifiedTests=false \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2DualTableManualBasic \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: cluster-log-dual-table-manual-basic-shard${{ matrix.shard }}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| # 11 IT classes split across 3 parallel shards to cut the historical ~62 min | |
| # wall clock to ~22 min. See cluster-it-1c1d.yml for the shard pattern. | |
| dual-table-manual-enhanced: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster: [HighPerformanceMode] | |
| os: [ubuntu-latest] | |
| shard: [0, 1, 2] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: Build IT shard list | |
| shell: bash | |
| # See cluster-it-1c1d.yml for the shard-list pattern. Write under | |
| # $RUNNER_TEMP (outside the repo) so Apache RAT doesn't flag the file. | |
| run: | | |
| set -euo pipefail | |
| SHARD=${{ matrix.shard }} | |
| TOTAL=3 | |
| grep -rlE --include='*IT.java' '\bMultiClusterIT2DualTableManualEnhanced\b' integration-test/src/test/java \ | |
| | awk -F'/' '{print $NF}' | sed 's/\.java$//' \ | |
| | sort \ | |
| | awk -v s=$SHARD -v t=$TOTAL 'NR%t==s' \ | |
| > "$RUNNER_TEMP/it-shard.txt" | |
| echo "Shard $SHARD/$TOTAL contains $(wc -l < "$RUNNER_TEMP/it-shard.txt") test classes" | |
| head -5 "$RUNNER_TEMP/it-shard.txt" | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.clusterReadyRetryCount=90 \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ | |
| -Dfailsafe.includesFile="$RUNNER_TEMP/it-shard.txt" \ | |
| -DfailIfNoTests=false \ | |
| -Dfailsafe.failIfNoSpecifiedTests=false \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2DualTableManualEnhanced \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: cluster-log-dual-table-manual-enhanced-shard${{ matrix.shard }}-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| triple: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| java: [ 17 ] | |
| cluster1: [ ScalableSingleNodeMode ] | |
| cluster2: [ ScalableSingleNodeMode ] | |
| cluster3: [ ScalableSingleNodeMode ] | |
| os: [ ubuntu-latest ] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.clusterReadyRetryCount=90 \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }},${{ matrix.cluster3 }} \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT3 \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: cluster-log-triple-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}-${{ matrix.cluster3 }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 |