name: Daily IT

on:
  schedule:
    # Run at UTC 19:00 every day (CST 03:00 AM)
    - cron: "0 19 * * *"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
  MAVEN_ARGS: --batch-mode --no-transfer-progress
  DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}

jobs:
  Simple:
    strategy:
      fail-fast: false
      max-parallel: 20
      matrix:
        java: [8, 17]
    runs-on: [self-hosted, iotdb]
    #      group: self-hosted
    #      labels: iotdb
    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: ${{ matrix.java }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: IT/UT Test
        shell: bash
        run: |
          mvn clean verify \
          -P with-integration-tests \
          -DskipUTs \
          -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \
          -pl integration-test \
          -am -PDailyIT
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: cluster-log-java${{ matrix.java }}-${{ runner.os }}
          path: integration-test/target/cluster-logs
          retention-days: 3
  SingleRegionTableModel:
    strategy:
      fail-fast: false
      max-parallel: 15
      matrix:
        java: [8, 17]
    runs-on: [self-hosted, iotdb]
    #      group: self-hosted
    #      labels: iotdb
    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: ${{ matrix.java }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: IT/UT Test
        shell: bash
        # we do not compile client-cpp for saving time, it is tested in client.yml
        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
        run: |
          mvn clean verify \
          -P with-integration-tests \
          -DskipUTs \
          -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 -DintegrationTest.dataRegionPerDataNode=1 \
          -pl integration-test \
          -am -PTableSimpleIT
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }}
          path: integration-test/target/cluster-logs
          retention-days: 3
  PipeSingle:
    strategy:
      fail-fast: false
      max-parallel: 15
      matrix:
        java: [17]
        # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
        cluster1:
          [
            LightWeightStandaloneMode,
            ScalableSingleNodeMode,
            HighPerformanceMode,
            IoTConsensusV2BatchMode,
            IoTConsensusV2StreamMode,
          ]
        cluster2:
          [
            LightWeightStandaloneMode,
            ScalableSingleNodeMode,
            HighPerformanceMode,
          ]
        os: [ubuntu-latest]
        exclude:
          - cluster1: LightWeightStandaloneMode
            cluster2: LightWeightStandaloneMode
          - cluster1: LightWeightStandaloneMode
            cluster2: ScalableSingleNodeMode
          - cluster1: ScalableSingleNodeMode
            cluster2: LightWeightStandaloneMode
          - cluster1: ScalableSingleNodeMode
            cluster2: HighPerformanceMode
          - cluster1: HighPerformanceMode
            cluster2: LightWeightStandaloneMode
          - cluster1: HighPerformanceMode
            cluster2: HighPerformanceMode
          - cluster1: IoTConsensusV2BatchMode
            cluster2: LightWeightStandaloneMode
          - cluster1: IoTConsensusV2BatchMode
            cluster2: HighPerformanceMode
          - cluster1: IoTConsensusV2StreamMode
            cluster2: LightWeightStandaloneMode
          - cluster1: IoTConsensusV2StreamMode
            cluster2: HighPerformanceMode
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: ${{ matrix.java }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache Maven packages
        uses: actions/cache@v5
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-
      - name: Sleep for a random duration between 0 and 10000 milliseconds
        run: |
          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
      - name: IT Test
        shell: bash
        # we do not compile client-cpp for saving time, it is tested in client.yml
        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
        run: |
          retry() {
            local -i max_attempts=3
            local -i attempt=1
            local -i retry_sleep=5
            local test_output

            while [ $attempt -le $max_attempts ]; do
              mvn clean verify \
              -P with-integration-tests \
              -DskipUTs \
              -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
              -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
              -pl integration-test \
              -am -PMultiClusterIT1 \
              -ntp >> ~/run-tests-$attempt.log && return 0
              test_output=$(cat ~/run-tests-$attempt.log) 

              echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="          
              echo "$test_output"
              echo "==================== END: ~/run-tests-$attempt.log ======================"

              if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
                echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
              fi

              if echo "$test_output" | grep -q "Could not transfer artifact"; then
                if [ $attempt -lt $max_attempts ]; then
                  echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
                  sleep $retry_sleep
                  attempt=$((attempt + 1))
                else
                  echo "Test failed after $max_attempts attempts due to artifact transfer issue."
                  echo "Treating this as a success because the issue is likely transient."
                  return 0
                fi
              elif [ $? -ne 0 ]; then
                echo "Test failed with a different error."
                return 1
              else
                echo "Tests passed"
                return 0
              fi
            done
          }
          retry
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
          path: integration-test/target/cluster-logs
          retention-days: 30
  PipeDualTreeAutoBasic:
    strategy:
      fail-fast: false
      max-parallel: 15
      matrix:
        java: [17]
        # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
        cluster:
          [
            LightWeightStandaloneMode,
            ScalableSingleNodeMode,
            HighPerformanceMode,
            IoTConsensusV2BatchMode,
            IoTConsensusV2StreamMode,
          ]
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: ${{ matrix.java }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache Maven packages
        uses: actions/cache@v5
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-
      - name: Sleep for a random duration between 0 and 10000 milliseconds
        run: |
          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
      - name: IT Test
        shell: bash
        # we do not compile client-cpp for saving time, it is tested in client.yml
        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
        run: |
          retry() {
            local -i max_attempts=3
            local -i attempt=1
            local -i retry_sleep=5
            local test_output

            while [ $attempt -le $max_attempts ]; do
              mvn clean verify \
              -P with-integration-tests \
              -DskipUTs \
              -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
              -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
              -pl integration-test \
              -am -PMultiClusterIT2DualTreeAutoBasic \
              -ntp >> ~/run-tests-$attempt.log && return 0
              test_output=$(cat ~/run-tests-$attempt.log) 

              echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="          
              echo "$test_output"
              echo "==================== END: ~/run-tests-$attempt.log ======================"

              if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
                echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
              fi

              if echo "$test_output" | grep -q "Could not transfer artifact"; then
                if [ $attempt -lt $max_attempts ]; then
                  echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
                  sleep $retry_sleep
                  attempt=$((attempt + 1))
                else
                  echo "Test failed after $max_attempts attempts due to artifact transfer issue."
                  echo "Treating this as a success because the issue is likely transient."
                  return 0
                fi
              elif [ $? -ne 0 ]; then
                echo "Test failed with a different error."
                return 1
              else
                echo "Tests passed"
                return 0
              fi
            done
          }
          retry
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
          path: integration-test/target/cluster-logs
          retention-days: 30
  PipeDualTreeAutoEnhanced:
    strategy:
      fail-fast: false
      max-parallel: 15
      matrix:
        java: [17]
        # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
        cluster1:
          [
            LightWeightStandaloneMode,
            ScalableSingleNodeMode,
            HighPerformanceMode,
            IoTConsensusV2BatchMode,
            IoTConsensusV2StreamMode,
          ]
        cluster2:
          [
            LightWeightStandaloneMode,
            ScalableSingleNodeMode,
            HighPerformanceMode,
          ]
        os: [ubuntu-latest]
        exclude:
          - cluster1: LightWeightStandaloneMode
            cluster2: LightWeightStandaloneMode
          - cluster1: LightWeightStandaloneMode
            cluster2: ScalableSingleNodeMode
          - cluster1: ScalableSingleNodeMode
            cluster2: LightWeightStandaloneMode
          - cluster1: ScalableSingleNodeMode
            cluster2: HighPerformanceMode
          - cluster1: HighPerformanceMode
            cluster2: LightWeightStandaloneMode
          - cluster1: HighPerformanceMode
            cluster2: HighPerformanceMode
          - cluster1: IoTConsensusV2BatchMode
            cluster2: LightWeightStandaloneMode
          - cluster1: IoTConsensusV2BatchMode
            cluster2: HighPerformanceMode
          - cluster1: IoTConsensusV2StreamMode
            cluster2: LightWeightStandaloneMode
          - cluster1: IoTConsensusV2StreamMode
            cluster2: HighPerformanceMode
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: ${{ matrix.java }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache Maven packages
        uses: actions/cache@v5
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-
      - name: Sleep for a random duration between 0 and 10000 milliseconds
        run: |
          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
      - name: IT Test
        shell: bash
        # we do not compile client-cpp for saving time, it is tested in client.yml
        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
        run: |
          retry() {
            local -i max_attempts=3
            local -i attempt=1
            local -i retry_sleep=5
            local test_output

            while [ $attempt -le $max_attempts ]; do
              mvn clean verify \
              -P with-integration-tests \
              -DskipUTs \
              -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
              -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
              -pl integration-test \
              -am -PMultiClusterIT2DualTreeAutoEnhanced \
              -ntp >> ~/run-tests-$attempt.log && return 0
              test_output=$(cat ~/run-tests-$attempt.log) 

              echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="          
              echo "$test_output"
              echo "==================== END: ~/run-tests-$attempt.log ======================"

              if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
                echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
              fi

              if echo "$test_output" | grep -q "Could not transfer artifact"; then
                if [ $attempt -lt $max_attempts ]; then
                  echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
                  sleep $retry_sleep
                  attempt=$((attempt + 1))
                else
                  echo "Test failed after $max_attempts attempts due to artifact transfer issue."
                  echo "Treating this as a success because the issue is likely transient."
                  return 0
                fi
              elif [ $? -ne 0 ]; then
                echo "Test failed with a different error."
                return 1
              else
                echo "Tests passed"
                return 0
              fi
            done
          }
          retry
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
          path: integration-test/target/cluster-logs
          retention-days: 30
  PipeDualTreeManual:
    strategy:
      fail-fast: false
      max-parallel: 15
      matrix:
        java: [17]
        # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
        cluster1:
          [
            LightWeightStandaloneMode,
            ScalableSingleNodeMode,
            HighPerformanceMode,
            IoTConsensusV2BatchMode,
            IoTConsensusV2StreamMode,
          ]
        cluster2:
          [
            LightWeightStandaloneMode,
            ScalableSingleNodeMode,
            HighPerformanceMode,
          ]
        os: [ubuntu-latest]
        exclude:
          - cluster1: LightWeightStandaloneMode
            cluster2: LightWeightStandaloneMode
          - cluster1: LightWeightStandaloneMode
            cluster2: ScalableSingleNodeMode
          - cluster1: ScalableSingleNodeMode
            cluster2: LightWeightStandaloneMode
          - cluster1: ScalableSingleNodeMode
            cluster2: HighPerformanceMode
          - cluster1: HighPerformanceMode
            cluster2: LightWeightStandaloneMode
          - cluster1: HighPerformanceMode
            cluster2: HighPerformanceMode
          - cluster1: IoTConsensusV2BatchMode
            cluster2: LightWeightStandaloneMode
          - cluster1: IoTConsensusV2BatchMode
            cluster2: HighPerformanceMode
          - cluster1: IoTConsensusV2StreamMode
            cluster2: LightWeightStandaloneMode
          - cluster1: IoTConsensusV2StreamMode
            cluster2: HighPerformanceMode
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: ${{ matrix.java }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache Maven packages
        uses: actions/cache@v5
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-
      - name: Sleep for a random duration between 0 and 10000 milliseconds
        run: |
          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
      - name: IT Test
        shell: bash
        # we do not compile client-cpp for saving time, it is tested in client.yml
        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
        run: |
          retry() {
            local -i max_attempts=3
            local -i attempt=1
            local -i retry_sleep=5
            local test_output

            while [ $attempt -le $max_attempts ]; do
              mvn clean verify \
              -P with-integration-tests \
              -DskipUTs \
              -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
              -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
              -pl integration-test \
              -am -PMultiClusterIT2DualTreeManual \
              -ntp >> ~/run-tests-$attempt.log && return 0
              test_output=$(cat ~/run-tests-$attempt.log) 

              echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="          
              echo "$test_output"
              echo "==================== END: ~/run-tests-$attempt.log ======================"

              if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
                echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
              fi

              if echo "$test_output" | grep -q "Could not transfer artifact"; then
                if [ $attempt -lt $max_attempts ]; then
                  echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
                  sleep $retry_sleep
                  attempt=$((attempt + 1))
                else
                  echo "Test failed after $max_attempts attempts due to artifact transfer issue."
                  echo "Treating this as a success because the issue is likely transient."
                  return 0
                fi
              elif [ $? -ne 0 ]; then
                echo "Test failed with a different error."
                return 1
              else
                echo "Tests passed"
                return 0
              fi
            done
          }
          retry
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
          path: integration-test/target/cluster-logs
          retention-days: 30
  SubscriptionTreeArchVerification:
    strategy:
      fail-fast: false
      max-parallel: 15
      matrix:
        java: [17]
        # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
        cluster1:
          [
            ScalableSingleNodeMode,
            IoTConsensusV2BatchMode,
            IoTConsensusV2StreamMode,
          ]
        cluster2: [ScalableSingleNodeMode]
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: ${{ matrix.java }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache Maven packages
        uses: actions/cache@v5
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-
      - name: Sleep for a random duration between 0 and 10000 milliseconds
        run: |
          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
      - name: IT Test
        shell: bash
        # we do not compile client-cpp for saving time, it is tested in client.yml
        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
        run: |
          retry() {
            local -i max_attempts=3
            local -i attempt=1
            local -i retry_sleep=5
            local test_output

            while [ $attempt -le $max_attempts ]; do
              mvn clean verify \
              -P with-integration-tests \
              -DskipUTs \
              -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
              -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
              -pl integration-test \
              -am -PMultiClusterIT2SubscriptionTreeArchVerification \
              -ntp >> ~/run-tests-$attempt.log && return 0
              test_output=$(cat ~/run-tests-$attempt.log) 

              echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="          
              echo "$test_output"
              echo "==================== END: ~/run-tests-$attempt.log ======================"

              if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
                echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
              fi

              if echo "$test_output" | grep -q "Could not transfer artifact"; then
                if [ $attempt -lt $max_attempts ]; then
                  echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
                  sleep $retry_sleep
                  attempt=$((attempt + 1))
                else
                  echo "Test failed after $max_attempts attempts due to artifact transfer issue."
                  echo "Treating this as a success because the issue is likely transient."
                  return 0
                fi
              elif [ $? -ne 0 ]; then
                echo "Test failed with a different error."
                return 1
              else
                echo "Tests passed"
                return 0
              fi
            done
          }
          retry
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: cluster-log-subscription-tree-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
          path: integration-test/target/cluster-logs
          retention-days: 30
  SubscriptionTableArchVerification:
    strategy:
      fail-fast: false
      max-parallel: 15
      matrix:
        java: [17]
        # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
        cluster1: [ScalableSingleNodeMode]
        cluster2: [ScalableSingleNodeMode]
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: ${{ matrix.java }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache Maven packages
        uses: actions/cache@v5
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-
      - name: Sleep for a random duration between 0 and 10000 milliseconds
        run: |
          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
      - name: IT Test
        shell: bash
        # we do not compile client-cpp for saving time, it is tested in client.yml
        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
        run: |
          retry() {
            local -i max_attempts=3
            local -i attempt=1
            local -i retry_sleep=5
            local test_output

            while [ $attempt -le $max_attempts ]; do
              mvn clean verify \
              -P with-integration-tests \
              -DskipUTs \
              -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
              -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
              -pl integration-test \
              -am -PMultiClusterIT2SubscriptionTableArchVerification \
              -ntp >> ~/run-tests-$attempt.log && return 0
              test_output=$(cat ~/run-tests-$attempt.log) 

              echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="          
              echo "$test_output"
              echo "==================== END: ~/run-tests-$attempt.log ======================"

              if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
                echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
              fi

              if echo "$test_output" | grep -q "Could not transfer artifact"; then
                if [ $attempt -lt $max_attempts ]; then
                  echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
                  sleep $retry_sleep
                  attempt=$((attempt + 1))
                else
                  echo "Test failed after $max_attempts attempts due to artifact transfer issue."
                  echo "Treating this as a success because the issue is likely transient."
                  return 0
                fi
              elif [ $? -ne 0 ]; then
                echo "Test failed with a different error."
                return 1
              else
                echo "Tests passed"
                return 0
              fi
            done
          }
          retry
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: cluster-log-subscription-table-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
          path: integration-test/target/cluster-logs
          retention-days: 30
  SubscriptionTreeRegressionConsumer:
    strategy:
      fail-fast: false
      max-parallel: 15
      matrix:
        java: [17]
        # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal
        cluster1:
          [
            ScalableSingleNodeMode,
            IoTConsensusV2BatchMode,
            IoTConsensusV2StreamMode,
          ]
        cluster2: [ScalableSingleNodeMode]
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: ${{ matrix.java }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache Maven packages
        uses: actions/cache@v5
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-
      - name: Sleep for a random duration between 0 and 10000 milliseconds
        run: |
          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
      - name: IT Test
        shell: bash
        # we do not compile client-cpp for saving time, it is tested in client.yml
        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
        run: |
          retry() {
            local -i max_attempts=3
            local -i attempt=1
            local -i retry_sleep=5
            local test_output

            while [ $attempt -le $max_attempts ]; do
              mvn clean verify \
              -P with-integration-tests \
              -DskipUTs \
              -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
              -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
              -pl integration-test \
              -am -PMultiClusterIT2SubscriptionTreeRegressionConsumer \
              -ntp >> ~/run-tests-$attempt.log && return 0
              test_output=$(cat ~/run-tests-$attempt.log) 

              echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="          
              echo "$test_output"
              echo "==================== END: ~/run-tests-$attempt.log ======================"

              if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
                echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
              fi

              if echo "$test_output" | grep -q "Could not transfer artifact"; then
                if [ $attempt -lt $max_attempts ]; then
                  echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
                  sleep $retry_sleep
                  attempt=$((attempt + 1))
                else
                  echo "Test failed after $max_attempts attempts due to artifact transfer issue."
                  echo "Treating this as a success because the issue is likely transient."
                  return 0
                fi
              elif [ $? -ne 0 ]; then
                echo "Test failed with a different error."
                return 1
              else
                echo "Tests passed"
                return 0
              fi
            done
          }
          retry
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: cluster-log-subscription-tree-regression-consumer-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
          path: integration-test/target/cluster-logs
          retention-days: 30
  SubscriptionTreeRegressionMisc:
    strategy:
      fail-fast: false
      max-parallel: 15
      matrix:
        java: [17]
        # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal
        cluster1:
          [
            ScalableSingleNodeMode,
            IoTConsensusV2BatchMode,
            IoTConsensusV2StreamMode,
          ]
        cluster2: [ScalableSingleNodeMode]
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: ${{ matrix.java }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache Maven packages
        uses: actions/cache@v5
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-
      - name: Sleep for a random duration between 0 and 10000 milliseconds
        run: |
          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
      - name: IT Test
        shell: bash
        # we do not compile client-cpp for saving time, it is tested in client.yml
        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
        run: |
          retry() {
            local -i max_attempts=3
            local -i attempt=1
            local -i retry_sleep=5
            local test_output

            while [ $attempt -le $max_attempts ]; do
              mvn clean verify \
              -P with-integration-tests \
              -DskipUTs \
              -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
              -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
              -pl integration-test \
              -am -PMultiClusterIT2SubscriptionTreeRegressionMisc \
              -ntp >> ~/run-tests-$attempt.log && return 0
              test_output=$(cat ~/run-tests-$attempt.log) 

              echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="          
              echo "$test_output"
              echo "==================== END: ~/run-tests-$attempt.log ======================"

              if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
                echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
              fi

              if echo "$test_output" | grep -q "Could not transfer artifact"; then
                if [ $attempt -lt $max_attempts ]; then
                  echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
                  sleep $retry_sleep
                  attempt=$((attempt + 1))
                else
                  echo "Test failed after $max_attempts attempts due to artifact transfer issue."
                  echo "Treating this as a success because the issue is likely transient."
                  return 0
                fi
              elif [ $? -ne 0 ]; then
                echo "Test failed with a different error."
                return 1
              else
                echo "Tests passed"
                return 0
              fi
            done
          }
          retry
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
          path: integration-test/target/cluster-logs
          retention-days: 30
  PipeDualTableManualBasic:
    strategy:
      fail-fast: false
      max-parallel: 15
      matrix:
        java: [17]
        # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
        cluster:
          [
            LightWeightStandaloneMode,
            ScalableSingleNodeMode,
            HighPerformanceMode,
            IoTConsensusV2BatchMode,
            IoTConsensusV2StreamMode,
          ]
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: ${{ matrix.java }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache Maven packages
        uses: actions/cache@v5
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-
      - name: Sleep for a random duration between 0 and 10000 milliseconds
        run: |
          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
      - name: IT Test
        shell: bash
        # we do not compile client-cpp for saving time, it is tested in client.yml
        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
        run: |
          retry() {
            local -i max_attempts=3
            local -i attempt=1
            local -i retry_sleep=5
            local test_output

            while [ $attempt -le $max_attempts ]; do
              mvn clean verify \
              -P with-integration-tests \
              -DskipUTs \
              -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
              -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
              -pl integration-test \
              -am -PMultiClusterIT2DualTableManualBasic \
              -ntp >> ~/run-tests-$attempt.log && return 0
              test_output=$(cat ~/run-tests-$attempt.log) 

              echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="          
              echo "$test_output"
              echo "==================== END: ~/run-tests-$attempt.log ======================"

              if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
                echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
              fi

              if echo "$test_output" | grep -q "Could not transfer artifact"; then
                if [ $attempt -lt $max_attempts ]; then
                  echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
                  sleep $retry_sleep
                  attempt=$((attempt + 1))
                else
                  echo "Test failed after $max_attempts attempts due to artifact transfer issue."
                  echo "Treating this as a success because the issue is likely transient."
                  return 0
                fi
              elif [ $? -ne 0 ]; then
                echo "Test failed with a different error."
                return 1
              else
                echo "Tests passed"
                return 0
              fi
            done
          }
          retry
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
          path: integration-test/target/cluster-logs
          retention-days: 30
  PipeDualTableManualEnhanced:
    strategy:
      fail-fast: false
      max-parallel: 15
      matrix:
        java: [17]
        # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
        cluster:
          [
            LightWeightStandaloneMode,
            ScalableSingleNodeMode,
            HighPerformanceMode,
            IoTConsensusV2BatchMode,
            IoTConsensusV2StreamMode,
          ]
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v5
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v5
        with:
          distribution: corretto
          java-version: ${{ matrix.java }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache Maven packages
        uses: actions/cache@v5
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-
      - name: Sleep for a random duration between 0 and 10000 milliseconds
        run: |
          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
      - name: IT Test
        shell: bash
        # we do not compile client-cpp for saving time, it is tested in client.yml
        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
        run: |
          retry() {
            local -i max_attempts=3
            local -i attempt=1
            local -i retry_sleep=5
            local test_output

            while [ $attempt -le $max_attempts ]; do
              mvn clean verify \
              -P with-integration-tests \
              -DskipUTs \
              -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
              -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
              -pl integration-test \
              -am -PMultiClusterIT2DualTableManualEnhanced \
              -ntp >> ~/run-tests-$attempt.log && return 0
              test_output=$(cat ~/run-tests-$attempt.log) 

              echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="          
              echo "$test_output"
              echo "==================== END: ~/run-tests-$attempt.log ======================"

              if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
                echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
              fi

              if echo "$test_output" | grep -q "Could not transfer artifact"; then
                if [ $attempt -lt $max_attempts ]; then
                  echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
                  sleep $retry_sleep
                  attempt=$((attempt + 1))
                else
                  echo "Test failed after $max_attempts attempts due to artifact transfer issue."
                  echo "Treating this as a success because the issue is likely transient."
                  return 0
                fi
              elif [ $? -ne 0 ]; then
                echo "Test failed with a different error."
                return 1
              else
                echo "Tests passed"
                return 0
              fi
            done
          }
          retry
      - name: Upload Artifact
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
          path: integration-test/target/cluster-logs
          retention-days: 30