analytics/.github/workflows/test.yaml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Test

on:
  push:
    branches: [ "trunk" ]
  pull_request:
    branches: [ "trunk" ]
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  build-jdk11:
    name: Compile and build (JDK 11)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Setup JDK
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11
      - run: |
          sudo apt-get update

          if [ -f /etc/ssl/certs/java/cacerts/cacerts ]; then
              sudo mv /etc/ssl/certs/java/cacerts/ /etc/ssl/certs/java/cacerts-old
              sudo mv /etc/ssl/certs/java/cacerts-old/cacerts /etc/ssl/certs/java/
              sudo rmdir /etc/ssl/certs/java/cacerts-old
          fi

          apt-get download ant ant-optional
          sudo dpkg --force-all -i ant*.deb
          rm ant*.deb

          sudo bash -c 'for i in {2..20}; do echo 127.0.0.${i} localhost${i} >> /etc/hosts; done'

          for i in {2..20}
          do
            sudo ip addr add "127.0.0.${i}" dev lo
            sudo route add -host "127.0.0.${i}" dev lo;
          done

          export JDK_VERSION="11"
          export SPARK_VERSION="3"
          export SCALA_VERSION="2.12"
          export CASSANDRA_USE_JDK11="true"

          ./scripts/build-dependencies.sh

          ./gradlew --no-daemon --max-workers=2 codeCheckTasks
      - name: Cache Maven repository
        uses: actions/cache@v4
        with:
          path: ~/.m2
          key: maven-repo-jdk11-${{ github.sha }}
      - name: Cache workspace
        id: cache-build-save
        uses: actions/cache/save@v4
        with:
          path: ${{ github.workspace }}
          key: build-jdk11-${{ github.sha }}

  # JDK17 dependency build for the Spark 4 / Scala 2.13 / Cassandra 5.0 matrix.
  # Produces a distinct workspace cache (build-jdk17-...) so downstream test jobs
  # must restore from exactly one of build-jdk11 / build-jdk17 — mixing them
  # would clobber dependency jars.
  build-jdk17:
    name: Compile and build (JDK 17)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Setup JDK
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 17
      - run: |
          sudo apt-get update

          if [ -f /etc/ssl/certs/java/cacerts/cacerts ]; then
              sudo mv /etc/ssl/certs/java/cacerts/ /etc/ssl/certs/java/cacerts-old
              sudo mv /etc/ssl/certs/java/cacerts-old/cacerts /etc/ssl/certs/java/
              sudo rmdir /etc/ssl/certs/java/cacerts-old
          fi

          apt-get download ant ant-optional
          sudo dpkg --force-all -i ant*.deb
          rm ant*.deb

          sudo bash -c 'for i in {2..20}; do echo 127.0.0.${i} localhost${i} >> /etc/hosts; done'

          for i in {2..20}
          do
            sudo ip addr add "127.0.0.${i}" dev lo
            sudo route add -host "127.0.0.${i}" dev lo;
          done

          export JDK_VERSION="17"
          export SPARK_VERSION="4"
          export SCALA_VERSION="2.13"
          # JDK17 only targets Cassandra 5.0+; skip 4.0 / 4.1 dtest jar builds
          # (build-dtest-jars.sh reads this var to filter CANDIDATE_BRANCHES).
          export BRANCHES="cassandra-5.0"

          ./scripts/build-dependencies.sh

          ./gradlew codeCheckTasks
      - name: Cache Maven repository
        uses: actions/cache@v4
        with:
          path: ~/.m2
          key: maven-repo-jdk17-${{ github.sha }}
      - name: Cache workspace
        id: cache-build-save
        uses: actions/cache/save@v4
        with:
          path: ${{ github.workspace }}
          key: build-jdk17-${{ github.sha }}

  unit-test:
    name: Unit test - Scala ${{ matrix.scala }} ${{ matrix.sstable-format }} C${{ matrix.cassandra }} Spark${{ matrix.spark }} JDK${{ matrix.jdk }}
    # Each matrix entry only consumes one of these caches (selected by matrix.jdk),
    # but `needs:` cannot be matrix-conditional so we wait on both builds.
    needs: [build-jdk11, build-jdk17]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          - scala: '2.13'
            sstable-format: 'bti'
            cassandra: '5.0'
            jdk: '11'
            spark: '3'
          - scala: '2.12'
            sstable-format: 'big'
            cassandra: '4.1'
            jdk: '11'
            spark: '3'
          - scala: '2.12'
            sstable-format: 'big'
            cassandra: '4.0'
            jdk: '11'
            spark: '3'
          - scala: '2.13'
            sstable-format: 'bti'
            cassandra: '5.0'
            jdk: '17'
            spark: '4'
      fail-fast: false
    steps:
      - name: Setup JDK
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: ${{ matrix.jdk }}
      - run: |
          sudo bash -c 'for i in {2..20}; do echo 127.0.0.${i} localhost${i} >> /etc/hosts; done'
          for i in {2..20}
          do
            sudo ip addr add "127.0.0.${i}" dev lo
            sudo route add -host "127.0.0.${i}" dev lo;
          done
      - name: Cache Maven repository
        uses: actions/cache@v4
        with:
          path: ~/.m2
          key: maven-repo-jdk${{ matrix.jdk }}-${{ github.sha }}
      - name: Cache workspace
        id: cache-build-restore
        uses: actions/cache/restore@v4
        with:
          path: ${{ github.workspace }}
          key: build-jdk${{ matrix.jdk }}-${{ github.sha }}
      - run: |
          export SPARK_VERSION="${{ matrix.spark }}"
          export SCALA_VERSION="${{ matrix.scala }}"
          export JDK_VERSION="${{ matrix.jdk }}"
          export INTEGRATION_MAX_PARALLEL_FORKS=1
          export INTEGRATION_MAX_HEAP_SIZE="3072M"
          export CASSANDRA_VERSION="${{ matrix.cassandra }}"
          if [ "${{ matrix.jdk }}" = "11" ]; then
            export CASSANDRA_USE_JDK11=true
          fi

          ./gradlew --stacktrace clean assemble check -x cassandra-analytics-integration-tests:test -Dcassandra.analytics.bridges.sstable_format=${{ matrix.sstable-format }}
      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: unit-test-results-s${{ matrix.scala }}-${{ matrix.sstable-format }}-C${{ matrix.cassandra }}-Spark${{ matrix.spark }}-JDK${{ matrix.jdk }}
          path: |
            build/test-results/**
            build/reports/tests/**
          retention-days: 30

  integration-test:
    name: Integration test - ${{ matrix.config }} (${{ matrix.job_index }})
    # Each matrix entry only consumes one of these caches (selected by matrix.jdk),
    # but `needs:` cannot be matrix-conditional so we wait on both builds.
    needs: [build-jdk11, build-jdk17]
    runs-on: ubuntu-latest
    strategy:
      # GitHub Actions generate a cross-product of 'config' × 'job_index' (4 × 5 = 20 jobs).
      # The 'include' entries don't add new combinations — they augment existing ones
      # by matching on 'config' and injecting 'scala', 'cassandra', 'jdk', and 'spark'
      # into each match. To add a new version: add one entry to 'config' and one to
      # 'include'.
      matrix:
        config: ['s2.13-c5.0.5', 's2.12-c4.1.4', 's2.12-c4.0.17', 's2.13-c5.0.5-spark4']
        job_index: [0, 1, 2, 3, 4]
        job_total: [5]
        include:
          - config: 's2.13-c5.0.5'
            scala: '2.13'
            cassandra: '5.0.5'
            jdk: '11'
            spark: '3'
          - config: 's2.12-c4.1.4'
            scala: '2.12'
            cassandra: '4.1.4'
            jdk: '11'
            spark: '3'
          - config: 's2.12-c4.0.17'
            scala: '2.12'
            cassandra: '4.0.17'
            jdk: '11'
            spark: '3'
          - config: 's2.13-c5.0.5-spark4'
            scala: '2.13'
            cassandra: '5.0.5'
            jdk: '17'
            spark: '4'
      fail-fast: false
    steps:
      - name: Setup JDK
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: ${{ matrix.jdk }}
      - run: |
          sudo bash -c 'for i in {2..20}; do echo 127.0.0.${i} localhost${i} >> /etc/hosts; done'
          for i in {2..20}
          do
            sudo ip addr add "127.0.0.${i}" dev lo
            sudo route add -host "127.0.0.${i}" dev lo;
          done
      - name: Cache Maven repository
        uses: actions/cache@v4
        with:
          path: ~/.m2
          key: maven-repo-jdk${{ matrix.jdk }}-${{ github.sha }}
      - name: Cache workspace
        id: cache-build-restore
        uses: actions/cache/restore@v4
        with:
          path: ${{ github.workspace }}
          key: build-jdk${{ matrix.jdk }}-${{ github.sha }}
      - run: |
          export SPARK_VERSION="${{ matrix.spark }}"
          export SCALA_VERSION="${{ matrix.scala }}"
          export JDK_VERSION="${{ matrix.jdk }}"
          export INTEGRATION_MAX_PARALLEL_FORKS=1
          export INTEGRATION_MAX_HEAP_SIZE="3072M"
          if [ "${{ matrix.jdk }}" = "11" ]; then
            export CASSANDRA_USE_JDK11=true
          fi

          export DTEST_JAR="dtest-${{ matrix.cassandra }}.jar"
          export CASSANDRA_VERSION=$(echo ${{ matrix.cassandra }} | cut -d'.' -f 1,2)

          ./gradlew --stacktrace clean assemble

          cd cassandra-analytics-integration-tests/src/test/java
          # Shuffle test classes using the commit SHA as seed for reproducible randomization,
          # then shard across runners via round-robin on the shuffled order.
          CLASSNAMES=$(find . -name '*Test.java' | cut -c 3- | sed 's@/@.@g' | sed 's/.\{5\}$//' \
            | python3 -c "import random,sys; lines=sys.stdin.read().splitlines(); random.seed('$GITHUB_SHA'); random.shuffle(lines); print('\n'.join(lines))" \
            | awk 'NR % ${{ matrix.job_total }} == ${{ matrix.job_index }}')
          cd ../../../..

          EXIT_STATUS=0
          # Execution of "gradle test --test $TEST_NAME" returns non-zero exit code when commend did not run any test
          # (e.g. when all tests are ignored). Currently there is no option to change Gradle behaviour.
          # Workaround the issue by explicitly skipping test classes that cannot be executed on Cassandra 4.0.
          C40_EXCLUSIONS=("org.apache.cassandra.analytics.BulkWriteDownInstanceMultipleTokensTest" "org.apache.cassandra.analytics.BulkWriteDownSidecarMultipleTokensTest" "org.apache.cassandra.analytics.CassandraAnalyticsSimpleMultipleTokensTest" "org.apache.cassandra.analytics.RandomPartitionerTest" "org.apache.cassandra.analytics.testcontainer.BulkWriteS3CompatModeSimpleMultipleTokensTest" "org.apache.cassandra.analytics.data.ClearSnapshotTest")
          for TEST_NAME in $CLASSNAMES; do
            SKIP="false"
            for C40_EXCLUSION in "${C40_EXCLUSIONS[@]}";
            do
              if [[ "$CASSANDRA_VERSION" == "4.0" && "$TEST_NAME" == ${C40_EXCLUSION} ]]; then
                SKIP="true"
                break
              fi
            done

            test_id=$(date +%H%M%S)
            mkdir -p "test-reports/$test_id/html"

            if [ $SKIP == "false" ]; then
              echo Executing test $TEST_NAME
              ./gradlew --stacktrace cassandra-analytics-integration-tests:test --tests $TEST_NAME --no-daemon || EXIT_STATUS=$?;
              find build/test-results -name "*.xml" -exec cp {} "test-reports/$test_id/" \; 2>/dev/null || true
              [ -d "build/reports/tests" ] && cp -r build/reports/tests/. "test-reports/$test_id/html/" 2>/dev/null || true
            else
              echo "Skipping test $TEST_NAME"
            fi
          done;

          exit $EXIT_STATUS
      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: integration-tests-${{ matrix.config }}-${{ matrix.job_index }}
          path: test-reports/
          retention-days: 30