| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| name: PR Build (Linux) |
| |
| # Reusable: invoked by ci.yml. Triggering, path filters, and concurrency |
| # live in the umbrella workflow. |
| on: |
| workflow_call: |
| |
| env: |
| RUST_VERSION: stable |
| RUST_BACKTRACE: 1 |
| # Force GNU ld on Linux: recent Rust stable defaults to rust-lld on |
| # x86_64-unknown-linux-gnu, and rust-lld cannot resolve -ljvm against the |
| # Zulu JDK layout installed by setup-java. Keep bfd for all cargo invocations. |
| RUSTFLAGS: "-Clink-arg=-fuse-ld=bfd" |
| |
| jobs: |
| |
| # Fast lint check - gates all other jobs |
| lint: |
| name: Lint |
| runs-on: ubuntu-24.04 |
| container: |
| image: amd64/rust |
| steps: |
| - uses: actions/checkout@v7 |
| |
| - name: Check Rust formatting |
| run: | |
| rustup component add rustfmt |
| cd native && cargo fmt --all -- --check |
| |
| # Fast syntactic-only scalafix check. Parses sources without compiling, so it |
| # surfaces version-independent style issues (e.g. RedundantSyntax) in seconds, |
| # long before the lint-java matrix finishes its ~3.5 min build. It also scans |
| # the spark-4.1 / spark-4.2 sources that lint-java skips (those profiles can't |
| # run -Psemanticdb yet), so it is the only gate covering them. The full rule |
| # set, including the semantic rules, still runs in lint-java. |
| scalafix-syntactic: |
| name: Lint Scala (syntactic) |
| runs-on: ubuntu-24.04 |
| steps: |
| - uses: actions/checkout@v7 |
| |
| - name: Setup coursier |
| uses: coursier/setup-action@v3 |
| with: |
| jvm: temurin:21 |
| |
| - name: Run syntactic scalafix check (no compile) |
| run: | |
| cs launch scalafix:0.14.6 -- \ |
| --check \ |
| --syntactic \ |
| --config .scalafix-syntactic.conf \ |
| --exclude '**/target/**' \ |
| spark |
| |
| lint-java: |
| needs: lint |
| name: Lint Java (${{ matrix.profile.name }}) |
| runs-on: ubuntu-24.04 |
| container: |
| image: amd64/rust |
| env: |
| JAVA_TOOL_OPTIONS: ${{ (matrix.profile.java_version == '17' || matrix.profile.java_version == '21') && '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' || '' }} |
| strategy: |
| matrix: |
| profile: |
| - name: "Spark 3.4, JDK 11, Scala 2.12" |
| java_version: "11" |
| maven_opts: "-Pspark-3.4 -Pscala-2.12" |
| - name: "Spark 3.5, JDK 17, Scala 2.12" |
| java_version: "17" |
| maven_opts: "-Pspark-3.5 -Pscala-2.12" |
| - name: "Spark 4.0, JDK 17" |
| java_version: "17" |
| maven_opts: "-Pspark-4.0" |
| - name: "Spark 4.0, JDK 21" |
| java_version: "21" |
| maven_opts: "-Pspark-4.0" |
| # Spark 4.1 and 4.2 are intentionally absent: the lint job invokes -Psemanticdb, |
| # but semanticdb-scalac for those Scala patch versions (2.13.17 / 2.13.18) is not |
| # yet published, so we cannot currently run scalafix against the spark-4.1 or |
| # spark-4.2 profiles. |
| fail-fast: false |
| steps: |
| - uses: actions/checkout@v7 |
| |
| - name: Setup Rust & Java toolchain |
| uses: ./.github/actions/setup-builder |
| with: |
| rust-version: ${{ env.RUST_VERSION }} |
| jdk-version: ${{ matrix.profile.java_version }} |
| |
| - name: Cache Maven dependencies |
| uses: actions/cache@v6 |
| with: |
| path: | |
| ~/.m2/repository |
| /root/.m2/repository |
| key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-lint |
| restore-keys: | |
| ${{ runner.os }}-java-maven- |
| |
| - name: Run scalafix check |
| run: | |
| ./mvnw -B package -DskipTests scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb ${{ matrix.profile.maven_opts }} |
| |
| - name: Setup Node.js |
| uses: actions/setup-node@v7 |
| with: |
| node-version: '24' |
| |
| - name: Install prettier |
| run: | |
| npm install -g prettier |
| |
| - name: Run prettier |
| run: | |
| npx prettier "**/*.md" --write |
| |
| - name: Mark workspace as safe for git |
| run: | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" |
| |
| - name: Check for any local git changes (such as generated docs) |
| run: | |
| ./dev/ci/check-working-tree-clean.sh |
| |
| # Compile-only verification for Spark 4.1. Tests are intentionally skipped: the spark-4.1 |
| # profile is currently a build target only, and several runtime/test failures are tracked |
| # in follow-up PRs. Excluded from lint-java because semanticdb-scalac_2.13.17 is not yet |
| # published and the lint job activates -Psemanticdb. |
| build-spark-4-1: |
| needs: lint |
| name: Build Spark 4.1, JDK 17 |
| runs-on: ubuntu-24.04 |
| container: |
| image: amd64/rust |
| steps: |
| - uses: actions/checkout@v7 |
| |
| - name: Setup Rust & Java toolchain |
| uses: ./.github/actions/setup-builder |
| with: |
| rust-version: ${{ env.RUST_VERSION }} |
| jdk-version: 17 |
| |
| - name: Cache Maven dependencies |
| uses: actions/cache@v6 |
| with: |
| path: | |
| ~/.m2/repository |
| /root/.m2/repository |
| key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-spark-4.1-build |
| restore-keys: | |
| ${{ runner.os }}-java-maven- |
| |
| - name: Compile (skip tests) |
| run: ./mvnw -B install -DskipTests -Dmaven.test.skip=true -Pspark-4.1 |
| |
| # Build native library once and share with all test jobs |
| build-native: |
| needs: lint |
| name: Build Native Library |
| runs-on: ubuntu-24.04 |
| container: |
| image: amd64/rust |
| steps: |
| - uses: actions/checkout@v7 |
| - name: Setup Rust toolchain |
| uses: ./.github/actions/setup-builder |
| with: |
| rust-version: ${{ env.RUST_VERSION }} |
| jdk-version: 17 # JDK only needed for JVM module proto generation |
| |
| - name: Restore Cargo cache |
| uses: actions/cache/restore@v6 |
| with: |
| path: | |
| ~/.cargo/registry |
| ~/.cargo/git |
| native/target |
| key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} |
| restore-keys: | |
| ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}- |
| |
| - name: Build native library (CI profile) |
| run: | |
| cd native |
| # CI profile: same overflow behavior as release, but faster compilation |
| # (no LTO, parallel codegen) |
| cargo build --profile ci |
| env: |
| RUSTFLAGS: "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd" |
| |
| - name: Upload native library |
| uses: actions/upload-artifact@v7 |
| with: |
| name: native-lib-linux |
| path: native/target/ci/libcomet.so |
| retention-days: 1 |
| |
| - name: Save Cargo cache |
| uses: actions/cache/save@v6 |
| if: github.ref == 'refs/heads/main' |
| with: |
| path: | |
| ~/.cargo/registry |
| ~/.cargo/git |
| native/target |
| key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} |
| |
| # Run Rust tests (runs in parallel with build-native, uses debug builds) |
| linux-test-rust: |
| needs: lint |
| name: ubuntu-latest/rust-test |
| runs-on: ubuntu-24.04 |
| container: |
| image: amd64/rust |
| steps: |
| - uses: actions/checkout@v7 |
| |
| - name: Setup Rust & Java toolchain |
| uses: ./.github/actions/setup-builder |
| with: |
| rust-version: ${{ env.RUST_VERSION }} |
| jdk-version: 17 |
| |
| - name: Restore Cargo cache |
| uses: actions/cache/restore@v6 |
| with: |
| path: | |
| ~/.cargo/registry |
| ~/.cargo/git |
| native/target |
| # Note: Java version intentionally excluded - Rust target is JDK-independent |
| key: ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} |
| restore-keys: | |
| ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}- |
| |
| - name: Rust test steps |
| uses: ./.github/actions/rust-test |
| |
| - name: Save Cargo cache |
| uses: actions/cache/save@v6 |
| if: github.ref == 'refs/heads/main' |
| with: |
| path: | |
| ~/.cargo/registry |
| ~/.cargo/git |
| native/target |
| key: ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} |
| |
| linux-test: |
| needs: build-native |
| strategy: |
| matrix: |
| # the goal with these profiles is to get coverage of all Java, Scala, and Spark |
| # versions without testing all possible combinations, which would be overkill |
| profile: |
| - name: "Spark 3.4, JDK 11, Scala 2.12" |
| java_version: "11" |
| maven_opts: "-Pspark-3.4 -Pscala-2.12" |
| |
| - name: "Spark 3.5, JDK 17, Scala 2.13" |
| java_version: "17" |
| maven_opts: "-Pspark-3.5 -Pscala-2.13" |
| |
| - name: "Spark 4.0, JDK 21" |
| java_version: "21" |
| maven_opts: "-Pspark-4.0" |
| |
| - name: "Spark 4.1, JDK 17" |
| java_version: "17" |
| maven_opts: "-Pspark-4.1" |
| |
| - name: "Spark 4.2, JDK 17" |
| java_version: "17" |
| maven_opts: "-Pspark-4.2" |
| # Suites are grouped by functional area into balanced buckets so that no test |
| # job runs much longer than ~23 min. See |
| # docs/superpowers/specs/2026-05-22-pr-build-consolidation-design.md for the |
| # per-suite timing analysis behind this grouping. |
| suite: |
| - name: "scans" |
| value: | |
| org.apache.comet.parquet.CometParquetWriterSuite |
| org.apache.comet.parquet.ParquetReadV1Suite |
| org.apache.comet.parquet.ParquetReadV2Suite |
| org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite |
| org.apache.comet.parquet.ParquetTimestampLtzAsNtzSuite |
| org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite |
| org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite |
| org.apache.spark.sql.comet.ParquetEncryptionITCase |
| org.apache.comet.exec.CometNativeReaderSuite |
| org.apache.comet.CometIcebergNativeSuite |
| org.apache.comet.CometIcebergEncryptionSuite |
| org.apache.comet.CometIcebergRewriteActionSuite |
| org.apache.comet.iceberg.IcebergReflectionSuite |
| org.apache.comet.csv.CometCsvNativeReadSuite |
| org.apache.comet.CometFuzzTestSuite |
| org.apache.comet.CometFuzzIcebergSuite |
| org.apache.comet.DataGeneratorSuite |
| - name: "shuffle" |
| value: | |
| org.apache.comet.exec.CometShuffleSuite |
| org.apache.comet.exec.CometShuffle4_0Suite |
| org.apache.comet.exec.CometNativeColumnarToRowSuite |
| org.apache.comet.exec.CometNativeShuffleSuite |
| org.apache.comet.exec.CometShuffleEncryptionSuite |
| org.apache.comet.exec.CometShuffleManagerSuite |
| org.apache.comet.exec.CometAsyncShuffleSuite |
| org.apache.comet.exec.DisableAQECometShuffleSuite |
| org.apache.comet.exec.DisableAQECometAsyncShuffleSuite |
| org.apache.spark.shuffle.sort.SpillSorterSuite |
| - name: "exec" |
| value: | |
| org.apache.comet.exec.CometAggregateSuite |
| org.apache.comet.exec.CometExec3_4PlusSuite |
| org.apache.comet.exec.CometExecSuite |
| org.apache.comet.exec.CometGenerateExecSuite |
| org.apache.comet.exec.CometWindowExecSuite |
| org.apache.comet.exec.CometJoinSuite |
| org.apache.spark.sql.comet.CometMapInBatchSuite |
| org.apache.comet.CometNativeSuite |
| org.apache.comet.CometConfSuite |
| org.apache.comet.CometPublicApiSuite |
| org.apache.comet.CometSetOpWithGroupBySuite |
| org.apache.comet.CometSparkSessionExtensionsSuite |
| org.apache.spark.CometPluginsSuite |
| org.apache.spark.CometPluginsDefaultSuite |
| org.apache.spark.CometPluginsNonOverrideSuite |
| org.apache.spark.CometPluginsUnifiedModeOverrideSuite |
| org.apache.comet.rules.CometScanRuleSuite |
| org.apache.comet.rules.CometScanSchemeFallbackSuite |
| org.apache.comet.rules.CometExecRuleSuite |
| org.apache.comet.rules.RevertNativeForTransitionHeavyStagesSuite |
| org.apache.spark.sql.CometTPCDSQuerySuite |
| org.apache.spark.sql.CometTPCDSQueryTestSuite |
| org.apache.spark.sql.CometTPCHQuerySuite |
| org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite |
| org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite |
| org.apache.spark.sql.comet.CometTaskMetricsSuite |
| org.apache.spark.sql.comet.CometDppFallbackRepro3949Suite |
| org.apache.spark.sql.comet.CometShuffleFallbackStickinessSuite |
| org.apache.spark.sql.comet.PlanDataInjectorSuite |
| org.apache.spark.sql.comet.CometDecimalArithmeticViewSuite |
| org.apache.spark.sql.comet.CometScanWithPlanDataSuite |
| org.apache.spark.sql.comet.util.UtilsSuite |
| org.apache.comet.vector.NativeUtilSuite |
| org.apache.comet.objectstore.NativeConfigSuite |
| org.apache.spark.sql.CometToPrettyStringSuite |
| org.apache.spark.sql.CometCollationSuite |
| org.apache.comet.CometFuzzAggregateSuite |
| org.apache.spark.sql.comet.execution.arrow.CometArrowStreamSuite |
| org.apache.spark.sql.CometSparkInternalFunctionsSuite |
| - name: "expressions" |
| value: | |
| org.apache.comet.CometExpressionSuite |
| org.apache.comet.CometSqlFileTestSuite |
| org.apache.comet.CometExpressionCoverageSuite |
| org.apache.comet.CometHashExpressionSuite |
| org.apache.comet.CometTemporalExpressionSuite |
| org.apache.comet.CometArrayExpressionSuite |
| org.apache.comet.CometCastSuite |
| org.apache.comet.CometDateTimeUtilsSuite |
| org.apache.comet.CometMathExpressionSuite |
| org.apache.comet.CometStringExpressionSuite |
| org.apache.comet.CometBitwiseExpressionSuite |
| org.apache.comet.CometMapExpressionSuite |
| org.apache.comet.CometCsvExpressionSuite |
| org.apache.comet.CometJsonExpressionSuite |
| org.apache.comet.CometJsonJvmSuite |
| org.apache.comet.SparkErrorConverterSuite |
| org.apache.comet.expressions.conditional.CometIfSuite |
| org.apache.comet.expressions.conditional.CometCoalesceSuite |
| org.apache.comet.expressions.conditional.CometCaseWhenSuite |
| org.apache.comet.CometRegExpJvmSuite |
| org.apache.comet.CometCodegenSuite |
| org.apache.comet.CometCodegenSourceSuite |
| org.apache.comet.CometCodegenHOFSuite |
| org.apache.comet.CometFuzzMathSuite |
| org.apache.comet.CometCodegenFuzzSuite |
| org.apache.comet.CometStringDecodeSuite |
| org.apache.comet.CometWidthBucketSuite |
| fail-fast: false |
| name: ${{ matrix.profile.name }} [${{ matrix.suite.name }}] |
| runs-on: ubuntu-24.04 |
| container: |
| image: amd64/rust |
| env: |
| JAVA_TOOL_OPTIONS: ${{ (matrix.profile.java_version == '17' || matrix.profile.java_version == '21') && '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' || '' }} |
| |
| steps: |
| - uses: actions/checkout@v7 |
| |
| - name: Setup Rust & Java toolchain |
| uses: ./.github/actions/setup-builder |
| with: |
| rust-version: ${{ env.RUST_VERSION }} |
| jdk-version: ${{ matrix.profile.java_version }} |
| |
| - name: Download native library |
| uses: actions/download-artifact@v8 |
| with: |
| name: native-lib-linux |
| # Download to release/ since Maven's -Prelease expects libcomet.so there |
| path: native/target/release/ |
| |
| # Restore cargo registry cache (for any cargo commands that might run) |
| - name: Cache Cargo registry |
| uses: actions/cache@v6 |
| with: |
| path: | |
| ~/.cargo/registry |
| ~/.cargo/git |
| key: ${{ runner.os }}-cargo-registry-${{ hashFiles('native/**/Cargo.lock') }} |
| restore-keys: | |
| ${{ runner.os }}-cargo-registry- |
| |
| - name: Java test steps |
| uses: ./.github/actions/java-test |
| with: |
| artifact_name: ${{ matrix.profile.name }}-${{ matrix.suite.name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} |
| suites: ${{ matrix.suite.value }} |
| maven_opts: ${{ matrix.profile.maven_opts }} |
| upload-test-reports: true |
| skip-native-build: true |
| |
| # TPC-H correctness test - verifies benchmark queries produce correct results |
| verify-benchmark-results-tpch: |
| needs: build-native |
| name: Verify TPC-H Results |
| runs-on: ubuntu-24.04 |
| container: |
| image: amd64/rust |
| env: |
| JAVA_TOOL_OPTIONS: --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED |
| steps: |
| - uses: actions/checkout@v7 |
| |
| - name: Setup Rust & Java toolchain |
| uses: ./.github/actions/setup-builder |
| with: |
| rust-version: ${{ env.RUST_VERSION }} |
| jdk-version: 17 |
| |
| - name: Download native library |
| uses: actions/download-artifact@v8 |
| with: |
| name: native-lib-linux |
| path: native/target/release/ |
| |
| - name: Cache Maven dependencies |
| uses: actions/cache@v6 |
| with: |
| path: | |
| ~/.m2/repository |
| /root/.m2/repository |
| key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-java-maven- |
| |
| - name: Cache TPC-H data |
| id: cache-tpch |
| uses: actions/cache@v6 |
| with: |
| path: ./tpch |
| key: tpch-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} |
| |
| - name: Build project |
| run: | |
| ./mvnw -B -Prelease install -DskipTests |
| |
| - name: Generate TPC-H data (SF=1) |
| if: steps.cache-tpch.outputs.cache-hit != 'true' |
| run: | |
| cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCHData" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--location `pwd`/.. --scaleFactor 1 --numPartitions 1 --overwrite" |
| |
| - name: Run TPC-H queries |
| run: | |
| SPARK_HOME=`pwd` SPARK_TPCH_DATA=`pwd`/tpch/sf1_parquet ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCHQuerySuite test |
| |
| # TPC-DS correctness tests - verifies benchmark queries produce correct results. |
| # The three join strategies run sequentially in one job so the project is built once. |
| verify-benchmark-results-tpcds: |
| needs: build-native |
| name: Verify TPC-DS Results |
| runs-on: ubuntu-24.04 |
| container: |
| image: amd64/rust |
| env: |
| JAVA_TOOL_OPTIONS: --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED |
| steps: |
| - uses: actions/checkout@v7 |
| |
| - name: Setup Rust & Java toolchain |
| uses: ./.github/actions/setup-builder |
| with: |
| rust-version: ${{ env.RUST_VERSION }} |
| jdk-version: 17 |
| |
| - name: Download native library |
| uses: actions/download-artifact@v8 |
| with: |
| name: native-lib-linux |
| path: native/target/release/ |
| |
| - name: Cache Maven dependencies |
| uses: actions/cache@v6 |
| with: |
| path: | |
| ~/.m2/repository |
| /root/.m2/repository |
| key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-java-maven- |
| |
| - name: Cache TPC-DS data |
| id: cache-tpcds |
| uses: actions/cache@v6 |
| with: |
| path: ./tpcds-sf-1 |
| key: tpcds-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} |
| |
| - name: Build project |
| run: | |
| ./mvnw -B -Prelease install -DskipTests |
| |
| - name: Checkout tpcds-kit |
| if: steps.cache-tpcds.outputs.cache-hit != 'true' |
| uses: actions/checkout@v7 |
| with: |
| repository: databricks/tpcds-kit |
| path: ./tpcds-kit |
| |
| - name: Build tpcds-kit |
| if: steps.cache-tpcds.outputs.cache-hit != 'true' |
| run: | |
| apt-get update && apt-get install -y yacc bison flex gcc-12 g++-12 |
| update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 120 --slave /usr/bin/g++ g++ /usr/bin/g++-12 |
| cd tpcds-kit/tools && make OS=LINUX |
| |
| - name: Generate TPC-DS data (SF=1) |
| if: steps.cache-tpcds.outputs.cache-hit != 'true' |
| run: | |
| cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCDSData" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--dsdgenDir `pwd`/../tpcds-kit/tools --location `pwd`/../tpcds-sf-1 --scaleFactor 1 --numPartitions 1" |
| |
| - name: Run TPC-DS queries (Sort merge join) |
| run: | |
| SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test |
| env: |
| SPARK_TPCDS_JOIN_CONF: | |
| spark.sql.autoBroadcastJoinThreshold=-1 |
| spark.sql.join.preferSortMergeJoin=true |
| |
| - name: Run TPC-DS queries (Broadcast hash join) |
| run: | |
| SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test |
| env: |
| SPARK_TPCDS_JOIN_CONF: | |
| spark.sql.autoBroadcastJoinThreshold=10485760 |
| |
| - name: Run TPC-DS queries (Shuffled hash join) |
| run: | |
| SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test |
| env: |
| SPARK_TPCDS_JOIN_CONF: | |
| spark.sql.autoBroadcastJoinThreshold=-1 |
| spark.sql.join.forceApplyShuffledHashJoin=true |
