pom.xml

<?xml version="1.0" encoding="UTF-8"?>

<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
-->


<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
    <groupId>org.apache</groupId>
    <artifactId>apache</artifactId>
    <version>23</version>
  </parent>
  <groupId>org.apache.datafusion</groupId>
  <artifactId>comet-parent-spark${spark.version.short}_${scala.binary.version}</artifactId>
  <version>1.1.0-SNAPSHOT</version>
  <packaging>pom</packaging>
  <name>Comet Project Parent POM</name>

  <modules>
    <module>common</module>
    <module>spark</module>
    <module>spark-integration</module>
  </modules>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
    <java.version>17</java.version>
    <!--
      Default Delta dep version (only consumed when `-Pcontrib-delta` is also
      active). Spark profiles override this: spark-3.5 -> 3.3.2, spark-4.1 ->
      4.1.0. The top-level default lets Maven invocations that don't activate a
      Spark profile (e.g. `mvn -Pcontrib-delta spotless:apply`) resolve the
      property without an error.
    -->
    <delta.version>4.1.0</delta.version>
    <!-- Used only to verify reflected Celeborn internals; never bundled with Comet. -->
    <celeborn.version>0.7.0</celeborn.version>
    <maven.compiler.source>${java.version}</maven.compiler.source>
    <maven.compiler.target>${java.version}</maven.compiler.target>
    <maven-compiler-plugin.version>3.11.0</maven-compiler-plugin.version>
    <maven-assembly-plugin.version>3.6.0</maven-assembly-plugin.version>
    <maven-shade-plugin.version>3.2.4</maven-shade-plugin.version>
    <maven-surefire-plugin.version>3.5.4</maven-surefire-plugin.version>
    <maven-source-plugin.version>3.3.0</maven-source-plugin.version>
    <maven-enforcer-plugin.version>3.3.0</maven-enforcer-plugin.version>
    <maven-failsafe-plugin.version>3.1.0</maven-failsafe-plugin.version>
    <asm.version>9.1</asm.version>
    <build-helper-maven-plugin.version>3.4.0</build-helper-maven-plugin.version>
    <flatten-maven-plugin.version>1.3.0</flatten-maven-plugin.version>
    <scalastyle-maven-plugin.version>1.0.0</scalastyle-maven-plugin.version>
    <git-commit-id-maven-plugin.version>9.0.2</git-commit-id-maven-plugin.version>
    <exec-maven-plugin.version>3.1.0</exec-maven-plugin.version>
    <protoc-jar-maven-plugin.version>3.11.4</protoc-jar-maven-plugin.version>
    <scalafix-maven-plugin.version>0.1.7_0.10.4</scalafix-maven-plugin.version>
    <extra-enforcer-rules.version>1.7.0</extra-enforcer-rules.version>
    <scalafmt.version>3.6.1</scalafmt.version>
    <apache-rat-plugin.version>0.16.1</apache-rat-plugin.version>
    <scala.version>2.13.17</scala.version>
    <scala.binary.version>2.13</scala.binary.version>
    <scala.plugin.version>4.9.6</scala.plugin.version>
    <scalatest.version>3.2.16</scalatest.version>
    <scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
    <spark.version>4.1.3</spark.version>
    <spark.version.short>4.1</spark.version.short>
    <spark.maven.scope>provided</spark.maven.scope>
    <protobuf.version>3.25.5</protobuf.version>
    <parquet.version>1.16.0</parquet.version>
    <parquet.maven.scope>provided</parquet.maven.scope>
    <hadoop.version>3.3.4</hadoop.version>
    <arrow.version>18.3.0</arrow.version>
    <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
    <spotless.version>2.43.0</spotless.version>
    <jacoco.version>0.8.11</jacoco.version>
    <semanticdb.version>4.13.6</semanticdb.version>
    <slf4j.version>2.0.17</slf4j.version>
    <guava.version>33.2.1-jre</guava.version>
    <testcontainers.version>1.21.4</testcontainers.version>
    <amazon-awssdk-v2.version>2.31.51</amazon-awssdk-v2.version>
    <jni.dir>${project.basedir}/../native/target/debug</jni.dir>
    <platform>darwin</platform>
    <arch>x86_64</arch>
    <comet.shade.packageName>org.apache.comet.shaded</comet.shade.packageName>
    <!-- Used by some tests inherited from Spark to get project root directory -->
    <spark.test.home>${session.executionRootDirectory}</spark.test.home>

    <!-- Reverse default (skip installation), and then enable only for child modules -->
    <maven.deploy.skip>true</maven.deploy.skip>

    <!-- For testing with JDK 17 -->
    <extraJavaTestArgs>
      -XX:+IgnoreUnrecognizedVMOptions
      --add-opens=java.base/java.lang=ALL-UNNAMED
      --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
      --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
      --add-opens=java.base/java.io=ALL-UNNAMED
      --add-opens=java.base/java.net=ALL-UNNAMED
      --add-opens=java.base/java.nio=ALL-UNNAMED
      --add-opens=java.base/java.util=ALL-UNNAMED
      --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
      --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
      --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
      --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
      --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
      --add-opens=java.base/sun.security.action=ALL-UNNAMED
      --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
      -Djdk.reflect.useDirectMethodHandle=false
    </extraJavaTestArgs>
    <argLine>-ea -Xmx4g -Xss4m ${extraJavaTestArgs}</argLine>
    <shims.majorVerSrc>spark-4.x</shims.majorVerSrc>
    <shims.minorVerSrc>spark-4.1</shims.minorVerSrc>
    <!-- Source root shared by Spark 4.1 and later. The default build targets Spark 4.1, so this
         defaults to spark-4.1+; the spark-4.0 and 3.x profiles override it to an empty
         placeholder (spark-none) so they do not compile these 4.1+ shims. -->
    <shims.minorPlusVerSrc>spark-4.1+</shims.minorPlusVerSrc>
  </properties>

  <dependencyManagement>
    <dependencies>
      <!-- Spark dependencies -->
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-sql_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <scope>${spark.maven.scope}</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-hadoop</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-column</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-format-structures</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
          </exclusion>

          <!-- We're using "org.slf4j:jcl-over-slf4j" -->
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>

          <exclusion>
            <groupId>org.apache.arrow</groupId>
            <artifactId>*</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <scope>${spark.maven.scope}</scope>
        <exclusions>

          <!-- We're using "org.slf4j:jcl-over-slf4j" -->
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>

          <!-- Comet uses arrow-memory-unsafe -->
          <exclusion>
            <groupId>org.apache.arrow</groupId>
            <artifactId>*</artifactId>
          </exclusion>
        </exclusions>
      </dependency>

      <!-- Arrow dependencies -->
      <dependency>
        <groupId>org.apache.arrow</groupId>
        <artifactId>arrow-vector</artifactId>
        <version>${arrow.version}</version>
        <!-- Exclude the following in favor of those from Spark -->
        <exclusions>
          <exclusion>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-annotations</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-core</artifactId>
          </exclusion>
          <exclusion>
            <groupId>io.netty</groupId>
            <artifactId>netty-common</artifactId>
          </exclusion>
          <exclusion>
            <groupId>com.google.code.findbugs</groupId>
            <artifactId>jsr305</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.arrow</groupId>
        <artifactId>arrow-memory-unsafe</artifactId>
        <version>${arrow.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.arrow</groupId>
        <artifactId>arrow-c-data</artifactId>
        <version>${arrow.version}</version>
      </dependency>

      <!-- Parquet dependencies -->
      <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-column</artifactId>
        <version>${parquet.version}</version>
        <scope>${parquet.maven.scope}</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-format-structures</artifactId>
        <version>${parquet.version}</version>
        <scope>${parquet.maven.scope}</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-hadoop</artifactId>
        <version>${parquet.version}</version>
        <scope>${parquet.maven.scope}</scope>
        <exclusions>
          <!-- Exclude the following in favor of jakarta.annotation:jakarta.annotation-api -->
          <exclusion>
            <groupId>javax.annotation</groupId>
            <artifactId>javax.annotation-api</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-avro</artifactId>
        <version>${parquet.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.parquet</groupId>
        <artifactId>parquet-hadoop</artifactId>
        <version>${parquet.version}</version>
        <classifier>tests</classifier>
        <scope>test</scope>
        <exclusions>
          <!-- Exclude the following in favor of jakarta.annotation:jakarta.annotation-api -->
          <exclusion>
            <groupId>javax.annotation</groupId>
            <artifactId>javax.annotation-api</artifactId>
          </exclusion>
        </exclusions>
      </dependency>

      <!-- Others -->
      <dependency>
        <groupId>org.scala-lang</groupId>
        <artifactId>scala-library</artifactId>
        <version>${scala.version}</version>
      </dependency>
      <dependency>
        <groupId>org.scala-lang.modules</groupId>
        <artifactId>scala-collection-compat_${scala.binary.version}</artifactId>
        <version>2.12.0</version>
      </dependency>
      <dependency>
        <groupId>com.google.protobuf</groupId>
        <artifactId>protobuf-java</artifactId>
        <version>${protobuf.version}</version>
      </dependency>
      <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-api</artifactId>
        <version>${slf4j.version}</version>
        <scope>${spark.maven.scope}</scope>
      </dependency>

      <!-- Shaded deps marked as provided. These are promoted to compile scope
           in the modules where we want the shaded classes to appear in the
           associated jar. -->
      <dependency>
        <groupId>com.google.guava</groupId>
        <artifactId>guava</artifactId>
        <version>${guava.version}</version>
      </dependency>
      <!-- End of shaded deps -->

      <!-- Test dependencies -->
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-core_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <classifier>tests</classifier>
        <scope>test</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.datafusion</groupId>
            <artifactId>*</artifactId>
          </exclusion>

          <!-- We are using arrow-memory-unsafe -->
          <exclusion>
            <groupId>org.apache.arrow</groupId>
            <artifactId>*</artifactId>
          </exclusion>

          <!-- We're using "org.slf4j:jcl-over-slf4j" -->
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>

          <exclusion>
            <groupId>com.google.code.findbugs</groupId>
            <artifactId>jsr305</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <classifier>tests</classifier>
        <scope>test</scope>
        <exclusions>

          <!-- We are using arrow-memory-unsafe -->
          <exclusion>
            <groupId>org.apache.arrow</groupId>
            <artifactId>*</artifactId>
          </exclusion>

          <!-- We're using "org.slf4j:jcl-over-slf4j" -->
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>

          <exclusion>
            <groupId>com.google.code.findbugs</groupId>
            <artifactId>jsr305</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-sql_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <classifier>tests</classifier>
        <scope>test</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-hadoop</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-column</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.datafusion</groupId>
            <artifactId>*</artifactId>
          </exclusion>

          <!-- We're using "org.slf4j:jcl-over-slf4j" -->
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>

          <exclusion>
            <groupId>org.apache.arrow</groupId>
            <artifactId>*</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-hive_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <scope>test</scope>
        <exclusions>
          <exclusion>
            <groupId>org.apache.datafusion</groupId>
            <artifactId>*</artifactId>
          </exclusion>
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.arrow</groupId>
            <artifactId>*</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
      <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.13.2</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.assertj</groupId>
        <artifactId>assertj-core</artifactId>
        <version>3.27.7</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.scalatest</groupId>
        <artifactId>scalatest_${scala.binary.version}</artifactId>
        <version>${scalatest.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.scala-lang</groupId>
        <artifactId>scala-reflect</artifactId>
        <version>${scala.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>org.scalatestplus</groupId>
        <artifactId>junit-4-13_${scala.binary.version}</artifactId>
        <version>3.2.16.0</version>
        <scope>test</scope>
      </dependency>

      <!-- For benchmarks to access S3 -->
      <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-hadoop-cloud_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
        <classifier>tests</classifier>
        <scope>test</scope>
        <exclusions>
          <!-- We're using hadoop-client -->
          <exclusion>
            <groupId>org.apache.hadoop.thirdparty</groupId>
            <artifactId>hadoop-shaded-guava</artifactId>
          </exclusion>
          <exclusion>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-annotations</artifactId>
          </exclusion>
          <exclusion>
            <groupId>javax.xml.bind</groupId>
            <artifactId>jaxb-api</artifactId>
          </exclusion>

          <!-- We're using "org.slf4j:jcl-over-slf4j" -->
          <exclusion>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
          </exclusion>

          <exclusion>
            <groupId>org.apache.arrow</groupId>
            <artifactId>*</artifactId>
          </exclusion>
        </exclusions>
      </dependency>

      <!-- TestContainers for testing reading Parquet on S3 -->
      <dependency>
        <groupId>org.testcontainers</groupId>
        <artifactId>minio</artifactId>
        <version>${testcontainers.version}</version>
        <scope>test</scope>
      </dependency>
      <!--
        Override docker-java transitive (testcontainers 1.21.x pins 3.4.2, which speaks Docker
        API 1.32 max). Modern daemons (Colima, OrbStack, recent Docker Desktop) require API
        >= 1.40. docker-java 3.7.0+ defaults to API 1.44.
      -->
      <dependency>
        <groupId>com.github.docker-java</groupId>
        <artifactId>docker-java-api</artifactId>
        <version>3.7.1</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>com.github.docker-java</groupId>
        <artifactId>docker-java-transport-zerodep</artifactId>
        <version>3.7.1</version>
        <scope>test</scope>
      </dependency>
      <!--
        AWS SDK modules for Iceberg REST catalog + S3 tests.
        iceberg-spark-runtime treats the AWS SDK as provided scope, so tests
        that exercise Iceberg's S3FileIO (via ResolvingFileIO) must supply these.
        AwsProperties references all service client types in method signatures,
        and Java serialization introspection resolves them at class-load time.
      -->
      <dependency>
        <groupId>software.amazon.awssdk</groupId>
        <artifactId>s3</artifactId>
        <version>${amazon-awssdk-v2.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>software.amazon.awssdk</groupId>
        <artifactId>sts</artifactId>
        <version>${amazon-awssdk-v2.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>software.amazon.awssdk</groupId>
        <artifactId>dynamodb</artifactId>
        <version>${amazon-awssdk-v2.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>software.amazon.awssdk</groupId>
        <artifactId>glue</artifactId>
        <version>${amazon-awssdk-v2.version}</version>
        <scope>test</scope>
      </dependency>
      <dependency>
        <groupId>software.amazon.awssdk</groupId>
        <artifactId>kms</artifactId>
        <version>${amazon-awssdk-v2.version}</version>
        <scope>test</scope>
      </dependency>

      <dependency>
        <groupId>org.codehaus.jackson</groupId>
        <artifactId>jackson-mapper-asl</artifactId>
        <version>${codehaus.jackson.version}</version>
        <scope>test</scope>
      </dependency>

      <dependency>
        <groupId>org.rogach</groupId>
        <artifactId>scallop_${scala.binary.version}</artifactId>
        <version>5.1.0</version>
      </dependency>

      <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client-minicluster</artifactId>
        <version>${hadoop.version}</version>
        <scope>test</scope>
      </dependency>

    </dependencies>

  </dependencyManagement>

  <profiles>
    <profile>
      <id>release</id>
      <properties>
        <jni.dir>${project.basedir}/../native/target/release</jni.dir>
      </properties>
    </profile>

    <profile>
      <id>Win-x86</id>
      <activation>
        <os>
          <family>Windows</family>
          <arch>x86</arch>
        </os>
      </activation>
      <properties>
        <platform>win32</platform>
        <arch>x86_64</arch>
      </properties>
    </profile>

    <profile>
      <id>Win-amd64</id>
      <activation>
        <os>
          <family>Windows</family>
          <arch>amd64</arch>
        </os>
      </activation>
      <properties>
        <platform>win32</platform>
        <arch>amd64</arch>
      </properties>
    </profile>

    <profile>
      <id>Darwin-x86</id>
      <activation>
        <os>
          <family>mac</family>
          <arch>x86</arch>
        </os>
      </activation>
      <properties>
        <platform>darwin</platform>
        <arch>x86_64</arch>
      </properties>
    </profile>

    <profile>
      <id>Darwin-aarch64</id>
      <activation>
        <os>
          <family>mac</family>
          <arch>aarch64</arch>
        </os>
      </activation>
      <properties>
        <platform>darwin</platform>
        <arch>aarch64</arch>
      </properties>
    </profile>

    <profile>
      <id>Linux-amd64</id>
      <activation>
        <os>
          <family>Linux</family>
          <arch>amd64</arch>
        </os>
      </activation>
      <properties>
        <platform>linux</platform>
        <arch>amd64</arch>
      </properties>
    </profile>

    <profile>
      <id>Linux-aarch64</id>
      <activation>
        <os>
          <family>Linux</family>
          <arch>aarch64</arch>
        </os>
      </activation>
      <properties>
        <platform>linux</platform>
        <arch>aarch64</arch>
      </properties>
    </profile>

    <profile>
      <id>spark-3.4</id>
      <properties>
        <scala.version>2.12.17</scala.version>
        <scala.binary.version>2.12</scala.binary.version>
        <spark.version>3.4.3</spark.version>
        <spark.version.short>3.4</spark.version.short>
        <parquet.version>1.13.1</parquet.version>
        <semanticdb.version>4.8.8</semanticdb.version>
        <slf4j.version>2.0.6</slf4j.version>
        <shims.majorVerSrc>spark-3.x</shims.majorVerSrc>
        <shims.minorVerSrc>spark-3.4</shims.minorVerSrc>
        <shims.minorPlusVerSrc>spark-none</shims.minorPlusVerSrc>
        <java.version>17</java.version>
        <maven.compiler.source>${java.version}</maven.compiler.source>
        <maven.compiler.target>${java.version}</maven.compiler.target>
      </properties>
    </profile>

    <profile>
      <id>spark-3.5</id>
      <properties>
        <scala.version>2.12.18</scala.version>
        <scala.binary.version>2.12</scala.binary.version>
        <spark.version>3.5.9</spark.version>
        <spark.version.short>3.5</spark.version.short>
        <parquet.version>1.13.1</parquet.version>
        <semanticdb.version>4.8.8</semanticdb.version>
        <slf4j.version>2.0.7</slf4j.version>
        <shims.majorVerSrc>spark-3.x</shims.majorVerSrc>
        <shims.minorVerSrc>spark-3.5</shims.minorVerSrc>
        <shims.minorPlusVerSrc>spark-none</shims.minorPlusVerSrc>
        <java.version>17</java.version>
        <maven.compiler.source>${java.version}</maven.compiler.source>
        <maven.compiler.target>${java.version}</maven.compiler.target>
      </properties>
    </profile>

    <profile>
      <id>spark-4.0</id>
      <properties>
        <scala.version>2.13.16</scala.version>
        <scala.binary.version>2.13</scala.binary.version>
        <spark.version>4.0.4</spark.version>
        <spark.version.short>4.0</spark.version.short>
        <parquet.version>1.15.2</parquet.version>
        <semanticdb.version>4.13.6</semanticdb.version>
        <slf4j.version>2.0.16</slf4j.version>
        <shims.majorVerSrc>spark-4.x</shims.majorVerSrc>
        <shims.minorVerSrc>spark-4.0</shims.minorVerSrc>
        <shims.minorPlusVerSrc>spark-none</shims.minorPlusVerSrc>
        <java.version>17</java.version>
        <maven.compiler.source>${java.version}</maven.compiler.source>
        <maven.compiler.target>${java.version}</maven.compiler.target>
      </properties>
    </profile>

    <profile>
      <id>spark-4.1</id>
      <properties>
        <!-- Spark 4.1 is compiled against Scala 2.13.17 and emits calls into stdlib methods
             added in that release (e.g. MurmurHash3$.caseClassHash$default$2()). Comet must
             match to avoid runtime NoSuchMethodError. Note: semanticdb-scalac_2.13.17 is not
             yet published, so the -Psemanticdb / scalafix lint job is skipped for spark-4.1. -->
        <scala.version>2.13.17</scala.version>
        <scala.binary.version>2.13</scala.binary.version>
        <spark.version>4.1.3</spark.version>
        <spark.version.short>4.1</spark.version.short>
        <parquet.version>1.16.0</parquet.version>
        <semanticdb.version>4.13.6</semanticdb.version>
        <slf4j.version>2.0.17</slf4j.version>
        <shims.majorVerSrc>spark-4.x</shims.majorVerSrc>
        <shims.minorPlusVerSrc>spark-4.1+</shims.minorPlusVerSrc>
        <shims.minorVerSrc>spark-4.1</shims.minorVerSrc>
        <java.version>17</java.version>
        <maven.compiler.source>${java.version}</maven.compiler.source>
        <maven.compiler.target>${java.version}</maven.compiler.target>
      </properties>
    </profile>

    <profile>
      <!-- Experimental Spark 4.2 support, targeting the 4.2.0 release. -->
      <id>spark-4.2</id>
      <properties>
        <scala.version>2.13.18</scala.version>
        <scala.binary.version>2.13</scala.binary.version>
        <spark.version>4.2.0</spark.version>
        <spark.version.short>4.2</spark.version.short>
        <parquet.version>1.17.0</parquet.version>
        <semanticdb.version>4.13.6</semanticdb.version>
        <slf4j.version>2.0.17</slf4j.version>
        <shims.majorVerSrc>spark-4.x</shims.majorVerSrc>
        <shims.minorPlusVerSrc>spark-4.1+</shims.minorPlusVerSrc>
        <shims.minorVerSrc>spark-4.2</shims.minorVerSrc>
        <!-- Use jdk17 by default -->
        <java.version>17</java.version>
        <maven.compiler.source>${java.version}</maven.compiler.source>
        <maven.compiler.target>${java.version}</maven.compiler.target>
      </properties>
    </profile>

    <profile>
      <id>scala-2.12</id>
      <properties>
        <scala.version>2.12.18</scala.version>
        <scala.binary.version>2.12</scala.binary.version>
        <semanticdb.version>4.8.8</semanticdb.version>
      </properties>
    </profile>

    <profile>
      <id>scala-2.13</id>
      <properties>
        <scala.version>2.13.16</scala.version>
        <scala.binary.version>2.13</scala.binary.version>
        <semanticdb.version>4.13.6</semanticdb.version>
      </properties>
    </profile>

    <profile>
      <id>semanticdb</id>
      <properties>
        <scalastyle.skip>true</scalastyle.skip>
        <spotless.check.skip>true</spotless.check.skip>
        <enforcer.skip>true</enforcer.skip>
      </properties>
      <build>
        <pluginManagement>
          <plugins>
            <plugin>
              <groupId>net.alchim31.maven</groupId>
              <artifactId>scala-maven-plugin</artifactId>
              <version>${scala.plugin.version}</version>
              <executions>
                <execution>
                  <goals>
                    <goal>compile</goal>
                    <goal>testCompile</goal>
                  </goals>
                </execution>
              </executions>
              <configuration>
                <args>
                  <arg>-Ywarn-unused</arg> <!-- if you need exactly RemoveUnused -->
                </args>
                <javacArgs>
                  <javacArg>-source</javacArg>
                  <javacArg>${java.version}</javacArg>
                  <javacArg>-target</javacArg>
                  <javacArg>${java.version}</javacArg>
                  <javacArg>-Xlint:all,-serial,-path,-try,-processing</javacArg>
                </javacArgs>
                <compilerPlugins>
                  <compilerPlugin>
                    <groupId>org.scalameta</groupId>
                    <artifactId>semanticdb-scalac_${scala.version}</artifactId>
                    <version>${semanticdb.version}</version>
                  </compilerPlugin>
                </compilerPlugins>
              </configuration>
            </plugin>
            <plugin>
              <groupId>io.github.evis</groupId>
              <artifactId>scalafix-maven-plugin_${scala.binary.version}</artifactId>
              <version>${scalafix-maven-plugin.version}</version>
            </plugin>
          </plugins>
        </pluginManagement>
      </build>
    </profile>
    <profile>
          <id>strict-warnings</id>
          <build>
              <plugins>
                  <plugin>
                      <groupId>net.alchim31.maven</groupId>
                      <artifactId>scala-maven-plugin</artifactId>
                      <configuration>
                          <args>
                              <arg>-deprecation</arg>
                              <arg>-unchecked</arg>
                              <arg>-feature</arg>
                              <arg>-Xlint:_</arg>
                              <arg>-Ywarn-dead-code</arg>
                              <arg>-Ywarn-numeric-widen</arg>
                              <arg>-Ywarn-value-discard</arg>
                              <arg>-Ywarn-unused:imports,patvars,privates,locals,params,-implicits</arg>
                              <arg>-Xfatal-warnings</arg>
                          </args>
                      </configuration>
                  </plugin>
              </plugins>
          </build>
    </profile>
  </profiles>

  <build>
    <pluginManagement>
      <plugins>
        <plugin>
          <groupId>net.alchim31.maven</groupId>
          <artifactId>scala-maven-plugin</artifactId>
          <version>${scala.plugin.version}</version>
          <executions>
            <execution>
              <id>eclipse-add-source</id>
              <goals>
                <goal>add-source</goal>
              </goals>
            </execution>
            <execution>
              <id>scala-compile-first</id>
              <phase>process-resources</phase>
              <goals>
                <goal>compile</goal>
                <goal>add-source</goal>
              </goals>
            </execution>
            <execution>
              <id>scala-test-compile-first</id>
              <phase>process-test-resources</phase>
              <goals>
                <goal>testCompile</goal>
              </goals>
            </execution>
          </executions>
          <configuration>
            <scalaVersion>${scala.version}</scalaVersion>
            <checkMultipleScalaVersions>true</checkMultipleScalaVersions>
            <failOnMultipleScalaVersions>true</failOnMultipleScalaVersions>
            <recompileMode>incremental</recompileMode>
            <args>
              <arg>-unchecked</arg>
              <arg>-deprecation</arg>
              <arg>-feature</arg>
              <arg>-explaintypes</arg>
              <arg>-Xlint:adapted-args</arg>
            </args>
            <jvmArgs>
              <jvmArg>-Xms1024m</jvmArg>
              <jvmArg>-Xmx1024m</jvmArg>
            </jvmArgs>
            <javacArgs>
              <javacArg>-source</javacArg>
              <javacArg>${maven.compiler.source}</javacArg>
              <javacArg>-target</javacArg>
              <javacArg>${maven.compiler.target}</javacArg>
              <javacArg>-Xlint:all,-serial,-path,-try,-processing</javacArg>
            </javacArgs>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.scalatest</groupId>
          <artifactId>scalatest-maven-plugin</artifactId>
          <version>${scalatest-maven-plugin.version}</version>
          <configuration>
            <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
            <junitxml>.</junitxml>
            <filereports>SparkTestSuite.txt</filereports>
            <stdout>D</stdout>
            <stderr/>
            <tagsToExclude>org.apache.comet.IntegrationTestSuite</tagsToExclude>
            <systemProperties>
              <!-- emit test logs to target/unit-tests.log -->
              <log4j.configurationFile>file:src/test/resources/log4j2.properties</log4j.configurationFile>
              <java.awt.headless>true</java.awt.headless>
              <java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
            </systemProperties>
          </configuration>
          <executions>
            <execution>
              <id>test</id>
              <goals>
                <goal>test</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-shade-plugin</artifactId>
          <version>${maven-shade-plugin.version}</version>
          <dependencies>
            <dependency>
              <groupId>org.ow2.asm</groupId>
              <artifactId>asm</artifactId>
              <version>${asm.version}</version>
            </dependency>
            <dependency>
              <groupId>org.ow2.asm</groupId>
              <artifactId>asm-commons</artifactId>
              <version>${asm.version}</version>
            </dependency>
          </dependencies>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-surefire-plugin</artifactId>
          <version>${maven-surefire-plugin.version}</version>
          <configuration>
            <systemPropertyVariables>
              <log4j.configurationFile>file:src/test/resources/log4j2.properties</log4j.configurationFile>
            </systemPropertyVariables>
            <failIfNoSpecifiedTests>false</failIfNoSpecifiedTests>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-source-plugin</artifactId>
          <version>${maven-source-plugin.version}</version>
          <configuration>
            <attach>true</attach>
          </configuration>
          <executions>
            <execution>
              <id>create-source-jar</id>
              <goals>
                <goal>jar-no-fork</goal>
                <goal>test-jar-no-fork</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-compiler-plugin</artifactId>
          <version>${maven-compiler-plugin.version}</version>
          <configuration>
            <source>${java.version}</source>
            <target>${java.version}</target>
            <skipMain>true</skipMain>
            <skip>true</skip>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-failsafe-plugin</artifactId>
          <version>${maven-failsafe-plugin.version}</version>
        </plugin>
        <plugin>
          <groupId>com.diffplug.spotless</groupId>
          <artifactId>spotless-maven-plugin</artifactId>
          <version>${spotless.version}</version>
          <configuration>
            <java>
              <toggleOffOn />
              <googleJavaFormat />
              <removeUnusedImports />
              <importOrder>
                <order>java|javax,scala,org,org.apache,com,org.apache.comet,\#,\#org.apache.comet</order>
              </importOrder>
              <licenseHeader>
                <file>${maven.multiModuleProjectDirectory}/dev/copyright/java-header.txt</file>
              </licenseHeader>
            </java>
            <scala>
              <includes>
                <include>src/main/scala/**/*.scala</include>
                <include>src/test/scala/**/*.scala</include>
                <include>src/test/celeborn-reflection-compatibility/**/*.scala</include>
                <!-- Include spark shim sources -->
                <include>src/main/spark-*/**/*.scala</include>
                <include>src/test/spark-*/**/*.scala</include>
              </includes>
              <toggleOffOn />
              <scalafmt>
                <version>${scalafmt.version}</version>
                <file>${maven.multiModuleProjectDirectory}/scalafmt.conf</file>
              </scalafmt>
              <licenseHeader>
                <file>${maven.multiModuleProjectDirectory}/dev/copyright/scala-header.txt</file>
              </licenseHeader>
            </scala>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.codehaus.mojo</groupId>
          <artifactId>flatten-maven-plugin</artifactId>
          <version>${flatten-maven-plugin.version}</version>
        </plugin>
        <plugin>
          <groupId>org.jacoco</groupId>
          <artifactId>jacoco-maven-plugin</artifactId>
          <version>${jacoco.version}</version>
        </plugin>
        <plugin>
          <groupId>org.codehaus.mojo</groupId>
          <artifactId>build-helper-maven-plugin</artifactId>
          <version>${build-helper-maven-plugin.version}</version>
        </plugin>
      </plugins>
    </pluginManagement>
    <plugins>
      <plugin>
        <groupId>org.scalastyle</groupId>
        <artifactId>scalastyle-maven-plugin</artifactId>
        <version>${scalastyle-maven-plugin.version}</version>
        <configuration>
          <verbose>false</verbose>
          <failOnViolation>true</failOnViolation>
          <includeTestSourceDirectory>false</includeTestSourceDirectory>
          <failOnWarning>false</failOnWarning>
          <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
          <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
          <configLocation>${maven.multiModuleProjectDirectory}/dev/scalastyle-config.xml</configLocation>
          <outputFile>${basedir}/target/scalastyle-output.xml</outputFile>
          <inputEncoding>${project.build.sourceEncoding}</inputEncoding>
          <outputEncoding>${project.reporting.outputEncoding}</outputEncoding>
        </configuration>
        <executions>
          <execution>
            <goals>
              <goal>check</goal>
            </goals>
            <phase>compile</phase>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>com.diffplug.spotless</groupId>
        <artifactId>spotless-maven-plugin</artifactId>
        <executions>
          <execution>
            <goals>
              <goal>check</goal>
            </goals>
            <phase>compile</phase>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-source-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-failsafe-plugin</artifactId>
        <executions>
          <execution>
            <goals>
              <goal>integration-test</goal>
              <goal>verify</goal>
            </goals>
            <configuration>
              <trimStackTrace>false</trimStackTrace>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.apache.rat</groupId>
        <artifactId>apache-rat-plugin</artifactId>
        <version>${apache-rat-plugin.version}</version>
        <executions>
          <execution>
            <phase>verify</phase>
            <goals>
              <goal>check</goal>
            </goals>
          </execution>
        </executions>
        <configuration>
          <consoleOutput>true</consoleOutput>
          <excludes>
            <exclude>**/*.iml</exclude>
            <exclude>**/*.log</exclude>
            <exclude>**/*.md.vm</exclude>
            <exclude>**/.classpath</exclude>
            <exclude>**/.project</exclude>
            <exclude>**/.settings/**</exclude>
            <exclude>**/build/**</exclude>
            <exclude>**/target/**</exclude>
            <exclude>**/apache-spark/**</exclude>
            <exclude>**/apache-iceberg/**</exclude>
            <exclude>.dockerignore</exclude>
            <exclude>.git/**</exclude>
            <exclude>.github/**</exclude>
            <exclude>.gitignore</exclude>
            <exclude>.gitmodules</exclude>
            <exclude>**/.idea/**</exclude>
            <exclude>**/dependency-reduced-pom.xml</exclude>
            <exclude>**/testdata/**</exclude>
            <exclude>**/.lldbinit</exclude>
            <exclude>rust-toolchain</exclude>
            <exclude>Makefile</exclude>
            <exclude>dev/Dockerfile*</exclude>
            <exclude>dev/diffs/**</exclude>
            <exclude>dev/deploy-file</exclude>
            <exclude>**/test/resources/**</exclude>
            <exclude>**/benchmarks/*.txt</exclude>
            <exclude>**/inspections/*.txt</exclude>
            <exclude>tpcds-kit/**</exclude>
            <exclude>tpcds-sf-1/**</exclude>
            <exclude>tpch/**</exclude>
            <exclude>docs/*.txt</exclude>
            <exclude>docs/logos/*.png</exclude>
            <exclude>docs/logos/*.svg</exclude>
            <exclude>docs/source/_static/images/**</exclude>
            <exclude>docs/source/contributor-guide/*.svg</exclude>
            <exclude>dev/release/rat_exclude_files.txt</exclude>
            <exclude>dev/release/requirements.txt</exclude>
            <exclude>dev/release/venv/**</exclude>
            <exclude>dev/release/comet-rm/workdir/**</exclude>
            <exclude>dev/dist/**</exclude>
            <exclude>dev/release/rat.txt</exclude>
            <exclude>dev/release/filtered_rat.txt</exclude>
            <exclude>dev/release/*.jar</exclude>
            <exclude>native/proto/src/generated/**</exclude>
            <exclude>benchmarks/tpc/queries/**</exclude>
            <exclude>benchmarks/results/**/*.txt</exclude>
            <exclude>.ai/**</exclude>
            <exclude>.claude/**</exclude>
            <exclude>docs/superpowers/**</exclude>
          </excludes>
        </configuration>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-enforcer-plugin</artifactId>
        <version>${maven-enforcer-plugin.version}</version>
        <executions>
          <execution>
            <id>require-java-17</id>
            <goals>
              <goal>enforce</goal>
            </goals>
            <configuration>
              <rules>
                <requireJavaVersion>
                  <version>[17,)</version>
                  <message>Comet requires JDK 17 or later to build.</message>
                </requireJavaVersion>
              </rules>
            </configuration>
          </execution>
          <execution>
            <id>no-duplicate-declared-dependencies</id>
            <goals>
              <goal>enforce</goal>
            </goals>
            <configuration>
              <rules>
                <banCircularDependencies/>
                <banDuplicatePomDependencyVersions/>
                <banDuplicateClasses>
                  <scopes>
                    <scope>compile</scope>
                    <scope>provided</scope>
                  </scopes>
                  <ignoreClasses>
                    <ignoreClass>org.apache.spark.unused.UnusedStubClass</ignoreClass>
                  </ignoreClasses>
                  <dependencies>
                    <dependency>
                      <groupId>org.apache.spark</groupId>
                      <artifactId>spark-sql_${scala.binary.version}</artifactId>
                      <ignoreClasses>
                        <!-- the followings conflict between spark-sql and spark-sql:test -->
                        <ignoreClass>javax.annotation.meta.TypeQualifier</ignoreClass>
                        <ignoreClass>javax.annotation.Nonnull</ignoreClass>
                        <ignoreClass>javax.annotation.meta.When</ignoreClass>
                        <ignoreClass>javax.annotation.Nonnull$Checker</ignoreClass>
                        <ignoreClass>javax.annotation.meta.TypeQualifierValidator</ignoreClass>
                        <!-- this class is not properly excluded from comet-spark right now -->
                        <ignoreClass>org.apache.parquet.filter2.predicate.SparkFilterApi</ignoreClass>
                        <!-- we explicitly include a duplicate to allow older versions of Spark -->
                        <!-- this can be removed once we no longer support spark 3.x  -->
                        <ignoreClass>org.apache.spark.sql.ExtendedExplainGenerator</ignoreClass>
                      </ignoreClasses>
                    </dependency>
                    <dependency>
                      <groupId>com.google.code.findbugs</groupId>
                      <artifactId>jsr305</artifactId>
                      <ignoreClasses>
                        <!-- the followings conflict between spark-sql and findbugs -->
                        <ignoreClass>javax.annotation.meta.TypeQualifier</ignoreClass>
                        <ignoreClass>javax.annotation.Nonnull</ignoreClass>
                        <ignoreClass>javax.annotation.meta.When</ignoreClass>
                        <ignoreClass>javax.annotation.Nonnull$Checker</ignoreClass>
                        <ignoreClass>javax.annotation.meta.TypeQualifierValidator</ignoreClass>
                        <ignoreClass>javax.annotation.Nullable</ignoreClass>
                        <ignoreClass>javax.annotation.meta.TypeQualifierNickname</ignoreClass>
                      </ignoreClasses>
                    </dependency>
                    <dependency>
                      <groupId>com.google.guava</groupId>
                      <artifactId>guava</artifactId>
                      <ignoreClasses>
                        <ignoreClass>com.google.thirdparty.publicsuffix.TrieParser</ignoreClass>
                        <ignoreClass>com.google.thirdparty.publicsuffix.PublicSuffixPatterns</ignoreClass>
                        <ignoreClass>com.google.thirdparty.publicsuffix.PublicSuffixType</ignoreClass>
                      </ignoreClasses>
                    </dependency>
                    <dependency>
                      <groupId>org.apache.datasketches</groupId>
                      <artifactId>datasketches-memory</artifactId>
                      <ignoreClasses>
                        <!-- Spark 4.2.0 bundles its own copy of this class in spark-catalyst,
                             which collides with the transitive datasketches-memory jar. -->
                        <ignoreClass>org.apache.datasketches.memory.internal.ResourceImpl</ignoreClass>
                      </ignoreClasses>
                    </dependency>
                  </dependencies>
                  <findAllDuplicates>true</findAllDuplicates>
                  <ignoreWhenIdentical>true</ignoreWhenIdentical>

                </banDuplicateClasses>
              </rules>
            </configuration>
          </execution>
        </executions>
        <dependencies>
          <dependency>
            <groupId>org.codehaus.mojo</groupId>
            <artifactId>extra-enforcer-rules</artifactId>
            <version>${extra-enforcer-rules.version}</version>
          </dependency>
        </dependencies>
      </plugin>
      <plugin>
        <groupId>org.codehaus.mojo</groupId>
        <artifactId>flatten-maven-plugin</artifactId>
        <executions>
          <!-- enable flattening -->
          <execution>
            <id>flatten</id>
            <phase>process-resources</phase>
            <goals>
              <goal>flatten</goal>
            </goals>
          </execution>
          <!-- ensure proper cleanup -->
          <execution>
            <id>flatten.clean</id>
            <phase>clean</phase>
            <goals>
              <goal>clean</goal>
            </goals>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.jacoco</groupId>
        <artifactId>jacoco-maven-plugin</artifactId>
        <executions>
          <execution>
            <id>default-prepare-agent</id>
            <goals>
              <goal>prepare-agent</goal>
            </goals>
          </execution>
          <execution>
            <id>report</id>
            <phase>test</phase>
            <goals>
              <goal>report</goal>
            </goals>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>
</project>