spark/pom.xml

<?xml version="1.0" encoding="UTF-8"?>

<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
-->


<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
    <groupId>org.apache.datafusion</groupId>
    <artifactId>comet-parent-spark${spark.version.short}_${scala.binary.version}</artifactId>
    <version>1.1.0-SNAPSHOT</version>
    <relativePath>../pom.xml</relativePath>
  </parent>

  <artifactId>comet-spark-spark${spark.version.short}_${scala.binary.version}</artifactId>
  <name>comet-spark</name>

  <properties>
    <!-- Reverse default (skip installation), and then enable only for child modules -->
    <maven.deploy.skip>false</maven.deploy.skip>
  </properties>

  <dependencies>
    <dependency>
      <groupId>org.apache.datafusion</groupId>
      <artifactId>comet-common-spark${spark.version.short}_${scala.binary.version}</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_${scala.binary.version}</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.parquet</groupId>
      <artifactId>parquet-column</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.parquet</groupId>
      <artifactId>parquet-format-structures</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.arrow</groupId>
      <artifactId>arrow-vector</artifactId>
    </dependency>
    <dependency>
      <groupId>org.scala-lang.modules</groupId>
      <artifactId>scala-collection-compat_${scala.binary.version}</artifactId>
    </dependency>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
    </dependency>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-reflect</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>com.google.protobuf</groupId>
      <artifactId>protobuf-java</artifactId>
    </dependency>
    <dependency>
      <groupId>org.scalatest</groupId>
      <artifactId>scalatest_${scala.binary.version}</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.scalatestplus</groupId>
      <artifactId>junit-4-13_${scala.binary.version}</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_${scala.binary.version}</artifactId>
      <classifier>tests</classifier>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
      <classifier>tests</classifier>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_${scala.binary.version}</artifactId>
      <classifier>tests</classifier>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-hadoop-cloud_${scala.binary.version}</artifactId>
      <classifier>tests</classifier>
    </dependency>
    <dependency>
      <!-- Test-only: provides an embedded Hive metastore so Iceberg's type=hive catalog can
           create encrypted tables (only Hive wires the KMS-backed EncryptionManager). Version and
           scope come from the parent pom's dependencyManagement. -->
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-hive_${scala.binary.version}</artifactId>
    </dependency>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>com.google.guava</groupId>
      <artifactId>guava</artifactId>
      <exclusions>
        <exclusion>
          <groupId>*</groupId>
          <artifactId>*</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.codehaus.jackson</groupId>
      <artifactId>jackson-mapper-asl</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.parquet</groupId>
      <artifactId>parquet-hadoop</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.parquet</groupId>
      <artifactId>parquet-hadoop</artifactId>
      <classifier>tests</classifier>
      <!-- Note we don't use test scope for this artifact. This is because it's only needed
         to provide InMemoryKMS class that is shaded below, to make Spark test happy. -->
    </dependency>
    <!-- Arrow memory + C-data deps required at compile scope. Shading relocates these into
         org.apache.comet.shaded.arrow during the package phase to avoid colliding with Spark's
         own Arrow on the executor classpath. -->
    <dependency>
      <groupId>org.apache.arrow</groupId>
      <artifactId>arrow-memory-unsafe</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.arrow</groupId>
      <artifactId>arrow-c-data</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client-minicluster</artifactId>
      <scope>test</scope>
      <exclusions>
        <!-- hadoop clients are provided by spark -->
        <exclusion>
          <artifactId>hadoop-client-api</artifactId>
          <groupId>org.apache.hadoop</groupId>
        </exclusion>
        <exclusion>
          <artifactId>hadoop-client-runtime</artifactId>
          <groupId>org.apache.hadoop</groupId>
        </exclusion>
        <exclusion>
          <artifactId>snappy-java</artifactId>
          <groupId>org.xerial.snappy</groupId>
        </exclusion>
        <exclusion>
          <artifactId>junit</artifactId>
          <groupId>junit</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.testcontainers</groupId>
      <artifactId>minio</artifactId>
    </dependency>
    <!-- AWS SDK modules required by Iceberg's S3FileIO (see parent pom for details) -->
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>s3</artifactId>
    </dependency>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>sts</artifactId>
    </dependency>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>dynamodb</artifactId>
    </dependency>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>glue</artifactId>
    </dependency>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>kms</artifactId>
    </dependency>
    <!-- Jetty and Iceberg dependencies for testing native Iceberg scan -->
    <!-- Note: The specific versions are defined in profiles below based on Spark version -->
  </dependencies>

  <profiles>
    <!-- Iceberg dependencies vary by Spark version (Iceberg 1.8.1 for Spark 3.x, 1.10.0 for Spark 4.0, 1.11.0 for Spark 4.1) -->
    <profile>
      <id>spark-3.4</id>
      <dependencies>
        <dependency>
          <groupId>org.apache.iceberg</groupId>
          <artifactId>iceberg-spark-runtime-${spark.version.short}_${scala.binary.version}</artifactId>
          <version>1.5.2</version>
          <scope>test</scope>
        </dependency>
        <!-- Jetty 9.4.x for Spark 3.4 (JDK 11, javax.* packages) -->
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-server</artifactId>
          <version>9.4.53.v20231009</version>
          <scope>test</scope>
        </dependency>
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-servlet</artifactId>
          <version>9.4.53.v20231009</version>
          <scope>test</scope>
        </dependency>
      </dependencies>
    </profile>

    <profile>
      <id>spark-3.5</id>
      <dependencies>
        <dependency>
          <groupId>org.apache.iceberg</groupId>
          <artifactId>iceberg-spark-runtime-${spark.version.short}_${scala.binary.version}</artifactId>
          <version>1.8.1</version>
          <scope>test</scope>
        </dependency>
        <!-- Jetty 9.4.x for Spark 3.5 (JDK 11, javax.* packages) -->
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-server</artifactId>
          <version>9.4.53.v20231009</version>
          <scope>test</scope>
        </dependency>
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-servlet</artifactId>
          <version>9.4.53.v20231009</version>
          <scope>test</scope>
        </dependency>
      </dependencies>
    </profile>

    <profile>
      <id>spark-4.0</id>
      <dependencies>
        <dependency>
          <groupId>org.apache.iceberg</groupId>
          <artifactId>iceberg-spark-runtime-${spark.version.short}_${scala.binary.version}</artifactId>
          <version>1.10.0</version>
          <scope>test</scope>
        </dependency>
        <!-- Jetty 11.x for Spark 4.0 (jakarta.servlet) -->
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-server</artifactId>
          <version>11.0.24</version>
          <scope>test</scope>
        </dependency>
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-servlet</artifactId>
          <version>11.0.24</version>
          <scope>test</scope>
        </dependency>
      </dependencies>
    </profile>
    <profile>
      <id>spark-4.1</id>
      <activation>
        <activeByDefault>true</activeByDefault>
      </activation>
      <dependencies>
        <dependency>
          <groupId>org.apache.iceberg</groupId>
          <artifactId>iceberg-spark-runtime-${spark.version.short}_${scala.binary.version}</artifactId>
          <version>1.11.0</version>
          <scope>test</scope>
        </dependency>
        <!-- Jetty 11.x for Spark 4.1 (jakarta.servlet); matches Spark 4.1.3's jetty.version -->
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-server</artifactId>
          <version>11.0.26</version>
          <scope>test</scope>
        </dependency>
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-servlet</artifactId>
          <version>11.0.26</version>
          <scope>test</scope>
        </dependency>
      </dependencies>
    </profile>
    <profile>
      <id>spark-4.2</id>
      <dependencies>
        <!-- iceberg-spark-runtime-4.2 is not yet published; reuse the 4.0 runtime -->
        <dependency>
          <groupId>org.apache.iceberg</groupId>
          <artifactId>iceberg-spark-runtime-4.0_${scala.binary.version}</artifactId>
          <version>1.10.0</version>
          <scope>test</scope>
        </dependency>
        <!-- Jetty 11.x for Spark 4.2 (jakarta.servlet). Spark 4.2.0 itself ships Jetty 12, but
             the Iceberg REST catalog test helper needs jetty-servlet, which Jetty 12 replaced
             with the jetty-ee10-servlet artifact, so the test classpath stays on 11.x. -->
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-server</artifactId>
          <version>11.0.26</version>
          <scope>test</scope>
        </dependency>
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-servlet</artifactId>
          <version>11.0.26</version>
          <scope>test</scope>
        </dependency>
      </dependencies>
    </profile>
    <profile>
      <id>generate-docs</id>
      <build>
        <plugins>
          <plugin>
            <groupId>org.codehaus.mojo</groupId>
            <artifactId>exec-maven-plugin</artifactId>
            <version>${exec-maven-plugin.version}</version>
            <executions>
              <execution>
                <id>generate-user-guide-reference-docs</id>
                <phase>package</phase>
                <goals>
                  <goal>java</goal>
                </goals>
                <configuration>
                  <mainClass>org.apache.comet.GenerateDocs</mainClass>
                  <arguments>
                    <argument>${project.parent.basedir}/docs/source/user-guide/latest/</argument>
                  </arguments>
                  <classpathScope>compile</classpathScope>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
  </profiles>

  <build>
    <plugins>
      <plugin>
        <groupId>io.github.git-commit-id</groupId>
        <artifactId>git-commit-id-maven-plugin</artifactId>
        <version>${git-commit-id-maven-plugin.version}</version>
        <executions>
          <execution>
            <id>get-the-git-infos</id>
            <goals>
              <goal>revision</goal>
            </goals>
            <phase>initialize</phase>
          </execution>
        </executions>
        <configuration>
          <generateGitPropertiesFile>true</generateGitPropertiesFile>
          <generateGitPropertiesFilename>${project.build.outputDirectory}/comet-git-info.properties</generateGitPropertiesFilename>
          <commitIdGenerationMode>full</commitIdGenerationMode>
          <includeOnlyProperties>
            <includeOnlyProperty>^git.branch$</includeOnlyProperty>
            <includeOnlyProperty>^git.build.*$</includeOnlyProperty>
            <includeOnlyProperty>^git.commit.id.(abbrev|full)$</includeOnlyProperty>
            <includeOnlyProperty>^git.remote.*$</includeOnlyProperty>
          </includeOnlyProperties>
        </configuration>
      </plugin>
      <plugin>
        <groupId>com.github.os72</groupId>
        <artifactId>protoc-jar-maven-plugin</artifactId>
        <version>${protoc-jar-maven-plugin.version}</version>
        <executions>
          <execution>
            <phase>generate-sources</phase>
            <goals>
              <goal>run</goal>
            </goals>
            <configuration>
              <protocArtifact>com.google.protobuf:protoc:${protobuf.version}</protocArtifact>
              <inputDirectories>
                <include>../native/proto/src/proto</include>
              </inputDirectories>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.scalatest</groupId>
        <artifactId>scalatest-maven-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>shade</goal>
            </goals>
            <configuration>
              <createSourcesJar>true</createSourcesJar>
              <shadeSourcesContent>true</shadeSourcesContent>
              <shadedArtifactAttached>false</shadedArtifactAttached>
              <createDependencyReducedPom>true</createDependencyReducedPom>
              <artifactSet>
                <includes>
                  <!-- Bundle comet-common so the shaded uber-jar is self-contained -->
                  <include>org.apache.datafusion:comet-common-spark${spark.version.short}_${scala.binary.version}</include>
                  <!-- Shade & relocate Arrow to prevent conflicts with Spark's own Arrow -->
                  <include>org.apache.arrow:*</include>
                  <!-- Relocate Protobuf since Spark uses 2.5.0 while Comet uses 3.x -->
                  <include>com.google.protobuf:protobuf-java</include>
                  <include>com.google.guava:guava</include>
                </includes>
              </artifactSet>
              <filters>
                <filter>
                  <artifact>*:*</artifact>
                  <excludes>
                    <exclude>**/*.proto</exclude>
                    <exclude>**/*.thrift</exclude>
                    <exclude>git.properties</exclude>
                    <exclude>log4j.properties</exclude>
                    <exclude>log4j2.properties</exclude>
                    <exclude>arrow-git.properties</exclude>
                    <exclude>**/SparkFilterApi.*</exclude>
                  </excludes>
                </filter>
                <filter>
                  <artifact>org.apache.arrow:arrow-vector</artifact>
                  <excludes>
                    <!-- Comet doesn't need codegen templates on Arrow -->
                    <exclude>codegen/**</exclude>
                  </excludes>
                </filter>
                <filter>
                  <artifact>org.apache.parquet:parquet-hadoop:tests</artifact>
                  <includes>
                    <!-- Used by Spark test `ParquetEncryptionSuite` -->
                    <include>org/apache/parquet/crypto/keytools/mocks/InMemoryKMS*</include>
                  </includes>
                </filter>
              </filters>
              <relocations>
                <relocation>
                  <pattern>org.apache.arrow</pattern>
                  <shadedPattern>${comet.shade.packageName}.arrow</shadedPattern>
                  <excludes>
                    <!-- Keep the entire C Data Interface unshaded. Spark does not ship
                       arrow-c-data, so no classpath clash is possible. Shading a subset
                       splits package-private access between the shaded and original
                       packages (e.g. Data -> ArrayStreamExporter constructor) and breaks
                       JNI lookups in jni_wrapper.cc that reference these classes by literal
                       name. -->
                    <exclude>org/apache/arrow/c/**</exclude>
                  </excludes>
                </relocation>
                <relocation>
                  <pattern>com.google.protobuf</pattern>
                  <shadedPattern>${comet.shade.packageName}.protobuf</shadedPattern>
                </relocation>
                <relocation>
                  <pattern>com.google.common</pattern>
                  <shadedPattern>${comet.shade.packageName}.guava</shadedPattern>
                </relocation>
                <relocation>
                  <pattern>com.google.thirdparty</pattern>
                  <shadedPattern>${comet.shade.packageName}.guava.thirdparty</shadedPattern>
                </relocation>
              </relocations>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-failsafe-plugin</artifactId>
        <executions>
          <execution>
            <goals>
              <goal>integration-test</goal>
              <goal>verify</goal>
            </goals>
            <configuration>
              <trimStackTrace>false</trimStackTrace>
              <argLine>-ea ${extraJavaTestArgs}</argLine>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>net.alchim31.maven</groupId>
        <artifactId>scala-maven-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.codehaus.mojo</groupId>
        <artifactId>build-helper-maven-plugin</artifactId>
        <executions>
          <execution>
            <id>add-test-source</id>
            <phase>generate-test-sources</phase>
            <goals>
              <goal>add-test-source</goal>
            </goals>
            <configuration>
              <sources>
                <source>src/test/${shims.majorVerSrc}</source>
                <source>src/test/${shims.minorPlusVerSrc}</source>
                <source>src/test/${shims.minorVerSrc}</source>
              </sources>
            </configuration>
          </execution>
          <execution>
            <id>add-shim-source</id>
            <phase>generate-sources</phase>
            <goals>
              <goal>add-source</goal>
            </goals>
            <configuration>
              <sources>
                <source>src/main/${shims.majorVerSrc}</source>
                <source>src/main/${shims.minorPlusVerSrc}</source>
                <source>src/main/${shims.minorVerSrc}</source>
              </sources>
            </configuration>
          </execution>
        </executions>
      </plugin>
    </plugins>
    <resources>
      <resource>
        <directory>${project.basedir}/src/main/resources</directory>
      </resource>
      <resource>
        <directory>${project.basedir}/../native/target/x86_64-apple-darwin/release</directory>
        <includes>
          <include>libcomet.dylib</include>
        </includes>
        <targetPath>org/apache/comet/darwin/x86_64</targetPath>
      </resource>
      <resource>
        <directory>${project.basedir}/../native/target/aarch64-apple-darwin/release</directory>
        <includes>
          <include>libcomet.dylib</include>
        </includes>
        <targetPath>org/apache/comet/darwin/aarch64</targetPath>
      </resource>
      <resource>
        <directory>${jni.dir}</directory>
        <includes>
          <include>libcomet.dylib</include>
          <include>libcomet.so</include>
          <include>comet.dll</include>
        </includes>
        <targetPath>org/apache/comet/${platform}/${arch}</targetPath>
      </resource>
    </resources>
  </build>

</project>