spark/pom.xml

<?xml version="1.0" encoding="UTF-8"?>

<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
-->


<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
    <groupId>org.apache.datafusion</groupId>
    <artifactId>comet-parent-spark${spark.version.short}_${scala.binary.version}</artifactId>
    <version>0.15.0-SNAPSHOT</version>
    <relativePath>../pom.xml</relativePath>
  </parent>

  <artifactId>comet-spark-spark${spark.version.short}_${scala.binary.version}</artifactId>
  <name>comet-spark</name>

  <properties>
    <!-- Reverse default (skip installation), and then enable only for child modules -->
    <maven.deploy.skip>false</maven.deploy.skip>
  </properties>

  <dependencies>
    <dependency>
      <groupId>org.apache.datafusion</groupId>
      <artifactId>comet-common-spark${spark.version.short}_${scala.binary.version}</artifactId>
      <version>${project.version}</version>
      <exclusions>
        <exclusion>
          <groupId>org.apache.arrow</groupId>
          <artifactId>*</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_${scala.binary.version}</artifactId>
    </dependency>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
    </dependency>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-reflect</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>com.google.protobuf</groupId>
      <artifactId>protobuf-java</artifactId>
    </dependency>
    <dependency>
      <groupId>org.scalatest</groupId>
      <artifactId>scalatest_${scala.binary.version}</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.scalatestplus</groupId>
      <artifactId>junit-4-13_${scala.binary.version}</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_${scala.binary.version}</artifactId>
      <classifier>tests</classifier>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
      <classifier>tests</classifier>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_${scala.binary.version}</artifactId>
      <classifier>tests</classifier>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-hadoop-cloud_${scala.binary.version}</artifactId>
      <classifier>tests</classifier>
    </dependency>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>com.google.guava</groupId>
      <artifactId>guava</artifactId>
      <exclusions>
        <exclusion>
          <groupId>*</groupId>
          <artifactId>*</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.codehaus.jackson</groupId>
      <artifactId>jackson-mapper-asl</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.parquet</groupId>
      <artifactId>parquet-hadoop</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.parquet</groupId>
      <artifactId>parquet-hadoop</artifactId>
      <classifier>tests</classifier>
      <!-- Note we don't use test scope for this artifact. This is because it's only needed
         to provide InMemoryKMS class that is shaded below, to make Spark test happy. -->
    </dependency>
    <!-- We shade & relocate Arrow dependencies in comet-common, so comet-spark module no longer
         depends on Arrow. However, when running `mvn test` we still need Arrow classes in the
         classpath, since the Maven shading happens in 'package' phase which is after 'test' -->
    <dependency>
      <groupId>org.apache.arrow</groupId>
      <artifactId>arrow-memory-unsafe</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.arrow</groupId>
      <artifactId>arrow-c-data</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client-minicluster</artifactId>
      <scope>test</scope>
      <exclusions>
        <!-- hadoop clients are provided by spark -->
        <exclusion>
          <artifactId>hadoop-client-api</artifactId>
          <groupId>org.apache.hadoop</groupId>
        </exclusion>
        <exclusion>
          <artifactId>hadoop-client-runtime</artifactId>
          <groupId>org.apache.hadoop</groupId>
        </exclusion>
        <exclusion>
          <artifactId>snappy-java</artifactId>
          <groupId>org.xerial.snappy</groupId>
        </exclusion>
        <exclusion>
          <artifactId>junit</artifactId>
          <groupId>junit</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.testcontainers</groupId>
      <artifactId>minio</artifactId>
    </dependency>
    <!-- AWS SDK modules required by Iceberg's S3FileIO (see parent pom for details) -->
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>s3</artifactId>
    </dependency>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>sts</artifactId>
    </dependency>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>dynamodb</artifactId>
    </dependency>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>glue</artifactId>
    </dependency>
    <dependency>
      <groupId>software.amazon.awssdk</groupId>
      <artifactId>kms</artifactId>
    </dependency>
    <!-- Jetty and Iceberg dependencies for testing native Iceberg scan -->
    <!-- Note: The specific versions are defined in profiles below based on Spark version -->
  </dependencies>

  <profiles>
    <!-- Iceberg dependencies vary by Spark version (Iceberg 1.8.1 for Spark 3.x, 1.10.0 for Spark 4.0) -->
    <profile>
      <id>spark-3.4</id>
      <dependencies>
        <dependency>
          <groupId>org.apache.iceberg</groupId>
          <artifactId>iceberg-spark-runtime-${spark.version.short}_${scala.binary.version}</artifactId>
          <version>1.5.2</version>
          <scope>test</scope>
        </dependency>
        <!-- Jetty 9.4.x for Spark 3.4 (JDK 11, javax.* packages) -->
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-server</artifactId>
          <version>9.4.53.v20231009</version>
          <scope>test</scope>
        </dependency>
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-servlet</artifactId>
          <version>9.4.53.v20231009</version>
          <scope>test</scope>
        </dependency>
      </dependencies>
    </profile>

    <profile>
      <id>spark-3.5</id>
      <activation>
        <activeByDefault>true</activeByDefault>
      </activation>
      <dependencies>
        <dependency>
          <groupId>org.apache.iceberg</groupId>
          <artifactId>iceberg-spark-runtime-${spark.version.short}_${scala.binary.version}</artifactId>
          <version>1.8.1</version>
          <scope>test</scope>
        </dependency>
        <!-- Jetty 9.4.x for Spark 3.5 (JDK 11, javax.* packages) -->
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-server</artifactId>
          <version>9.4.53.v20231009</version>
          <scope>test</scope>
        </dependency>
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-servlet</artifactId>
          <version>9.4.53.v20231009</version>
          <scope>test</scope>
        </dependency>
      </dependencies>
    </profile>

    <profile>
      <id>spark-4.0</id>
      <dependencies>
        <dependency>
          <groupId>org.apache.iceberg</groupId>
          <artifactId>iceberg-spark-runtime-${spark.version.short}_${scala.binary.version}</artifactId>
          <version>1.10.0</version>
          <scope>test</scope>
        </dependency>
        <!-- Jetty 11.x for Spark 4.0 (jakarta.servlet) -->
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-server</artifactId>
          <version>11.0.24</version>
          <scope>test</scope>
        </dependency>
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-servlet</artifactId>
          <version>11.0.24</version>
          <scope>test</scope>
        </dependency>
      </dependencies>
    </profile>
    <profile>
      <id>generate-docs</id>
      <build>
        <plugins>
          <plugin>
            <groupId>org.codehaus.mojo</groupId>
            <artifactId>exec-maven-plugin</artifactId>
            <version>${exec-maven-plugin.version}</version>
            <executions>
              <execution>
                <id>generate-user-guide-reference-docs</id>
                <phase>package</phase>
                <goals>
                  <goal>java</goal>
                </goals>
                <configuration>
                  <mainClass>org.apache.comet.GenerateDocs</mainClass>
                  <arguments>
                    <argument>${project.parent.basedir}/docs/source/user-guide/latest/</argument>
                  </arguments>
                  <classpathScope>compile</classpathScope>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
  </profiles>

  <build>
    <plugins>
      <plugin>
        <groupId>com.github.os72</groupId>
        <artifactId>protoc-jar-maven-plugin</artifactId>
        <version>${protoc-jar-maven-plugin.version}</version>
        <executions>
          <execution>
            <phase>generate-sources</phase>
            <goals>
              <goal>run</goal>
            </goals>
            <configuration>
              <protocArtifact>com.google.protobuf:protoc:${protobuf.version}</protocArtifact>
              <inputDirectories>
                <include>../native/proto/src/proto</include>
              </inputDirectories>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.scalatest</groupId>
        <artifactId>scalatest-maven-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>shade</goal>
            </goals>
            <configuration>
              <createSourcesJar>true</createSourcesJar>
              <shadeSourcesContent>true</shadeSourcesContent>
              <shadedArtifactAttached>false</shadedArtifactAttached>
              <createDependencyReducedPom>true</createDependencyReducedPom>
              <artifactSet>
                <includes>
                  <include>org.apache.datafusion:comet-common-spark${spark.version.short}_${scala.binary.version}</include>
                  <!-- Relocate Protobuf since Spark uses 2.5.0 while Comet uses 3.x -->
                  <include>com.google.protobuf:protobuf-java</include>
                  <include>com.google.guava:guava</include>
                  <include>org.scala-lang.modules:scala-collection-compat_${scala.binary.version}</include>
                </includes>
              </artifactSet>
              <filters>
                <filter>
                  <artifact>*:*</artifact>
                  <excludes>
                    <exclude>**/*.proto</exclude>
                    <exclude>**/*.thrift</exclude>
                    <exclude>git.properties</exclude>
                    <exclude>log4j.properties</exclude>
                    <exclude>log4j2.properties</exclude>
                    <exclude>**/SparkFilterApi.*</exclude>
                  </excludes>
                </filter>
                <filter>
                  <artifact>org.apache.parquet:parquet-hadoop:tests</artifact>
                  <includes>
                    <!-- Used by Spark test `ParquetEncryptionSuite` -->
                    <include>org/apache/parquet/crypto/keytools/mocks/InMemoryKMS*</include>
                  </includes>
                </filter>
              </filters>
              <relocations>
                <relocation>
                  <pattern>com.google.protobuf</pattern>
                  <shadedPattern>${comet.shade.packageName}.protobuf</shadedPattern>
                </relocation>
                <relocation>
                  <pattern>com.google.common</pattern>
                  <shadedPattern>${comet.shade.packageName}.guava</shadedPattern>
                </relocation>
                <relocation>
                  <pattern>com.google.thirdparty</pattern>
                  <shadedPattern>${comet.shade.packageName}.guava.thirdparty</shadedPattern>
                </relocation>
              </relocations>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-failsafe-plugin</artifactId>
        <executions>
          <execution>
            <goals>
              <goal>integration-test</goal>
              <goal>verify</goal>
            </goals>
            <configuration>
              <trimStackTrace>false</trimStackTrace>
              <argLine>-ea ${extraJavaTestArgs}</argLine>
            </configuration>
          </execution>
        </executions>
      </plugin>
      <plugin>
        <groupId>net.alchim31.maven</groupId>
        <artifactId>scala-maven-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.codehaus.mojo</groupId>
        <artifactId>build-helper-maven-plugin</artifactId>
        <executions>
          <execution>
            <id>add-test-source</id>
            <phase>generate-test-sources</phase>
            <goals>
              <goal>add-test-source</goal>
            </goals>
            <configuration>
              <sources>
                <source>src/test/${shims.majorVerSrc}</source>
                <source>src/test/${shims.minorVerSrc}</source>
              </sources>
            </configuration>
          </execution>
          <execution>
            <id>add-shim-source</id>
            <phase>generate-sources</phase>
            <goals>
              <goal>add-source</goal>
            </goals>
            <configuration>
              <sources>
                <source>src/main/${shims.majorVerSrc}</source>
                <source>src/main/${shims.minorVerSrc}</source>
              </sources>
            </configuration>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>

</project>