pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/pom.xml

<?xml version="1.0"?>
<!--

    Licensed to the Apache Software Foundation (ASF) under one
    or more contributor license agreements.  See the NOTICE file
    distributed with this work for additional information
    regarding copyright ownership.  The ASF licenses this file
    to you under the Apache License, Version 2.0 (the
    "License"); you may not use this file except in compliance
    with the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing,
    software distributed under the License is distributed on an
    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    KIND, either express or implied.  See the License for the
    specific language governing permissions and limitations
    under the License.

-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
    <artifactId>v0_deprecated</artifactId>
    <groupId>org.apache.pinot</groupId>
    <version>0.12.1</version>
    <relativePath>..</relativePath>
  </parent>
  <artifactId>pinot-spark</artifactId>
  <name>Pinot Spark</name>
  <url>https://pinot.apache.org/</url>
  <properties>
    <pinot.root>${basedir}/../../../..</pinot.root>
    <spark.version>2.4.7</spark.version>
    <scala.binary.version>2.12</scala.binary.version>
    <scala.version>2.12.15</scala.version>
    <hadoop.version>2.10.1</hadoop.version>
    <phase.prop>package</phase.prop>
  </properties>
  <profiles>
    <profile>
      <id>build-shaded-jar</id>
      <activation>
        <activeByDefault>false</activeByDefault>
      </activation>
      <build>
        <plugins>
          <plugin>
            <artifactId>maven-shade-plugin</artifactId>
            <version>3.2.1</version>
            <executions>
              <execution>
                <phase>package</phase>
                <goals>
                  <goal>shade</goal>
                </goals>
                <configuration>
                  <!--
                  Usually in hadoop environment, there are multiple jars with different versions.
                  Most of the NoSuchMethodExceptions are caused by class loading conflicts.
                  Class relocation ensures the reference of certain packages/classes in Pinot code to
                  shaded libs, e.g. jackson or guava.
                  Ref: https://maven.apache.org/plugins/maven-shade-plugin/examples/class-relocation.html
                  -->
                  <relocations>
                    <relocation>
                      <pattern>com.google.common.base</pattern>
                      <shadedPattern>${shade.prefix}.com.google.common.base</shadedPattern>
                    </relocation>
                    <relocation>
                      <pattern>com.google.common.cache</pattern>
                      <shadedPattern>${shade.prefix}.com.google.common.cache</shadedPattern>
                    </relocation>
                    <relocation>
                      <pattern>com.fasterxml.jackson</pattern>
                      <shadedPattern>${shade.prefix}.com.fasterxml.jackson</shadedPattern>
                    </relocation>
                    <relocation>
                      <pattern>org.apache.http</pattern>
                      <shadedPattern>${shade.prefix}.org.apache.http</shadedPattern>
                    </relocation>
                  </relocations>
                  <transformers>
                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
                    <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                      <mainClass>org.apache.pinot.spark.PinotSparkJobLauncher</mainClass>
                    </transformer>
                  </transformers>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
  </profiles>
  <dependencies>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-ingestion-common</artifactId>
      <version>${project.version}</version>
      <exclusions>
        <exclusion>
          <groupId>org.apache.pinot</groupId>
          <artifactId>pinot-common</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-core</artifactId>
      <exclusions>
        <exclusion>
          <groupId>commons-lang</groupId>
          <artifactId>commons-lang</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.pinot</groupId>
          <artifactId>pinot-common</artifactId>
        </exclusion>
        <exclusion>
          <groupId>com.thoughtworks.paranamer</groupId>
          <artifactId>paranamer</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-common</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-parquet</artifactId>
      <version>${project.version}</version>
      <exclusions>
        <exclusion>
          <groupId>org.apache.pinot</groupId>
          <artifactId>pinot-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>com.thoughtworks.paranamer</groupId>
          <artifactId>paranamer</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>hadoop-common</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-thrift</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-csv</artifactId>
      <version>${project.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-orc</artifactId>
      <version>${project.version}</version>
      <exclusions>
        <exclusion>
          <groupId>org.apache.pinot</groupId>
          <artifactId>pinot-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>hadoop-mapreduce-client-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>hadoop-hdfs</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>hadoop-common</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.pinot</groupId>
      <artifactId>pinot-protobuf</artifactId>
      <version>${project.version}</version>
      <exclusions>
        <exclusion>
          <groupId>com.google.protobuf</groupId>
          <artifactId>protobuf-java</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>commons-logging</groupId>
      <artifactId>commons-logging</artifactId>
    </dependency>
    <dependency>
      <groupId>commons-lang</groupId>
      <artifactId>commons-lang</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.commons</groupId>
      <artifactId>commons-math3</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-core</artifactId>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-databind</artifactId>
    </dependency>
    <!--Spark-->
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-hdfs</artifactId>
      <version>${hadoop.version}</version>
      <scope>compile</scope>
      <exclusions>
        <exclusion>
          <groupId>io.netty</groupId>
          <artifactId>netty</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>${hadoop.version}</version>
      <exclusions>
        <exclusion>
          <groupId>net.minidev</groupId>
          <artifactId>json-smart</artifactId>
        </exclusion>
        <exclusion>
          <groupId>net.java.dev.jets3t</groupId>
          <artifactId>jets3t</artifactId>
        </exclusion>
      </exclusions>
      <scope>compile</scope>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.module</groupId>
      <artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
      <scope>provided</scope>
    </dependency>

    <!--Test-->
    <dependency>
      <groupId>org.mockito</groupId>
      <artifactId>mockito-core</artifactId>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <groupId>org.objenesis</groupId>
          <artifactId>objenesis</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <!--Spark Test-->
    <dependency>
      <groupId>com.holdenkarau</groupId>
      <artifactId>spark-testing-base_${scala.binary.version}</artifactId>
      <version>${spark.version}_1.1.1</version>
      <exclusions>
        <exclusion>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>*</artifactId>
        </exclusion>
        <!-- We have to explicitly exclude spark-yarn, due to a bug in the current
             version of the maven-enforcer-plugin, where it doesn't handle
             wildcard exclusions in transitive dependencies. So projects that
             depend on this project wind up not excluding spark-yarn. -->
        <exclusion>
          <groupId>org.apache.spark</groupId>
          <artifactId>spark-yarn_${scala.binary.version}</artifactId>
        </exclusion>
        <exclusion>
          <groupId>net.minidev</groupId>
          <artifactId>json-smart</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.spark</groupId>
          <artifactId>spark-tags_${scala.binary.version}</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.zookeeper</groupId>
          <artifactId>zookeeper</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.antlr</groupId>
          <artifactId>antlr-runtime</artifactId>
        </exclusion>
        <exclusion>
          <groupId>net.java.dev.jets3t</groupId>
          <artifactId>jets3t</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.scala-lang.modules</groupId>
          <artifactId>scala-xml_${scala.binary.version}</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.spark</groupId>
          <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.derby</groupId>
          <artifactId>derby</artifactId>
        </exclusion>
        <exclusion>
          <groupId>commons-pool</groupId>
          <artifactId>commons-pool</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.scala-lang</groupId>
          <artifactId>scala-library</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.scala-lang</groupId>
          <artifactId>scala-reflect</artifactId>
        </exclusion>
      </exclusions>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-yarn-api</artifactId>
      <version>${hadoop.version}</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client</artifactId>
      <version>${hadoop.version}</version>
      <exclusions>
        <exclusion>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>hadoop-common</artifactId>
        </exclusion>
      </exclusions>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-yarn-server-web-proxy</artifactId>
      <version>${hadoop.version}</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-yarn-client</artifactId>
      <version>${hadoop.version}</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.antlr</groupId>
      <artifactId>antlr-runtime</artifactId>
      <version>3.4</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>net.java.dev.jets3t</groupId>
      <artifactId>jets3t</artifactId>
      <version>0.9.3</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.scala-lang.modules</groupId>
      <artifactId>scala-xml_${scala.binary.version}</artifactId>
      <version>1.0.6</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <groupId>org.scala-lang</groupId>
          <artifactId>scala-library</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-reflect</artifactId>
      <version>${scala.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_${scala.binary.version}</artifactId>
      <version>${spark.version}</version>
      <exclusions>
        <exclusion>
          <groupId>org.slf4j</groupId>
          <artifactId>slf4j-log4j12</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.curator</groupId>
          <artifactId>curator-recipes</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.scala-lang</groupId>
          <artifactId>scala-library</artifactId>
        </exclusion>
        <exclusion>
          <groupId>io.netty</groupId>
          <artifactId>netty</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_${scala.binary.version}</artifactId>
      <version>${spark.version}</version>
      <exclusions>
        <exclusion>
          <groupId>org.apache.parquet</groupId>
          <artifactId>parquet-column</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.spark</groupId>
          <artifactId>spark-tags_${scala.binary.version}</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.spark</groupId>
          <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.apache.orc</groupId>
          <artifactId>orc-core</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.scala-lang</groupId>
          <artifactId>scala-reflect</artifactId>
        </exclusion>
      </exclusions>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
      <version>${spark.version}</version>
      <exclusions>
        <exclusion>
          <groupId>org.apache.spark</groupId>
          <artifactId>spark-tags_${scala.binary.version}</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.objenesis</groupId>
          <artifactId>objenesis</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.scala-lang</groupId>
          <artifactId>scala-library</artifactId>
        </exclusion>
        <exclusion>
          <groupId>org.scala-lang</groupId>
          <artifactId>scala-reflect</artifactId>
        </exclusion>
      </exclusions>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.derby</groupId>
      <artifactId>derby</artifactId>
      <version>10.12.1.1</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-avro_${scala.binary.version}</artifactId>
      <version>${spark.version}</version>
      <scope>test</scope>
    </dependency>
  </dependencies>
</project>