runners/spark/pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<!--
    Licensed to the Apache Software Foundation (ASF) under one or more
    contributor license agreements.  See the NOTICE file distributed with
    this work for additional information regarding copyright ownership.
    The ASF licenses this file to You under the Apache License, Version 2.0
    (the "License"); you may not use this file except in compliance with
    the License.  You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

  <modelVersion>4.0.0</modelVersion>

  <parent>
    <groupId>org.apache.beam</groupId>
    <artifactId>beam-runners-parent</artifactId>
    <version>0.4.0</version>
    <relativePath>../pom.xml</relativePath>
  </parent>

  <artifactId>beam-runners-spark</artifactId>

  <name>Apache Beam :: Runners :: Spark</name>
  <packaging>jar</packaging>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
    <spark.version>1.6.3</spark.version>
    <hadoop.version>2.2.0</hadoop.version>
    <kafka.version>0.9.0.1</kafka.version>
    <dropwizard.metrics.version>3.1.2</dropwizard.metrics.version>
  </properties>

  <profiles>
    <profile>
      <id>jacoco</id>
      <build>
        <plugins>
          <plugin>
            <groupId>org.jacoco</groupId>
            <artifactId>jacoco-maven-plugin</artifactId>
          </plugin>
        </plugins>
      </build>
    </profile>

    <profile>
      <!-- This profile adds execution of RunnableOnService integration tests
           against a local Spark endpoint. -->
      <id>local-runnable-on-service-tests</id>
      <activation><activeByDefault>false</activeByDefault></activation>
      <build>
        <plugins>
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-surefire-plugin</artifactId>
            <executions>
              <execution>
                <id>runnable-on-service-tests</id>
                <phase>integration-test</phase>
                <goals>
                  <goal>test</goal>
                </goals>
                <configuration>
                  <groups>org.apache.beam.sdk.testing.RunnableOnService</groups>
                  <excludedGroups>
                    org.apache.beam.sdk.testing.UsesStatefulParDo,
                    org.apache.beam.sdk.testing.UsesTimersInParDo,
                    org.apache.beam.sdk.testing.UsesSplittableParDo
                  </excludedGroups>
                  <forkCount>1</forkCount>
                  <reuseForks>false</reuseForks>
                  <failIfNoTests>true</failIfNoTests>
                  <dependenciesToScan>
                    <dependency>org.apache.beam:beam-sdks-java-core</dependency>
                  </dependenciesToScan>
                  <systemPropertyVariables>
                    <beamTestPipelineOptions>
                      [
                        "--runner=TestSparkRunner",
                        "--streaming=false",
                        "--enableSparkMetricSinks=false"
                      ]
                    </beamTestPipelineOptions>
                    <beam.spark.test.reuseSparkContext>true</beam.spark.test.reuseSparkContext>
                    <spark.ui.enabled>false</spark.ui.enabled>
                  </systemPropertyVariables>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
  </profiles>

  <dependencies>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.10</artifactId>
      <version>${spark.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_2.10</artifactId>
      <version>${spark.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-network-common_2.10</artifactId>
      <version>${spark.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>${hadoop.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-mapreduce-client-core</artifactId>
      <version>${hadoop.version}</version>
      <scope>provided</scope>
    </dependency>
    <!-- Kryo bugfix version needed due to a state re-use issue in Kryo version 2.21 used in Spark 1.x
    See: https://issues.apache.org/jira/browse/SPARK-7708
    See: https://github.com/EsotericSoftware/kryo/issues/312
    -->
    <dependency>
      <groupId>com.esotericsoftware.kryo</groupId>
      <artifactId>kryo</artifactId>
      <version>2.21.1</version>
    </dependency>
    <dependency>
      <groupId>com.google.code.findbugs</groupId>
      <artifactId>jsr305</artifactId>
      <version>1.3.9</version>
    </dependency>
    <dependency>
      <groupId>com.google.guava</groupId>
      <artifactId>guava</artifactId>
    </dependency>
    <dependency>
      <groupId>com.google.auto.service</groupId>
      <artifactId>auto-service</artifactId>
      <optional>true</optional>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-core</artifactId>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-annotations</artifactId>
    </dependency>
      <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-databind</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.avro</groupId>
      <artifactId>avro</artifactId>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-api</artifactId>
    </dependency>
    <dependency>
      <groupId>joda-time</groupId>
      <artifactId>joda-time</artifactId>
    </dependency>
    <dependency>
      <groupId>com.google.http-client</groupId>
      <artifactId>google-http-client</artifactId>
      <version>${google-clients.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.commons</groupId>
      <artifactId>commons-compress</artifactId>
      <version>1.9</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>commons-io</groupId>
      <artifactId>commons-io</artifactId>
      <version>2.4</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.zookeeper</groupId>
      <artifactId>zookeeper</artifactId>
      <version>3.4.6</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>2.10.5</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-sdks-java-core</artifactId>
      <exclusions>
        <!-- Use Hadoop/Spark's backend logger instead of jdk14 for tests -->
        <exclusion>
          <groupId>org.slf4j</groupId>
          <artifactId>slf4j-jdk14</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-runners-core-java</artifactId>
      <exclusions>
        <!-- Use Hadoop/Spark's backend logger instead of jdk14 for tests -->
        <exclusion>
          <groupId>org.slf4j</groupId>
          <artifactId>slf4j-jdk14</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.avro</groupId>
      <artifactId>avro-mapred</artifactId>
      <version>${avro.version}</version>
      <classifier>hadoop2</classifier>
      <exclusions>
        <!-- exclude old Jetty version of servlet API -->
        <exclusion>
          <groupId>org.mortbay.jetty</groupId>
          <artifactId>servlet-api</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>io.dropwizard.metrics</groupId>
      <artifactId>metrics-core</artifactId>
      <version>${dropwizard.metrics.version}</version>
    </dependency>
    <dependency>
      <groupId>org.reflections</groupId>
      <artifactId>reflections</artifactId>
      <version>0.9.10</version>
    </dependency>

    <!-- KafkaIO -->
    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-sdks-java-io-kafka</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka-clients</artifactId>
      <version>0.9.0.1</version>
      <scope>test</scope>
    </dependency>

    <!-- test dependencies -->
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>hamcrest-core</artifactId>
          <groupId>org.hamcrest</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.hamcrest</groupId>
      <artifactId>hamcrest-all</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka_2.10</artifactId>
      <version>${kafka.version}</version>
      <scope>test</scope>
    </dependency>

    <!-- Depend on test jar to scan for RunnableOnService tests -->
    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-sdks-java-core</artifactId>
      <classifier>tests</classifier>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <groupId>org.slf4j</groupId>
          <artifactId>slf4j-jdk14</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

    <dependency>
      <groupId>org.mockito</groupId>
      <artifactId>mockito-all</artifactId>
      <scope>test</scope>
    </dependency>
  </dependencies>

  <build>
    <pluginManagement>
      <plugins>
        <!-- BEAM-931 -->
        <plugin>
          <groupId>org.codehaus.mojo</groupId>
          <artifactId>findbugs-maven-plugin</artifactId>
          <configuration>
            <skip>true</skip>
          </configuration>
        </plugin>

        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-surefire-plugin</artifactId>
          <configuration>
            <forkCount>1</forkCount>
            <reuseForks>false</reuseForks>
            <systemPropertyVariables>
              <beam.spark.test.reuseSparkContext>true</beam.spark.test.reuseSparkContext>
              <spark.ui.enabled>false</spark.ui.enabled>
            </systemPropertyVariables>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.codehaus.mojo</groupId>
          <artifactId>exec-maven-plugin</artifactId>
          <version>1.4.0</version>
          <configuration>
            <executable>java</executable>
            <classpathScope>test</classpathScope> <!-- include provided deps -->
            <arguments>
              <argument>-classpath</argument>
              <classpath />
              <argument>${mainClass}</argument>
              <argument>--inputFile=${input}</argument>
              <argument>--output=${output}</argument>
              <argument>--runner=${runner}</argument>
              <argument>--sparkMaster=${sparkMaster}</argument>
            </arguments>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.jacoco</groupId>
          <artifactId>jacoco-maven-plugin</artifactId>
          <version>0.7.5.201505241946</version>
          <executions>
            <execution>
              <goals>
                <goal>prepare-agent</goal>
              </goals>
            </execution>
            <execution>
              <id>report</id>
              <phase>test</phase>
              <goals>
                <goal>report</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-shade-plugin</artifactId>
          <executions>
            <execution>
              <phase>package</phase>
              <goals>
                <goal>shade</goal>
              </goals>
              <configuration>
                <relocations>
                  <!-- relocate Guava used by Beam (v18) since it conflicts with
                    version used by Hadoop (v11) -->
                  <relocation>
                    <pattern>com.google.common</pattern>
                    <shadedPattern>org.apache.beam.spark.relocated.com.google.common</shadedPattern>
                  </relocation>
                  <relocation>
                    <pattern>com.google.thirdparty</pattern>
                    <shadedPattern>org.apache.beam.spark.relocated.com.google.thirdparty</shadedPattern>
                  </relocation>
                  <relocation>
                    <pattern>com.esotericsoftware.kryo</pattern>
                    <shadedPattern>org.apache.beam.spark.relocated.com.esotericsoftware.kryo</shadedPattern>
                  </relocation>
                </relocations>
                <shadedArtifactAttached>true</shadedArtifactAttached>
                <shadedClassifierName>spark-app</shadedClassifierName>
                <transformers>
                  <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
                </transformers>
              </configuration>
            </execution>
          </executions>
        </plugin>
      </plugins>
    </pluginManagement>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-jar-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
      </plugin>
    </plugins>
  </build>

</project>