runners/spark/pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<!--
    Licensed to the Apache Software Foundation (ASF) under one or more
    contributor license agreements.  See the NOTICE file distributed with
    this work for additional information regarding copyright ownership.
    The ASF licenses this file to You under the Apache License, Version 2.0
    (the "License"); you may not use this file except in compliance with
    the License.  You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

  <modelVersion>4.0.0</modelVersion>

  <parent>
    <groupId>org.apache.beam</groupId>
    <artifactId>beam-runners-parent</artifactId>
    <version>2.3.0</version>
    <relativePath>../pom.xml</relativePath>
  </parent>

  <artifactId>beam-runners-spark</artifactId>

  <name>Apache Beam :: Runners :: Spark</name>
  <packaging>jar</packaging>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
    <dropwizard.metrics.version>3.1.2</dropwizard.metrics.version>
  </properties>

  <profiles>
    <profile>
      <id>jacoco</id>
      <build>
        <plugins>
          <plugin>
            <groupId>org.jacoco</groupId>
            <artifactId>jacoco-maven-plugin</artifactId>
          </plugin>
        </plugins>
      </build>
    </profile>

    <profile>
      <!-- This profile adds execution of ValidatesRunner integration tests
           against a local Spark endpoint. -->
      <id>local-validates-runner-tests</id>
      <activation><activeByDefault>false</activeByDefault></activation>
      <build>
        <plugins>
          <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-surefire-plugin</artifactId>
            <executions>
              <execution>
                <id>validates-runner-tests</id>
                <phase>integration-test</phase>
                <goals>
                  <goal>test</goal>
                </goals>
                <configuration>
                  <groups>
                    org.apache.beam.sdk.testing.ValidatesRunner,
                    org.apache.beam.runners.spark.UsesCheckpointRecovery
                  </groups>
                  <excludedGroups>
                    org.apache.beam.sdk.testing.UsesSplittableParDo,
                    org.apache.beam.sdk.testing.UsesCommittedMetrics,
                    org.apache.beam.sdk.testing.UsesTestStream,
                    org.apache.beam.sdk.testing.UsesCustomWindowMerging
                  </excludedGroups>
                  <parallel>none</parallel>
                  <forkCount>1</forkCount>
                  <reuseForks>false</reuseForks>
                  <failIfNoTests>true</failIfNoTests>
                  <dependenciesToScan>
                    <dependency>org.apache.beam:beam-sdks-java-core</dependency>
                  </dependenciesToScan>
                  <systemPropertyVariables>
                    <beamTestPipelineOptions>
                      [
                        "--runner=TestSparkRunner",
                        "--streaming=false",
                        "--enableSparkMetricSinks=false"
                      ]
                    </beamTestPipelineOptions>
                    <beam.spark.test.reuseSparkContext>true</beam.spark.test.reuseSparkContext>
                    <spark.ui.enabled>false</spark.ui.enabled>
                    <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
                  </systemPropertyVariables>
                  <threadCount>4</threadCount>
                </configuration>
              </execution>
              <execution>
                <id>streaming-tests</id>
                <phase>test</phase>
                <goals>
                  <goal>test</goal>
                </goals>
                <configuration>
                  <groups>
                    org.apache.beam.runners.spark.StreamingTest
                  </groups>
                  <systemPropertyVariables>
                    <beamTestPipelineOptions>
                      [
                      "--runner=TestSparkRunner",
                      "--forceStreaming=true",
                      "--enableSparkMetricSinks=true"
                      ]
                    </beamTestPipelineOptions>
                  </systemPropertyVariables>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
  </profiles>

  <dependencies>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.11</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_2.11</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-network-common_2.11</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <scope>provided</scope>
      <exclusions>
        <!-- Fix build on JDK-9 -->
        <exclusion>
          <groupId>jdk.tools</groupId>
          <artifactId>jdk.tools</artifactId>
        </exclusion>
        <exclusion>
          <groupId>javax.servlet</groupId>
          <artifactId>servlet-api</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-mapreduce-client-core</artifactId>
      <scope>provided</scope>
      <exclusions>
        <exclusion>
          <groupId>javax.servlet</groupId>
          <artifactId>servlet-api</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>com.esotericsoftware</groupId>
      <artifactId>kryo-shaded</artifactId>
      <version>3.0.3</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>com.google.code.findbugs</groupId>
      <artifactId>jsr305</artifactId>
    </dependency>
    <dependency>
      <groupId>com.google.guava</groupId>
      <artifactId>guava</artifactId>
    </dependency>
    <dependency>
      <groupId>com.google.auto.service</groupId>
      <artifactId>auto-service</artifactId>
      <optional>true</optional>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-annotations</artifactId>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.module</groupId>
      <artifactId>jackson-module-scala_2.11</artifactId>
      <version>${jackson.version}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.avro</groupId>
      <artifactId>avro</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-api</artifactId>
    </dependency>
    <dependency>
      <groupId>joda-time</groupId>
      <artifactId>joda-time</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.commons</groupId>
      <artifactId>commons-compress</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.commons</groupId>
      <artifactId>commons-lang3</artifactId>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.commons</groupId>
      <artifactId>commons-text</artifactId>
    </dependency>
    <dependency>
      <groupId>commons-io</groupId>
      <artifactId>commons-io</artifactId>
      <version>2.4</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.zookeeper</groupId>
      <artifactId>zookeeper</artifactId>
      <version>3.4.6</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>2.11.8</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-model-pipeline</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-sdks-java-core</artifactId>
      <exclusions>
        <!-- Use Hadoop/Spark's backend logger instead of jdk14 for tests -->
        <exclusion>
          <groupId>org.slf4j</groupId>
          <artifactId>slf4j-jdk14</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-runners-core-construction-java</artifactId>
    </dependency>
    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-runners-core-java</artifactId>
      <exclusions>
        <!-- Use Hadoop/Spark's backend logger instead of jdk14 for tests -->
        <exclusion>
          <groupId>org.slf4j</groupId>
          <artifactId>slf4j-jdk14</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>io.dropwizard.metrics</groupId>
      <artifactId>metrics-core</artifactId>
      <version>${dropwizard.metrics.version}</version>
    </dependency>

    <!-- KafkaIO -->
    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-sdks-java-io-kafka</artifactId>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka-clients</artifactId>
      <version>${kafka.clients.version}</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <groupId>net.jpountz.lz4</groupId>
          <artifactId>lz4</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka_2.11</artifactId>
      <version>${kafka.clients.version}</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <groupId>net.jpountz.lz4</groupId>
          <artifactId>lz4</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

    <!-- test dependencies -->
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>hamcrest-core</artifactId>
          <groupId>org.hamcrest</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.hamcrest</groupId>
      <artifactId>hamcrest-all</artifactId>
      <scope>provided</scope>
    </dependency>

    <!-- Depend on test jar to scan for ValidatesRunner tests -->
    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-sdks-java-core</artifactId>
      <classifier>tests</classifier>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <groupId>org.slf4j</groupId>
          <artifactId>slf4j-jdk14</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

    <dependency>
      <groupId>org.mockito</groupId>
      <artifactId>mockito-all</artifactId>
      <scope>test</scope>
    </dependency>

    <!-- transitive test dependencies from beam-sdk-java-core -->
    <dependency>
      <groupId>com.fasterxml.jackson.dataformat</groupId>
      <artifactId>jackson-dataformat-yaml</artifactId>
      <version>${jackson.version}</version>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-model-fn-execution</artifactId>
      <type>test-jar</type>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.beam</groupId>
      <artifactId>beam-runners-core-java</artifactId>
      <type>test-jar</type>
      <scope>test</scope>
    </dependency>
  </dependencies>

  <build>
    <pluginManagement>
      <plugins>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-surefire-plugin</artifactId>
          <configuration>
            <forkCount>1</forkCount>
            <reuseForks>false</reuseForks>
            <systemPropertyVariables>
              <beam.spark.test.reuseSparkContext>true</beam.spark.test.reuseSparkContext>
              <spark.ui.enabled>false</spark.ui.enabled>
              <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
            </systemPropertyVariables>
          </configuration>
          <executions>
            <execution>
              <id>default-test</id>
              <goals>
                <goal>test</goal>
              </goals>
              <configuration>
                <excludedGroups>
                  org.apache.beam.runners.spark.UsesCheckpointRecovery,
                  org.apache.beam.runners.spark.StreamingTest
                </excludedGroups>
                <systemPropertyVariables>
                  <beamTestPipelineOptions>
                    [
                    "--runner=TestSparkRunner",
                    "--streaming=false",
                    "--enableSparkMetricSinks=true"
                    ]
                  </beamTestPipelineOptions>
                </systemPropertyVariables>
              </configuration>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.codehaus.mojo</groupId>
          <artifactId>exec-maven-plugin</artifactId>
          <configuration>
            <executable>java</executable>
            <classpathScope>test</classpathScope> <!-- include provided deps -->
            <arguments>
              <argument>-classpath</argument>
              <classpath />
              <argument>${mainClass}</argument>
              <argument>--inputFile=${input}</argument>
              <argument>--output=${output}</argument>
              <argument>--runner=${runner}</argument>
              <argument>--sparkMaster=${sparkMaster}</argument>
            </arguments>
          </configuration>
        </plugin>
        <plugin>
          <groupId>org.jacoco</groupId>
          <artifactId>jacoco-maven-plugin</artifactId>
          <executions>
            <execution>
              <goals>
                <goal>prepare-agent</goal>
              </goals>
            </execution>
            <execution>
              <id>report</id>
              <phase>test</phase>
              <goals>
                <goal>report</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-dependency-plugin</artifactId>
          <configuration>
            <ignoredUnusedDeclaredDependencies>
              <ignoredUnusedDeclaredDependency>
                com.fasterxml.jackson.module:jackson-module-scala_2.11
              </ignoredUnusedDeclaredDependency>
            </ignoredUnusedDeclaredDependencies>
          </configuration>
        </plugin>
      </plugins>
    </pluginManagement>
  </build>
</project>