sdks/java/io/hadoop-format/build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import groovy.json.JsonOutput

plugins { id 'org.apache.beam.module' }
applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.io.hadoop.format')
provideIntegrationTestingDependencies()
enableJavaPerformanceTesting()

description = "Apache Beam :: SDKs :: Java :: IO :: Hadoop Format"
ext.summary = "IO to read data from sources and to write data to sinks that implement Hadoop MapReduce Format."

def log4j_version = "2.6.2"
def elastic_search_version = "5.0.0"

configurations.create("sparkRunner")
configurations.sparkRunner {
  // Ban certain dependencies to prevent a StackOverflow within Spark
  // because JUL -> SLF4J -> JUL, and similarly JDK14 -> SLF4J -> JDK14
  exclude group: "org.slf4j", module: "jul-to-slf4j"
  exclude group: "org.slf4j", module: "slf4j-jdk14"
}

// Ban dependencies from the test runtime classpath
configurations.testRuntimeClasspath {
  // Prevent a StackOverflow because of wiring LOG4J -> SLF4J -> LOG4J
  exclude group: "org.slf4j", module: "log4j-over-slf4j"
}

dependencies {
  compile project(path: ":sdks:java:core", configuration: "shadow")
  compile library.java.vendored_guava_26_0_jre
  compile library.java.slf4j_api
  compile project(":sdks:java:io:hadoop-common")
  provided library.java.hadoop_common
  provided library.java.hadoop_hdfs
  provided library.java.hadoop_mapreduce_client_core
  testCompile project(path: ":sdks:java:core", configuration: "shadowTest")
  testCompile project(path: ":sdks:java:io:common", configuration: "testRuntime")
  testCompile project(path: ":sdks:java:testing:test-utils", configuration: "testRuntime")
  testCompile project(":sdks:java:io:jdbc")
  testCompile project(path: ":examples:java", configuration: "testRuntime")

  testCompile "org.elasticsearch.plugin:transport-netty4-client:$elastic_search_version"
  testCompile "org.elasticsearch.client:transport:$elastic_search_version"
  testCompile "io.netty:netty-transport-native-epoll:4.1.0.CR3"
  testCompile "org.elasticsearch:elasticsearch:$elastic_search_version"
  testCompile ("org.elasticsearch:elasticsearch-hadoop:$elastic_search_version") {
    // TODO(https://issues.apache.org/jira/browse/BEAM-3715)
    // These are all optional deps of elasticsearch-hadoop. Why do they have to be excluded?
    exclude group: "cascading", module: "cascading-local"
    exclude group: "cascading", module: "cascading-hadoop"
    exclude group: "org.apache.hive", module: "hive-service"
    exclude group: "org.apache.pig", module: "pig"
    exclude group: "org.apache.spark", module: "spark-core_2.10"
    exclude group: "org.apache.spark", module: "spark-streaming_2.10"
    exclude group: "org.apache.spark", module: "spark-sql_2.10"
    exclude group: "org.apache.storm", module: "storm-core"
  }
  testCompile library.java.cassandra_driver_core
  testCompile library.java.cassandra_driver_mapping
  testCompile "org.apache.cassandra:cassandra-all:3.11.3"
  testCompile library.java.postgres
  testCompile "org.apache.logging.log4j:log4j-core:$log4j_version"
  testCompile library.java.junit
  testCompile library.java.hamcrest_core
  testCompile library.java.hamcrest_library
  testRuntimeOnly library.java.slf4j_jdk14
  testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow")
  compile library.java.commons_io_2x

  delegate.add("sparkRunner", project(":sdks:java:io:hadoop-format"))
  delegate.add("sparkRunner", project(path: ":sdks:java:io:hadoop-format", configuration: "testRuntime"))

  sparkRunner project(path: ":examples:java", configuration: "testRuntime")
  sparkRunner project(":runners:spark")
  sparkRunner project(":sdks:java:io:hadoop-file-system")
  sparkRunner library.java.spark_streaming
  sparkRunner library.java.spark_core
}

// The cassandra.yaml file currently assumes "target/..." exists.
// TODO: Update cassandra.yaml to inject new properties representing
// the root path. Also migrate cassandra.yaml to use any open ports
// instead of a static port.
task createTargetDirectoryForCassandra() {
  doLast {
    if (!project.file("target").exists()) {
      project.file("target").mkdirs();
    }
  }
}
test.dependsOn createTargetDirectoryForCassandra

def runnerClass = "org.apache.beam.runners.spark.TestSparkRunner"
task sparkRunner(type: Test) {
  group = "Verification"
  def beamTestPipelineOptions = [
          "--project=hadoop-format",
          "--tempRoot=/tmp/hadoop-format/",
          "--streaming=false",
          "--runner=" + runnerClass,
          "--enableSparkMetricSinks=false",
  ]
  classpath = configurations.sparkRunner
  include "**/HadoopFormatIOSequenceFileTest.class"
  useJUnit {
    includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
  }
  forkEvery 1
  maxParallelForks 4
  systemProperty "spark.ui.enabled", "false"
  systemProperty "spark.ui.showConsoleProgress", "false"
  systemProperty "beam.spark.test.reuseSparkContext", "true"
  systemProperty "beamTestPipelineOptions", JsonOutput.toJson(beamTestPipelineOptions)
}