sdks/java/testing/nexmark/build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

plugins { id 'org.apache.beam.module' }
applyJavaNature(
    automaticModuleName: 'org.apache.beam.sdk.nexmark',
    exportJavadoc: false,
    archivesBaseName: 'beam-sdks-java-nexmark',
)

description = "Apache Beam :: SDKs :: Java :: Nexmark"

// When running via Gradle, this property can be used to pass commandline arguments
// to the nexmark launch
def nexmarkArgsProperty = "nexmark.args"
// When running via Gradle, this property can be set to "true" to enable profiling for
// the nexmark pipeline. Currently only works for the Dataflow runner.
def nexmarkProfilingProperty = "nexmark.profile"

// When running via Gradle, this property sets the runner dependency
def nexmarkRunnerProperty = "nexmark.runner"
def nexmarkRunnerDependency = project.findProperty(nexmarkRunnerProperty)
        ?: ":runners:direct-java"
def nexmarkRunnerVersionProperty = "nexmark.runner.version"
def nexmarkRunnerVersion = project.findProperty(nexmarkRunnerVersionProperty)
def shouldProvideSpark = ":runners:spark:2".equals(nexmarkRunnerDependency)
def isDataflowRunner = ":runners:google-cloud-dataflow-java".equals(nexmarkRunnerDependency)
def isDataflowRunnerV2 = isDataflowRunner && "V2".equals(nexmarkRunnerVersion)
def runnerConfiguration = ":runners:direct-java".equals(nexmarkRunnerDependency) ? "shadow" : null

if (isDataflowRunner) {
  /*
   * We need to rely on manually specifying these evaluationDependsOn to ensure that
   * the following projects are evaluated before we evaluate this project. This is because
   * we are attempting to reference a property from the project directly.
   */
  if (isDataflowRunnerV2) {
    evaluationDependsOn(":runners:google-cloud-dataflow-java")
  } else {
    evaluationDependsOn(":runners:google-cloud-dataflow-java:worker:legacy-worker")
  }
}

configurations {
  // A configuration for running the Nexmark launcher directly from Gradle, which
  // uses Gradle to put the appropriate dependencies on the Classpath rather than
  // bundling them into a fat jar
  gradleRun
}

dependencies {
  compile library.java.vendored_guava_26_0_jre
  compile project(path: ":sdks:java:core", configuration: "shadow")
  compile project(":sdks:java:io:google-cloud-platform")
  compile project(":sdks:java:extensions:google-cloud-platform-core")
  compile project(":sdks:java:extensions:sql")
  compile project(":sdks:java:extensions:sql:zetasql")
  compile project(":sdks:java:io:kafka")
  compile project(":sdks:java:testing:test-utils")
  compile library.java.google_api_client
  compile library.java.junit
  compile "org.hamcrest:hamcrest:2.1"
  compile library.java.google_api_services_bigquery
  compile library.java.jackson_core
  compile library.java.jackson_annotations
  compile library.java.jackson_databind
  compile library.java.jackson_datatype_joda
  compile library.java.avro
  compile library.java.joda_time
  compile library.java.slf4j_api
  compile library.java.kafka_clients
  testRuntimeClasspath library.java.slf4j_jdk14
  testCompile project(path: ":sdks:java:io:google-cloud-platform", configuration: "testRuntime")
  testCompile project(path: ":sdks:java:testing:test-utils", configuration: "testRuntime")
  gradleRun project(project.path)
  gradleRun project(path: nexmarkRunnerDependency, configuration: runnerConfiguration)

  // The Spark runner requires the user to provide a Spark dependency. For self-contained
  // runs with the Spark runner, we can provide such a dependency. This is deliberately phrased
  // to not hardcode any runner other than :runners:direct-java
  if (shouldProvideSpark) {
    gradleRun library.java.spark_core, {
      exclude group:"org.slf4j", module:"jul-to-slf4j"
    }
    gradleRun library.java.spark_sql
    gradleRun library.java.spark_streaming
  }
}

if (shouldProvideSpark) {
  configurations.gradleRun {
    // Using Spark runner causes a StackOverflowError if slf4j-jdk14 is on the classpath
    exclude group: "org.slf4j", module: "slf4j-jdk14"
  }
}

def getNexmarkArgs = {
  def nexmarkArgsStr =  project.findProperty(nexmarkArgsProperty) ?: ""
  def nexmarkArgsList = new ArrayList<String>()
  Collections.addAll(nexmarkArgsList, nexmarkArgsStr.split())

  if (isDataflowRunner) {
    if (isDataflowRunnerV2) {
      nexmarkArgsList.add("--experiments=beam_fn_api,use_unified_worker,use_runner_v2,shuffle_mode=service")
      def sdkContainerImage = project.findProperty('sdkContainerImage') ?: project(":runners:google-cloud-dataflow-java").dockerImageName
      nexmarkArgsList.add("--sdkContainerImage=${sdkContainerImage}")

      // TODO(BEAM-12295) enable all queries once issues with runner V2 is fixed.
      if (nexmarkArgsList.contains("--streaming=true")) {
        nexmarkArgsList.add("--skipQueries=AVERAGE_PRICE_FOR_CATEGORY,AVERAGE_SELLING_PRICE_BY_SELLER,WINNING_BIDS,BOUNDED_SIDE_INPUT_JOIN,SESSION_SIDE_INPUT_JOIN,PORTABILITY_BATCH") // 4, 6, 9, 13, 14, 15
      } else {
        nexmarkArgsList.add("--skipQueries=LOCAL_ITEM_SUGGESTION,AVERAGE_PRICE_FOR_CATEGORY,AVERAGE_SELLING_PRICE_BY_SELLER,HIGHEST_BID,WINNING_BIDS,SESSION_SIDE_INPUT_JOIN,BOUNDED_SIDE_INPUT_JOIN") // 3, 4, 6, 7, 9, 13, 14, 15
      }
    } else {
      def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker:legacy-worker").shadowJar.archivePath
      // Provide job with a customizable worker jar.
      // With legacy worker jar, containerImage is set to empty (i.e. to use the internal build).
      // More context and discussions can be found in PR#6694.
      nexmarkArgsList.add("--dataflowWorkerJar=${dataflowWorkerJar}".toString())
      nexmarkArgsList.add('--workerHarnessContainerImage=')

      def nexmarkProfile =  project.findProperty(nexmarkProfilingProperty) ?: ""
      if (nexmarkProfile.equals("true")) {
        nexmarkArgsList.add('--profilingAgentConfiguration={ "APICurated": true }')
      }
    }
  }
  return nexmarkArgsList
}

// Execute the Nexmark queries or suites via Gradle.
//
// Parameters:
//   -Pnexmark.runner
//       Specify a runner subproject, such as ":runners:spark:2" or ":runners:flink:1.13"
//       Defaults to ":runners:direct-java"
//
//   -Pnexmark.args
//       Specify the command line for invoking org.apache.beam.sdk.nexmark.Main
task run(type: JavaExec) {
  def nexmarkArgsList = getNexmarkArgs()
  if (isDataflowRunner) {
    if (isDataflowRunnerV2) {
      dependsOn ":runners:google-cloud-dataflow-java:buildAndPushDockerContainer"
      finalizedBy ":runners:google-cloud-dataflow-java:cleanUpDockerImages"
    } else {
      dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar"
    }
  }
  main = "org.apache.beam.sdk.nexmark.Main"
  classpath = configurations.gradleRun
  args nexmarkArgsList.toArray()
}

// Execute the Nexmark queries or suites via Gradle using Java 11.
task runJava11(type: JavaExec) {
  def nexmarkArgsList = getNexmarkArgs()
  if (isDataflowRunner) {
    if (isDataflowRunnerV2) {
      dependsOn ":runners:google-cloud-dataflow-java:buildAndPushDockerContainer"
      finalizedBy ":runners:google-cloud-dataflow-java:cleanUpDockerImages"
    } else {
      dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:shadowJar"
    }
  }
  javaLauncher = javaToolchains.launcherFor {
    languageVersion = JavaLanguageVersion.of(11)
  }
  main = "org.apache.beam.sdk.nexmark.Main"
  classpath = configurations.gradleRun
  args nexmarkArgsList.toArray()
}

if (!project.hasProperty("compileAndRunTestsWithJava11")) {
  tasks.withType(JavaCompile).configureEach {
    javaCompiler = javaToolchains.compilerFor {
      languageVersion = JavaLanguageVersion.of(8)
    }
  }
}