runners/flink/build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import groovy.json.JsonOutput

apply plugin: org.apache.beam.gradle.BeamModulePlugin
applyJavaNature()


description = "Apache Beam :: Runners :: Flink"

/*
 * We need to rely on manually specifying these evaluationDependsOn to ensure that
 * the following projects are evaluated before we evaluate this project. This is because
 * we are attempting to reference the "sourceSets.test.output" directly.
 */
evaluationDependsOn(":beam-sdks-java-core")
evaluationDependsOn(":beam-runners-core-java")


test {
  systemProperty "log4j.configuration", "log4j-test.properties"
  //systemProperty "org.slf4j.simpleLogger.defaultLogLevel", "debug"
  jvmArgs "-XX:-UseGCOverheadLimit"
  if (System.getProperty("beamSurefireArgline")) {
    jvmArgs System.getProperty("beamSurefireArgline")
  }
}

configurations {
  validatesRunner
}

def flink_version = "1.5.3"

dependencies {
  compile library.java.guava
  shadow project(path: ":beam-sdks-java-core", configuration: "shadow")
  shadow project(path: ":beam-runners-core-java", configuration: "shadow")
  shadow project(path: ":beam-runners-core-construction-java", configuration: "shadow")
  shadow project(path: ":beam-runners-java-fn-execution", configuration: "shadow")
  shadow library.java.jackson_annotations
  shadow library.java.slf4j_api
  shadow library.java.joda_time
  shadow library.java.commons_compress
  shadow library.java.args4j
  shadow "org.apache.flink:flink-clients_2.11:$flink_version"
  shadow "org.apache.flink:flink-core:$flink_version"
  shadow "org.apache.flink:flink-metrics-core:$flink_version"
  shadow "org.apache.flink:flink-java:$flink_version"
  shadow "org.apache.flink:flink-runtime_2.11:$flink_version"
  shadow "org.apache.flink:flink-streaming-java_2.11:$flink_version"
  shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest")
  // FlinkStateInternalsTest extends abstract StateInternalsTest
  shadowTest project(path: ":beam-runners-core-java", configuration: "shadowTest")
  shadowTest library.java.commons_lang3
  shadowTest library.java.hamcrest_core
  shadowTest library.java.junit
  shadowTest library.java.mockito_core
  shadowTest library.java.google_api_services_bigquery
  shadowTest library.java.slf4j_simple
  shadowTest project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadow")
  shadowTest library.java.jackson_dataformat_yaml
  shadowTest "org.apache.flink:flink-core:$flink_version:tests"
  shadowTest "org.apache.flink:flink-runtime_2.11:$flink_version:tests"
  shadowTest "org.apache.flink:flink-streaming-java_2.11:$flink_version:tests"
  shadowTest "org.apache.flink:flink-test-utils_2.11:$flink_version"
  shadowTest project(":beam-sdks-java-harness")
  validatesRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest")
  validatesRunner project(path: ":beam-runners-core-java", configuration: "shadowTest")
  validatesRunner project(path: project.path, configuration: "shadow")
}

class ValidatesRunnerConfig {
  String name
  boolean streaming
}

def createValidatesRunnerTask(Map m) {
  def config = m as ValidatesRunnerConfig
  tasks.create(name: config.name, type: Test) {
    group = "Verification"
    def runnerType = config.streaming ? "streaming" : "batch"
    description = "Validates the ${runnerType} runner"
    def pipelineOptions = JsonOutput.toJson(
        ["--runner=TestFlinkRunner",
         "--streaming=${config.streaming}",
         // TODO This should be changed to > 1
         "--parallelism=1",
        ])
    systemProperty "beamTestPipelineOptions", pipelineOptions
    classpath = configurations.validatesRunner
    testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs, project(":beam-runners-core-java").sourceSets.test.output.classesDirs)
    // maxParallelForks decreased from 4 in order to avoid OOM errors
    maxParallelForks 2
    useJUnit {
      includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
      excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders'
      excludeCategories 'org.apache.beam.sdk.testing.LargeKeys$Above100MB'
      excludeCategories 'org.apache.beam.sdk.testing.UsesCommittedMetrics'
      if (config.streaming) {
        excludeCategories 'org.apache.beam.sdk.testing.UsesImpulse'
      } else {
        excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo'
      }
      excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
    }
  }
}

createValidatesRunnerTask(name: "validatesRunnerBatch", streaming: false)
createValidatesRunnerTask(name: "validatesRunnerStreaming", streaming: true)

task validatesRunner {
  group = "Verification"
  description "Validates Flink runner"
  dependsOn validatesRunnerBatch
  dependsOn validatesRunnerStreaming
}

// Generates :beam-runners-flink_2.11:runQuickstartJavaFlinkLocal
createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'FlinkLocal')