| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * License); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an AS IS BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| import groovy.json.JsonOutput |
| plugins { id 'org.apache.beam.module' } |
| |
| // Shade away runner execution utilities till because this causes ServiceLoader conflicts with |
| // TransformPayloadTranslatorRegistrar amongst other runners. This only happens in the DirectRunner |
| // because it is likely to appear on the classpath of another runner. |
| def dependOnProjects = [":beam-runners-core-construction-java", |
| ":beam-runners-core-java", |
| ":beam-runners-local-java-core", |
| ":beam-runners-java-fn-execution", |
| ":beam-sdks-java-fn-execution"] |
| |
| applyJavaNature(shadowClosure: DEFAULT_SHADOW_CLOSURE << { |
| dependencies { |
| dependOnProjects.each { |
| include(project(path: it, configuration: "shadow")) |
| } |
| } |
| relocate "org.apache.beam.runners.core", getJavaRelocatedPath("runners.core") |
| relocate "org.apache.beam.runners.fnexecution", getJavaRelocatedPath("runners.fnexecution") |
| relocate "org.apache.beam.sdk.fn", getJavaRelocatedPath("sdk.fn") |
| relocate "org.apache.beam.runners.local", getJavaRelocatedPath("runners.local") |
| }) |
| |
| description = "Apache Beam :: Runners :: Direct Java" |
| |
| /* |
| * We need to rely on manually specifying these evaluationDependsOn to ensure that |
| * the following projects are evaluated before we evaluate this project. This is because |
| * we are attempting to reference the "sourceSets.test.output" directly. |
| */ |
| evaluationDependsOn(":beam-runners-core-construction-java") |
| evaluationDependsOn(":beam-runners-core-java") |
| evaluationDependsOn(":beam-sdks-java-core") |
| |
| configurations { |
| needsRunner |
| validatesRunner |
| validatesPortableRunner |
| } |
| |
| dependencies { |
| shadow library.java.vendored_guava_20_0 |
| shadow project(path: ":beam-model-pipeline", configuration: "shadow") |
| dependOnProjects.each { |
| compile project(path: it, configuration: "shadow") |
| } |
| shadow project(path: ":beam-sdks-java-core", configuration: "shadow") |
| shadow library.java.vendored_grpc_1_13_1 |
| shadow library.java.joda_time |
| shadow library.java.slf4j_api |
| shadow library.java.args4j |
| provided library.java.hamcrest_core |
| provided library.java.junit |
| testRuntime project(path: ":beam-sdks-java-harness") |
| shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest") |
| shadowTest project(path: ":beam-runners-core-java", configuration: "shadowTest") |
| shadowTest library.java.guava_testlib |
| shadowTest library.java.slf4j_jdk14 |
| shadowTest library.java.mockito_core |
| shadowTest library.java.stax2_api |
| shadowTest library.java.woodstox_core_asl |
| shadowTest library.java.google_cloud_dataflow_java_proto_library_all |
| shadowTest library.java.jackson_dataformat_yaml |
| needsRunner project(path: ":beam-runners-core-construction-java", configuration: "shadowTest") |
| needsRunner project(path: ":beam-runners-core-java", configuration: "shadowTest") |
| needsRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") |
| needsRunner project(path: project.path, configuration: "shadow") |
| needsRunner project(path: project.path, configuration: "shadowTest") |
| validatesRunner project(path: ":beam-runners-core-construction-java", configuration: "shadowTest") |
| validatesRunner project(path: ":beam-runners-core-java", configuration: "shadowTest") |
| validatesRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") |
| validatesRunner project(path: project.path, configuration: "shadow") |
| validatesRunner project(path: project.path, configuration: "shadowTest") |
| validatesPortableRunner project(path: ":beam-runners-core-construction-java", configuration: "shadowTest") |
| validatesPortableRunner project(path: ":beam-runners-core-java", configuration: "shadowTest") |
| validatesPortableRunner project(path: ":beam-runners-java-fn-execution", configuration: "shadowTest") |
| validatesPortableRunner project(path: ":beam-runners-reference-java", configuration: "shadowTest") |
| validatesPortableRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest") |
| validatesPortableRunner project(path: project.path, configuration: "shadow") |
| validatesPortableRunner project(path: project.path, configuration: "shadowTest") |
| } |
| |
| task needsRunnerTests(type: Test) { |
| group = "Verification" |
| description = "Runs tests that require a runner to validate that piplines/transforms work correctly" |
| |
| def pipelineOptions = JsonOutput.toJson(["--runner=DirectRunner", "--runnerDeterminedSharding=false"]) |
| systemProperty "beamTestPipelineOptions", pipelineOptions |
| |
| classpath = configurations.needsRunner |
| // NOTE: We only add to the test class dirs to ensure that the direct runner |
| // tests (i.e., those from this subproject) get scanned. |
| testClassesDirs += files(project(":beam-runners-core-construction-java").sourceSets.test.output.classesDirs) |
| testClassesDirs += files(project(":beam-runners-core-java").sourceSets.test.output.classesDirs) |
| testClassesDirs += files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) |
| useJUnit { |
| includeCategories "org.apache.beam.sdk.testing.NeedsRunner" |
| excludeCategories "org.apache.beam.sdk.testing.LargeKeys\$Above100MB" |
| // MetricsPusher isn't implemented in direct runner |
| excludeCategories "org.apache.beam.sdk.testing.UsesMetricsPusher" |
| excludeCategories "org.apache.beam.sdk.testing.UsesCrossLanguageTransforms" |
| } |
| } |
| |
| // NOTE: This will also run 'NeedsRunner' tests, which are run in the :needsRunnerTests task as well. |
| // The intention of this task is to mirror the :validatesRunner configuration for other runners, |
| // such that the test suite can be validated on the in-process DirectRunner. |
| task validatesRunner(type: Test) { |
| group = "Verification" |
| description "Validates Direct runner" |
| |
| def pipelineOptions = JsonOutput.toJson(["--runner=DirectRunner", "--runnerDeterminedSharding=false"]) |
| systemProperty "beamTestPipelineOptions", pipelineOptions |
| |
| classpath = configurations.validatesRunner |
| // NOTE: We only add to the test class dirs to ensure that the direct runner |
| // tests (i.e., those from this subproject) get scanned. |
| testClassesDirs += files(project(":beam-runners-core-construction-java").sourceSets.test.output.classesDirs) |
| testClassesDirs += files(project(":beam-runners-core-java").sourceSets.test.output.classesDirs) |
| testClassesDirs += files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs) |
| useJUnit { |
| includeCategories "org.apache.beam.sdk.testing.ValidatesRunner" |
| excludeCategories "org.apache.beam.sdk.testing.LargeKeys\$Above100MB" |
| excludeCategories 'org.apache.beam.sdk.testing.UsesMetricsPusher' |
| excludeCategories "org.apache.beam.sdk.testing.UsesCrossLanguageTransforms" |
| } |
| } |
| |
| def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing' |
| def gcsBucket = project.findProperty('gcsBucket') ?: 'temp-storage-for-release-validation-tests/nightly-snapshot-validation' |
| def bqDataset = project.findProperty('bqDataset') ?: 'beam_postrelease_mobile_gaming' |
| def pubsubTopic = project.findProperty('pubsubTopic') ?: 'java_mobile_gaming_topic' |
| |
| // Generates :beam-runners-direct-java:runQuickstartJavaDirect |
| createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'Direct') |
| |
| // Generates :beam-runners-direct-java:runMobileGamingJavaDirect |
| createJavaExamplesArchetypeValidationTask(type: 'MobileGaming', |
| runner: 'Direct', |
| gcpProject: gcpProject, |
| gcsBucket: gcsBucket, |
| bqDataset: bqDataset, |
| pubsubTopic: pubsubTopic) |
| |
| createPortableValidatesRunnerTask( |
| jobServerDriver: "org.apache.beam.runners.direct.portable.job.ReferenceRunnerJobServer", |
| testClasspathConfiguration: configurations.validatesPortableRunner, |
| jobServerConfig: "--port=0", |
| testCategories: { |
| includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' |
| excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders' |
| excludeCategories 'org.apache.beam.sdk.testing.LargeKeys$Above100MB' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesCrossLanguageTransforms' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesCustomWindowMerging' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesDistributionMetrics' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesFailureMessage' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesParDoLifecycle' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream' |
| // TODO(BEAM-5452): Support metrics. |
| excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesAttemptedMetrics' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesCommittedMetrics' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesCounterMetrics' |
| // TODO(BEAM-3743): Support SplittableDoFn |
| excludeCategories 'org.apache.beam.sdk.testing.UsesBoundedSplittableParDo' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesSplittableParDoWithWindowedSideInputs' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo' |
| // TODO(BEAM-2928): Support sideinput. |
| excludeCategories 'org.apache.beam.sdk.testing.UsesSideInputs' |
| // TODO(BEAM-2917): Support user state. |
| excludeCategories 'org.apache.beam.sdk.testing.UsesStatefulParDo' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesMapState' |
| excludeCategories 'org.apache.beam.sdk.testing.UsesSetState' |
| // TODO(BEAM-4680): Support user timers. |
| excludeCategories 'org.apache.beam.sdk.testing.UsesTimersInParDo' |
| }, |
| ) |
