| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * License); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an AS IS BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| plugins { id 'org.apache.beam.module' } |
| applyGoNature() |
| applyPythonNature() |
| |
| description = "Apache Beam :: SDKs :: Go :: Test" |
| |
| // ValidatesRunner tests for Dataflow. Runs tests in the integration directory |
| // with Dataflow to validate that the runner behaves as expected. |
| task dataflowValidatesRunner() { |
| group = "Verification" |
| |
| dependsOn ":sdks:go:test:goBuild" |
| |
| doLast { |
| def options = [ |
| "--runner dataflow", |
| ] |
| exec { |
| executable "sh" |
| args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" |
| } |
| } |
| } |
| |
| // ValidatesRunner tests for Dataflow. Runs tests in the integration directory |
| // with Dataflow to validate that the runner behaves as expected, on arm64 machines. |
| task dataflowValidatesRunnerARM64() { |
| group = "Verification" |
| |
| dependsOn ":sdks:go:test:goBuild" |
| |
| doLast { |
| def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. |
| "--machine_type=t2a-standard-1", |
| ] |
| def options = [ |
| "--runner dataflow", |
| "--pipeline_opts \"${pipelineOptions.join(' ')}\"", |
| ] |
| exec { |
| executable "sh" |
| args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" |
| } |
| } |
| } |
| |
| // ValidatesRunner tests for Flink. Runs tests in the integration directory |
| // with Flink to validate that the runner behaves as expected. |
| task flinkValidatesRunner { |
| group = "Verification" |
| |
| dependsOn ":sdks:go:test:goBuild" |
| dependsOn ":sdks:go:container:docker" |
| dependsOn ":sdks:java:container:java8:docker" |
| dependsOn ":runners:flink:${project.ext.latestFlinkVersion}:job-server:shadowJar" |
| dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" |
| doLast { |
| def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. |
| "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}", |
| ] |
| def options = [ |
| "--runner flink", |
| "--flink_job_server_jar ${project(":runners:flink:${project.ext.latestFlinkVersion}:job-server").shadowJar.archivePath}", |
| "--pipeline_opts \"${pipelineOptions.join(' ')}\"", |
| ] |
| exec { |
| executable "sh" |
| args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" |
| } |
| } |
| } |
| |
| // ValidatesRunner tests for Samza. Runs tests in the integration directory |
| // with Samza to validate that the runner behaves as expected. |
| task samzaValidatesRunner { |
| dependsOn ":sdks:go:test:goBuild" |
| dependsOn ":sdks:go:container:docker" |
| dependsOn ":sdks:java:container:java8:docker" |
| dependsOn ":runners:samza:job-server:shadowJar" |
| dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" |
| doLast { |
| def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. |
| "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}", |
| ] |
| def options = [ |
| "--runner samza", |
| "--samza_job_server_jar ${project(":runners:samza:job-server").shadowJar.archivePath}", |
| "--pipeline_opts \"${pipelineOptions.join(' ')}\"", |
| ] |
| exec { |
| executable "sh" |
| args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" |
| } |
| } |
| } |
| |
| // ValidatesRunner tests for Spark. Runs tests in the integration directory |
| // with Spark to validate that the runner behaves as expected. |
| task sparkValidatesRunner { |
| group = "Verification" |
| |
| dependsOn ":sdks:go:test:goBuild" |
| dependsOn ":sdks:java:container:java8:docker" |
| dependsOn ":runners:spark:3:job-server:shadowJar" |
| dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" |
| doLast { |
| def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. |
| "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}", |
| ] |
| def options = [ |
| "--runner spark", |
| "--spark_job_server_jar ${project(":runners:spark:3:job-server").shadowJar.archivePath}", |
| "--pipeline_opts \"${pipelineOptions.join(' ')}\"", |
| ] |
| exec { |
| executable "sh" |
| args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" |
| } |
| } |
| } |
| |
| // ValidatesRunner tests for the Python Portable runner (aka. ULR). Runs tests |
| // in the integration directory with the ULR to validate that the runner behaves |
| // as expected. |
| // |
| // The ULR can exhibit strange behavior when containers are built with outdated |
| // vendored directories. For best results use the clean task, like so: |
| // ./gradlew clean :sdks:go:test:ulrValidatesRunner |
| tasks.register("ulrValidatesRunner") { |
| group = "Verification" |
| |
| dependsOn ":sdks:go:test:goBuild" |
| dependsOn ":sdks:go:container:docker" |
| dependsOn ":sdks:java:container:java8:docker" |
| dependsOn "setupVirtualenv" |
| dependsOn ":sdks:python:buildPython" |
| dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" |
| doLast { |
| def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. |
| "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}", |
| ] |
| def options = [ |
| "--runner portable", |
| "--pipeline_opts \"${pipelineOptions.join(' ')}\"", |
| ] |
| exec { |
| executable "sh" |
| workingDir "${project.rootDir}/sdks/python" |
| args "-c", ". ${envdir}/bin/activate && pip install -e ." |
| } |
| exec { |
| executable "sh" |
| args "-c", ". ${envdir}/bin/activate && ./run_validatesrunner_tests.sh ${options.join(' ')}" |
| } |
| } |
| } |
| |
| // ValidatesRunner tests for Prism. Runs tests in the integration directory |
| // with prism in docker mod to validate that the runner behaves as expected. |
| task prismValidatesRunner { |
| group = "Verification" |
| |
| dependsOn ":sdks:go:test:goBuild" |
| dependsOn ":sdks:go:container:docker" |
| dependsOn ":sdks:java:container:java8:docker" |
| dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar" |
| doLast { |
| def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. |
| "--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}", |
| ] |
| def options = [ |
| "--runner prism", |
| "--pipeline_opts \"${pipelineOptions.join(' ')}\"", |
| ] |
| exec { |
| executable "sh" |
| args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" |
| } |
| } |
| } |
| |
| // A method for configuring a cross-language validates runner test task, |
| // intended to be used in calls to createCrossLanguageValidatesRunnerTask. |
| ext.goIoValidatesRunnerTask = { proj, name, scriptOpts, pipelineOpts -> |
| return proj.tasks.register(name) { |
| group = "Verification" |
| |
| dependsOn ":sdks:go:test:goBuild" |
| dependsOn ":sdks:go:container:docker" |
| dependsOn ":sdks:java:io:expansion-service:build" |
| dependsOn ":sdks:java:extensions:schemaio-expansion-service:build" |
| dependsOn ":sdks:java:io:debezium:expansion-service:build" |
| dependsOn ":sdks:java:io:google-cloud-platform:expansion-service:build" |
| dependsOn ":sdks:java:testing:kafka-service:buildTestKafkaServiceJar" |
| |
| doLast { |
| def ioExpJar = project(":sdks:java:io:expansion-service").shadowJar.archivePath |
| def schemaIoExpJar = project(":sdks:java:extensions:schemaio-expansion-service").shadowJar.archivePath |
| def debeziumIoExpJar = project(":sdks:java:io:debezium:expansion-service").shadowJar.archivePath |
| def gcpIoExpJar = project(":sdks:java:io:google-cloud-platform:expansion-service").shadowJar.archivePath |
| def kafkaJar = project(":sdks:java:testing:kafka-service:").buildTestKafkaServiceJar.archivePath |
| def pipelineOptions = [ // Pipeline options piped directly to Go SDK flags. |
| "--kafka_jar=${kafkaJar}", |
| "--expansion_jar=io:${ioExpJar}", |
| "--expansion_jar=schemaio:${schemaIoExpJar}", |
| "--expansion_jar=debeziumio:${debeziumIoExpJar}", |
| "--expansion_jar=gcpio:${gcpIoExpJar}", |
| "--bq_dataset=apache-beam-testing.beam_bigquery_io_test_temp", |
| "--bt_instance=projects/apache-beam-testing/instances/beam-test" |
| ] |
| pipelineOptions.addAll(pipelineOpts) |
| def options = [ |
| "--pipeline_opts \"${pipelineOptions.join(' ')}\"", |
| ] |
| options.addAll(scriptOpts) |
| logger.info("Running the command: sh -c ./run_validatesrunner_tests.sh ${options.join(' ')}") |
| exec { |
| executable "sh" |
| args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" |
| } |
| } |
| } |
| } |