| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * License); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an AS IS BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.beam.gradle |
| |
| import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar |
| import groovy.json.JsonOutput |
| import groovy.json.JsonSlurper |
| import org.gradle.api.attributes.Category |
| import org.gradle.api.GradleException |
| import org.gradle.api.JavaVersion |
| import org.gradle.api.Plugin |
| import org.gradle.api.Project |
| import org.gradle.api.Task |
| import org.gradle.api.artifacts.Configuration |
| import org.gradle.api.artifacts.ProjectDependency |
| import org.gradle.api.file.FileCollection |
| import org.gradle.api.file.FileTree |
| import org.gradle.api.plugins.quality.Checkstyle |
| import org.gradle.api.publish.maven.MavenPublication |
| import org.gradle.api.tasks.Copy |
| import org.gradle.api.tasks.Delete |
| import org.gradle.api.tasks.Exec |
| import org.gradle.api.tasks.JavaExec |
| import org.gradle.api.tasks.TaskProvider |
| import org.gradle.api.tasks.bundling.Jar |
| import org.gradle.api.tasks.compile.JavaCompile |
| import org.gradle.api.tasks.javadoc.Javadoc |
| import org.gradle.api.tasks.testing.Test |
| import org.gradle.api.tasks.PathSensitive |
| import org.gradle.api.tasks.PathSensitivity |
| import org.gradle.testing.jacoco.tasks.JacocoReport |
| |
| import java.net.ServerSocket |
| /** |
| * This plugin adds methods to configure a module with Beam's defaults, called "natures". |
| * |
| * <p>The natures available: |
| * |
| * <ul> |
| * <li>Java - Configures plugins commonly found in Java projects |
| * <li>Go - Configures plugins commonly found in Go projects |
| * <li>Docker - Configures plugins commonly used to build Docker containers |
| * <li>Grpc - Configures plugins commonly used to generate source from protos |
| * <li>Avro - Configures plugins commonly used to generate source from Avro specifications |
| * </ul> |
| * |
| * <p>For example, see applyJavaNature. |
| */ |
| class BeamModulePlugin implements Plugin<Project> { |
| |
| /** Licence header enforced by spotless */ |
| static final String javaLicenseHeader = """/* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| """ |
| static def getRandomPort() { |
| new ServerSocket(0).withCloseable { socket -> |
| def port = socket.getLocalPort() |
| if (port > 0) { |
| return port |
| } else { |
| throw new GradleException("couldn't find a free port.") |
| } |
| } |
| } |
| |
| /** A class defining the set of configurable properties accepted by applyJavaNature. */ |
| static class JavaNatureConfiguration { |
| /** Controls whether the spotbugs plugin is enabled and configured. */ |
| boolean enableSpotbugs = true |
| |
| /** Regexes matching generated classes which should not receive extended type checking. */ |
| List<String> generatedClassPatterns = [] |
| |
| /** Classes triggering Checker failures. A map from class name to the bug filed against checkerframework. */ |
| Map<String, String> classesTriggerCheckerBugs = [:] |
| |
| /** Controls whether the dependency analysis plugin is enabled. */ |
| boolean enableStrictDependencies = true |
| |
| /** Override the default "beam-" + `dash separated path` archivesBaseName. */ |
| String archivesBaseName = null |
| |
| /** |
| * List of additional lint warnings to disable. |
| * In addition, defaultLintSuppressions defined below |
| * will be applied to all projects. |
| */ |
| List<String> disableLintWarnings = [] |
| |
| /** Controls whether tests are run with shadowJar. */ |
| boolean testShadowJar = false |
| |
| /** |
| * Controls whether the shadow jar is validated to not contain any classes outside the org.apache.beam namespace. |
| * This protects artifact jars from leaking dependencies classes causing conflicts for users. |
| * |
| * Note that this can be disabled for subprojects that produce application artifacts that are not intended to |
| * be depended on by users. |
| */ |
| boolean validateShadowJar = true |
| |
| /** |
| * Controls whether 'jmh' specific configuration is enabled to build a JMH |
| * focused module. |
| * |
| * Add additional dependencies to the implementation configuration. |
| * |
| * Note that the JMH annotation processor is enabled by default and that |
| * a 'jmh' task is created which executes JMH. |
| * |
| * Publishing is not allowed for JMH enabled projects. |
| */ |
| boolean enableJmh = false |
| |
| /** |
| * The set of excludes that should be used during validation of the shadow jar. Projects should override |
| * the default with the most specific set of excludes that is valid for the contents of its shaded jar. |
| * |
| * By default we exclude any class underneath the org.apache.beam namespace. |
| */ |
| List<String> shadowJarValidationExcludes = ["org/apache/beam/**"] |
| |
| /** |
| * If unset, no shading is performed. The jar and test jar archives are used during publishing. |
| * Otherwise the shadowJar and shadowTestJar artifacts are used during publishing. |
| * |
| * The shadowJar / shadowTestJar tasks execute the specified closure to configure themselves. |
| */ |
| Closure shadowClosure |
| |
| /** Controls whether this project is published to Maven. */ |
| boolean publish = true |
| |
| /** Controls whether javadoc is exported for this project. */ |
| boolean exportJavadoc = true |
| |
| /** |
| * Automatic-Module-Name Header value to be set in MANFIEST.MF file. |
| * This is a required parameter unless publishing to Maven is disabled for this project. |
| * |
| * @see: https://github.com/GoogleCloudPlatform/cloud-opensource-java/blob/master/library-best-practices/JLBP-20.md |
| */ |
| String automaticModuleName = null |
| |
| /** |
| * The set of additional maven repositories that should be added into published POM file. |
| */ |
| List<Map> mavenRepositories = [] |
| } |
| |
| /** A class defining the set of configurable properties accepted by applyPortabilityNature. */ |
| static class PortabilityNatureConfiguration { |
| /** |
| * The set of excludes that should be used during validation of the shadow jar. Projects should override |
| * the default with the most specific set of excludes that is valid for the contents of its shaded jar. |
| * |
| * By default we exclude any class underneath the org.apache.beam namespace. |
| */ |
| List<String> shadowJarValidationExcludes = ["org/apache/beam/**"] |
| |
| /** Override the default "beam-" + `dash separated path` archivesBaseName. */ |
| String archivesBaseName = null |
| |
| /** Controls whether this project is published to Maven. */ |
| boolean publish = true |
| |
| /** |
| * Regexes matching generated Java classes which should not receive extended type checking. |
| * |
| * By default, skips anything in the `org.apache.beam.model` namespace. |
| */ |
| List<String> generatedClassPatterns = [ |
| "^org\\.apache\\.beam\\.model.*" |
| ] |
| |
| /** |
| * Automatic-Module-Name Header value to be set in MANFIEST.MF file. |
| * This is a required parameter unless publishing to Maven is disabled for this project. |
| * |
| * @see: https://github.com/GoogleCloudPlatform/cloud-opensource-java/blob/master/library-best-practices/JLBP-20.md |
| */ |
| String automaticModuleName |
| } |
| |
| // A class defining the set of configurable properties for createJavaExamplesArchetypeValidationTask |
| static class JavaExamplesArchetypeValidationConfiguration { |
| // Type [Quickstart, MobileGaming] for the postrelease validation is required. |
| // Used both for the test name run${type}Java${runner} |
| // and also for the script name, ${type}-java-${runner}.toLowerCase(). |
| String type |
| |
| // runner [Direct, Dataflow, Spark, Flink, FlinkLocal] |
| String runner |
| |
| // gcpProject sets the gcpProject argument when executing examples. |
| String gcpProject |
| |
| // gcpRegion sets the region for executing Dataflow examples. |
| String gcpRegion |
| |
| // gcsBucket sets the gcsProject argument when executing examples. |
| String gcsBucket |
| |
| // bqDataset sets the BigQuery Dataset when executing mobile-gaming examples |
| String bqDataset |
| |
| // pubsubTopic sets topics when executing streaming pipelines |
| String pubsubTopic |
| } |
| |
| // Reads and contains all necessary performance test parameters |
| static class JavaPerformanceTestConfiguration { |
| // Optional. Runner which will be used for running the tests. Possible values: dataflow/direct. |
| // PerfKitBenchmarker will have trouble reading 'null' value. It expects empty string if no config file is expected. |
| String runner = System.getProperty('integrationTestRunner', '') |
| |
| // Optional. Filesystem which will be used for running the tests. Possible values: hdfs. |
| // if not specified runner's local filesystem will be used. |
| String filesystem = System.getProperty('filesystem') |
| |
| // Required. Pipeline options to be used by the tested pipeline. |
| String integrationTestPipelineOptions = System.getProperty('integrationTestPipelineOptions') |
| } |
| |
| // Reads and contains all necessary performance test parameters |
| static class PythonPerformanceTestConfiguration { |
| // Fully qualified name of the test to run. |
| String tests = System.getProperty('tests') |
| |
| // Attribute tag that can filter the test set. |
| String attribute = System.getProperty('attr') |
| |
| // Extra test options pass to pytest. |
| String[] extraTestOptions = ["--capture=no"] |
| |
| // Name of Cloud KMS encryption key to use in some tests. |
| String kmsKeyName = System.getProperty('kmsKeyName') |
| |
| // Pipeline options to be used for pipeline invocation. |
| String pipelineOptions = System.getProperty('pipelineOptions', '') |
| } |
| |
| // A class defining the set of configurable properties accepted by containerImageName. |
| static class ContainerImageNameConfiguration { |
| String root = null // Sets the docker repository root (optional). |
| String name = null // Sets the short container image name, such as "go" (required). |
| String tag = null // Sets the image tag (optional). |
| } |
| |
| // A class defining the configuration for PortableValidatesRunner. |
| static class PortableValidatesRunnerConfiguration { |
| // Task name for validate runner case. |
| String name = 'validatesPortableRunner' |
| // Fully qualified JobServerClass name to use. |
| String jobServerDriver |
| // A string representing the jobServer Configuration. |
| String jobServerConfig |
| // Number of parallel test runs. |
| Integer numParallelTests = 1 |
| // Extra options to pass to TestPipeline |
| String[] pipelineOpts = [] |
| // Spin up the Harness inside a DOCKER container |
| Environment environment = Environment.DOCKER |
| // Categories for tests to run. |
| Closure testCategories = { |
| includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' |
| // Use the following to include / exclude categories: |
| // includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' |
| // excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders' |
| } |
| // Tests to include/exclude from running, by default all tests are included |
| Closure testFilter = { |
| // Use the following to include / exclude tests: |
| // includeTestsMatching 'org.apache.beam.sdk.transforms.FlattenTest.testFlattenWithDifferentInputAndOutputCoders2' |
| // excludeTestsMatching 'org.apache.beam.sdk.transforms.FlattenTest.testFlattenWithDifferentInputAndOutputCoders2' |
| } |
| // Configuration for the classpath when running the test. |
| Configuration testClasspathConfiguration |
| // Additional system properties. |
| Properties systemProperties = [] |
| |
| enum Environment { |
| DOCKER, // Docker-based Harness execution |
| PROCESS, // Process-based Harness execution |
| EMBEDDED, // Execute directly inside the execution engine (testing only) |
| } |
| } |
| |
| // A class defining the configuration for CrossLanguageValidatesRunner. |
| static class CrossLanguageValidatesRunnerConfiguration { |
| // Task name for cross-language validate runner case. |
| String name = 'validatesCrossLanguageRunner' |
| // Java pipeline options to use. |
| List<String> javaPipelineOptions = [ |
| "--runner=PortableRunner", |
| "--jobEndpoint=localhost:8099", |
| "--environmentCacheMillis=10000", |
| "--experiments=beam_fn_api", |
| ] |
| // Python pipeline options to use. |
| List<String> pythonPipelineOptions = [ |
| "--runner=PortableRunner", |
| "--job_endpoint=localhost:8099", |
| "--environment_cache_millis=10000", |
| "--experiments=beam_fn_api", |
| ] |
| // Go script options to use. |
| List<String> goScriptOptions = [ |
| "--runner portable", |
| "--endpoint localhost:8099", |
| "--tests \"./test/integration/xlang ./test/integration/io/xlang/...\"" |
| ] |
| // Additional pytest options |
| List<String> pytestOptions = [] |
| // Job server startup task. |
| TaskProvider startJobServer |
| // Job server cleanup task. |
| TaskProvider cleanupJobServer |
| // Number of parallel test runs. |
| Integer numParallelTests = 1 |
| // Whether the pipeline needs --sdk_location option |
| boolean needsSdkLocation = false |
| // semi_persist_dir for SDK containers |
| String semiPersistDir = "/tmp" |
| // classpath for running tests. |
| FileCollection classpath |
| } |
| |
| def isRelease(Project project) { |
| return parseBooleanProperty(project, 'isRelease'); |
| } |
| |
| /** |
| * Parses -Pprop as true for use as a flag, and otherwise uses Groovy's toBoolean |
| */ |
| def parseBooleanProperty(Project project, String property) { |
| if (!project.hasProperty(property)) { |
| return false; |
| } |
| |
| if (project.getProperty(property) == "") { |
| return true; |
| } |
| |
| return project.getProperty(property).toBoolean(); |
| } |
| |
| def defaultArchivesBaseName(Project p) { |
| return 'beam' + p.path.replace(':', '-') |
| } |
| |
| void apply(Project project) { |
| |
| /** ***********************************************************************************************/ |
| // Apply common properties/repositories and tasks to all projects. |
| |
| project.ext.mavenGroupId = 'org.apache.beam' |
| |
| // Automatically use the official release version if we are performing a release |
| // otherwise append '-SNAPSHOT' |
| project.version = '2.41.0' |
| if (!isRelease(project)) { |
| project.version += '-SNAPSHOT' |
| } |
| |
| // Default to dash-separated directories for artifact base name, |
| // which will also be the default artifactId for maven publications |
| project.apply plugin: 'base' |
| project.archivesBaseName = defaultArchivesBaseName(project) |
| |
| project.apply plugin: 'org.apache.beam.jenkins' |
| |
| // Register all Beam repositories and configuration tweaks |
| Repositories.register(project) |
| |
| // Apply a plugin which enables configuring projects imported into Intellij. |
| project.apply plugin: "idea" |
| |
| // Provide code coverage |
| // TODO: Should this only apply to Java projects? |
| project.apply plugin: "jacoco" |
| project.gradle.taskGraph.whenReady { graph -> |
| // Disable jacoco unless report requested such that task outputs can be properly cached. |
| // https://discuss.gradle.org/t/do-not-cache-if-condition-matched-jacoco-agent-configured-with-append-true-satisfied/23504 |
| def enabled = graph.allTasks.any { it instanceof JacocoReport || it.name.contains("javaPreCommit") } |
| project.tasks.withType(Test) { jacoco.enabled = enabled } |
| } |
| |
| // Apply a plugin which provides tasks for dependency / property / task reports. |
| // See https://docs.gradle.org/current/userguide/project_reports_plugin.html |
| // for further details. This is typically very useful to look at the "htmlDependencyReport" |
| // when attempting to resolve dependency issues. |
| project.apply plugin: "project-report" |
| |
| // Apply a plugin which fails the build if there is a dependency on a transitive |
| // non-declared dependency, since these can break users (as in BEAM-6558) |
| // |
| // Though this is Java-specific, it is required to be applied to the root |
| // project due to implementation-details of the plugin. It can be enabled/disabled |
| // via JavaNatureConfiguration per project. It is disabled by default until we can |
| // make all of our deps good. |
| project.apply plugin: "ca.cutterslade.analyze" |
| |
| // Adds a taskTree task that prints task dependency tree report to the console. |
| // Useful for investigating build issues. |
| // See: https://github.com/dorongold/gradle-task-tree |
| project.apply plugin: "com.dorongold.task-tree" |
| project.taskTree { noRepeat = true } |
| |
| project.ext.allFlinkVersions = project.flink_versions.split(',') |
| project.ext.latestFlinkVersion = project.ext.allFlinkVersions.last() |
| |
| /** ***********************************************************************************************/ |
| // Define and export a map dependencies shared across multiple sub-projects. |
| // |
| // Example usage: |
| // configuration { |
| // implementation library.java.avro |
| // testImplementation library.java.junit |
| // } |
| |
| // These versions are defined here because they represent |
| // a dependency version which should match across multiple |
| // Maven artifacts. |
| def activemq_version = "5.14.5" |
| def autovalue_version = "1.9" |
| def autoservice_version = "1.0.1" |
| def aws_java_sdk_version = "1.12.135" |
| def aws_java_sdk2_version = "2.17.127" |
| def cassandra_driver_version = "3.10.2" |
| def cdap_version = "6.5.1" |
| def checkerframework_version = "3.10.0" |
| def classgraph_version = "4.8.104" |
| def errorprone_version = "2.10.0" |
| // Try to keep gax_version consistent with gax-grpc version in google_cloud_platform_libraries_bom |
| def gax_version = "2.16.0" |
| def google_clients_version = "1.32.1" |
| def google_cloud_bigdataoss_version = "2.2.6" |
| // Try to keep google_cloud_spanner_version consistent with google_cloud_spanner_bom in google_cloud_platform_libraries_bom |
| def google_cloud_spanner_version = "6.23.3" |
| def google_code_gson_version = "2.9.0" |
| def google_oauth_clients_version = "1.33.3" |
| // Try to keep grpc_version consistent with gRPC version in google_cloud_platform_libraries_bom |
| def grpc_version = "1.45.1" |
| def guava_version = "31.1-jre" |
| def hadoop_version = "2.10.1" |
| def hamcrest_version = "2.1" |
| def influxdb_version = "2.19" |
| def httpclient_version = "4.5.13" |
| def httpcore_version = "4.4.14" |
| def jackson_version = "2.13.0" |
| def jaxb_api_version = "2.3.3" |
| def jsr305_version = "3.0.2" |
| def kafka_version = "2.4.1" |
| def nemo_version = "0.1" |
| def netty_version = "4.1.52.Final" |
| def postgres_version = "42.2.16" |
| def powermock_version = "2.0.9" |
| // Try to keep protobuf_version consistent with the protobuf version in google_cloud_platform_libraries_bom |
| def protobuf_version = "3.19.4" |
| def quickcheck_version = "1.0" |
| def sbe_tool_version = "1.25.1" |
| def slf4j_version = "1.7.30" |
| def spark2_version = "2.4.8" |
| def spark3_version = "3.1.2" |
| def spotbugs_version = "4.0.6" |
| def testcontainers_version = "1.16.3" |
| def arrow_version = "5.0.0" |
| def jmh_version = "1.34" |
| |
| // A map of maps containing common libraries used per language. To use: |
| // dependencies { |
| // compile library.java.slf4j_api |
| // } |
| project.ext.library = [ |
| java : [ |
| activemq_amqp : "org.apache.activemq:activemq-amqp:$activemq_version", |
| activemq_broker : "org.apache.activemq:activemq-broker:$activemq_version", |
| activemq_client : "org.apache.activemq:activemq-client:$activemq_version", |
| activemq_jaas : "org.apache.activemq:activemq-jaas:$activemq_version", |
| activemq_junit : "org.apache.activemq.tooling:activemq-junit:$activemq_version", |
| activemq_kahadb_store : "org.apache.activemq:activemq-kahadb-store:$activemq_version", |
| activemq_mqtt : "org.apache.activemq:activemq-mqtt:$activemq_version", |
| antlr : "org.antlr:antlr4:4.7", |
| antlr_runtime : "org.antlr:antlr4-runtime:4.7", |
| args4j : "args4j:args4j:2.33", |
| auto_value_annotations : "com.google.auto.value:auto-value-annotations:$autovalue_version", |
| avro : "org.apache.avro:avro:1.8.2", |
| avro_tests : "org.apache.avro:avro:1.8.2:tests", |
| aws_java_sdk_cloudwatch : "com.amazonaws:aws-java-sdk-cloudwatch:$aws_java_sdk_version", |
| aws_java_sdk_core : "com.amazonaws:aws-java-sdk-core:$aws_java_sdk_version", |
| aws_java_sdk_dynamodb : "com.amazonaws:aws-java-sdk-dynamodb:$aws_java_sdk_version", |
| aws_java_sdk_kinesis : "com.amazonaws:aws-java-sdk-kinesis:$aws_java_sdk_version", |
| aws_java_sdk_s3 : "com.amazonaws:aws-java-sdk-s3:$aws_java_sdk_version", |
| aws_java_sdk_sns : "com.amazonaws:aws-java-sdk-sns:$aws_java_sdk_version", |
| aws_java_sdk_sqs : "com.amazonaws:aws-java-sdk-sqs:$aws_java_sdk_version", |
| aws_java_sdk_sts : "com.amazonaws:aws-java-sdk-sts:$aws_java_sdk_version", |
| aws_java_sdk2_apache_client : "software.amazon.awssdk:apache-client:$aws_java_sdk2_version", |
| aws_java_sdk2_netty_client : "software.amazon.awssdk:netty-nio-client:$aws_java_sdk2_version", |
| aws_java_sdk2_auth : "software.amazon.awssdk:auth:$aws_java_sdk2_version", |
| aws_java_sdk2_cloudwatch : "software.amazon.awssdk:cloudwatch:$aws_java_sdk2_version", |
| aws_java_sdk2_dynamodb : "software.amazon.awssdk:dynamodb:$aws_java_sdk2_version", |
| aws_java_sdk2_kinesis : "software.amazon.awssdk:kinesis:$aws_java_sdk2_version", |
| aws_java_sdk2_sdk_core : "software.amazon.awssdk:sdk-core:$aws_java_sdk2_version", |
| aws_java_sdk2_aws_core : "software.amazon.awssdk:aws-core:$aws_java_sdk2_version", |
| aws_java_sdk2_sns : "software.amazon.awssdk:sns:$aws_java_sdk2_version", |
| aws_java_sdk2_sqs : "software.amazon.awssdk:sqs:$aws_java_sdk2_version", |
| aws_java_sdk2_sts : "software.amazon.awssdk:sts:$aws_java_sdk2_version", |
| aws_java_sdk2_s3 : "software.amazon.awssdk:s3:$aws_java_sdk2_version", |
| aws_java_sdk2_http_client_spi : "software.amazon.awssdk:http-client-spi:$aws_java_sdk2_version", |
| aws_java_sdk2_regions : "software.amazon.awssdk:regions:$aws_java_sdk2_version", |
| aws_java_sdk2_utils : "software.amazon.awssdk:utils:$aws_java_sdk2_version", |
| bigdataoss_gcsio : "com.google.cloud.bigdataoss:gcsio:$google_cloud_bigdataoss_version", |
| bigdataoss_util : "com.google.cloud.bigdataoss:util:$google_cloud_bigdataoss_version", |
| byte_buddy : "net.bytebuddy:byte-buddy:1.12.9", |
| cassandra_driver_core : "com.datastax.cassandra:cassandra-driver-core:$cassandra_driver_version", |
| cassandra_driver_mapping : "com.datastax.cassandra:cassandra-driver-mapping:$cassandra_driver_version", |
| cdap_api : "io.cdap.cdap:cdap-api:$cdap_version", |
| cdap_api_commons : "io.cdap.cdap:cdap-api-common:$cdap_version", |
| cdap_common : "io.cdap.cdap:cdap-common:$cdap_version", |
| cdap_etl_api : "io.cdap.cdap:cdap-etl-api:$cdap_version", |
| cdap_etl_api_spark : "io.cdap.cdap:cdap-etl-api-spark:$cdap_version", |
| cdap_hydrator_common : "io.cdap.plugin:hydrator-common:2.4.0", |
| cdap_plugin_hubspot : "io.cdap:hubspot-plugins:1.0.0", |
| cdap_plugin_salesforce : "io.cdap.plugin:salesforce-plugins:1.4.0", |
| cdap_plugin_service_now : "io.cdap.plugin:servicenow-plugins:1.1.0", |
| cdap_plugin_zendesk : "io.cdap.plugin:zendesk-plugins:1.0.0", |
| checker_qual : "org.checkerframework:checker-qual:$checkerframework_version", |
| classgraph : "io.github.classgraph:classgraph:$classgraph_version", |
| commons_codec : "commons-codec:commons-codec:1.15", |
| commons_compress : "org.apache.commons:commons-compress:1.21", |
| commons_csv : "org.apache.commons:commons-csv:1.8", |
| commons_io : "commons-io:commons-io:2.6", |
| commons_lang3 : "org.apache.commons:commons-lang3:3.9", |
| commons_math3 : "org.apache.commons:commons-math3:3.6.1", |
| error_prone_annotations : "com.google.errorprone:error_prone_annotations:$errorprone_version", |
| flogger_system_backend : "com.google.flogger:flogger-system-backend:0.7.3", |
| gax : "com.google.api:gax", // google_cloud_platform_libraries_bom sets version |
| gax_grpc : "com.google.api:gax-grpc", // google_cloud_platform_libraries_bom sets version |
| gax_grpc_test : "com.google.api:gax-grpc:$gax_version:testlib", // google_cloud_platform_libraries_bom sets version |
| gax_httpjson : "com.google.api:gax-httpjson", // google_cloud_platform_libraries_bom sets version |
| google_api_client : "com.google.api-client:google-api-client:$google_clients_version", // for the libraries using $google_clients_version below. |
| google_api_client_jackson2 : "com.google.api-client:google-api-client-jackson2:$google_clients_version", |
| google_api_client_java6 : "com.google.api-client:google-api-client-java6:$google_clients_version", |
| google_api_common : "com.google.api:api-common", // google_cloud_platform_libraries_bom sets version |
| google_api_services_bigquery : "com.google.apis:google-api-services-bigquery:v2-rev20211129-$google_clients_version", |
| google_api_services_clouddebugger : "com.google.apis:google-api-services-clouddebugger:v2-rev20210813-$google_clients_version", |
| google_api_services_cloudresourcemanager : "com.google.apis:google-api-services-cloudresourcemanager:v1-rev20211017-$google_clients_version", |
| google_api_services_dataflow : "com.google.apis:google-api-services-dataflow:v1b3-rev20210818-$google_clients_version", |
| google_api_services_healthcare : "com.google.apis:google-api-services-healthcare:v1-rev20211016-$google_clients_version", |
| google_api_services_pubsub : "com.google.apis:google-api-services-pubsub:v1-rev20211130-$google_clients_version", |
| google_api_services_storage : "com.google.apis:google-api-services-storage:v1-rev20211201-$google_clients_version", |
| google_auth_library_credentials : "com.google.auth:google-auth-library-credentials", // google_cloud_platform_libraries_bom sets version |
| google_auth_library_oauth2_http : "com.google.auth:google-auth-library-oauth2-http", // google_cloud_platform_libraries_bom sets version |
| google_cloud_bigquery : "com.google.cloud:google-cloud-bigquery", // google_cloud_platform_libraries_bom sets version |
| google_cloud_bigquery_storage : "com.google.cloud:google-cloud-bigquerystorage", // google_cloud_platform_libraries_bom sets version |
| google_cloud_bigtable : "com.google.cloud:google-cloud-bigtable", // google_cloud_platform_libraries_bom sets version |
| google_cloud_bigtable_client_core : "com.google.cloud.bigtable:bigtable-client-core:1.26.3", |
| google_cloud_bigtable_emulator : "com.google.cloud:google-cloud-bigtable-emulator:0.137.1", |
| google_cloud_core : "com.google.cloud:google-cloud-core", // google_cloud_platform_libraries_bom sets version |
| google_cloud_core_grpc : "com.google.cloud:google-cloud-core-grpc", // google_cloud_platform_libraries_bom sets version |
| google_cloud_datacatalog_v1beta1 : "com.google.cloud:google-cloud-datacatalog", // google_cloud_platform_libraries_bom sets version |
| google_cloud_dataflow_java_proto_library_all: "com.google.cloud.dataflow:google-cloud-dataflow-java-proto-library-all:0.5.160304", |
| google_cloud_datastore_v1_proto_client : "com.google.cloud.datastore:datastore-v1-proto-client:2.2.10", |
| google_cloud_firestore : "com.google.cloud:google-cloud-firestore", // google_cloud_platform_libraries_bom sets version |
| google_cloud_pubsub : "com.google.cloud:google-cloud-pubsub", // google_cloud_platform_libraries_bom sets version |
| google_cloud_pubsublite : "com.google.cloud:google-cloud-pubsublite", // google_cloud_platform_libraries_bom sets version |
| // The GCP Libraries BOM dashboard shows the versions set by the BOM: |
| // https://storage.googleapis.com/cloud-opensource-java-dashboard/com.google.cloud/libraries-bom/25.2.0/artifact_details.html |
| // Update libraries-bom version on sdks/java/container/license_scripts/dep_urls_java.yaml |
| google_cloud_platform_libraries_bom : "com.google.cloud:libraries-bom:25.2.0", |
| google_cloud_spanner : "com.google.cloud:google-cloud-spanner", // google_cloud_platform_libraries_bom sets version |
| google_cloud_spanner_test : "com.google.cloud:google-cloud-spanner:$google_cloud_spanner_version:tests", |
| google_code_gson : "com.google.code.gson:gson:$google_code_gson_version", |
| // google-http-client's version is explicitly declared for sdks/java/maven-archetypes/examples |
| // This version should be in line with the one in com.google.cloud:libraries-bom. |
| google_http_client : "com.google.http-client:google-http-client", // google_cloud_platform_libraries_bom sets version |
| google_http_client_apache_v2 : "com.google.http-client:google-http-client-apache-v2", // google_cloud_platform_libraries_bom sets version |
| google_http_client_gson : "com.google.http-client:google-http-client-gson", // google_cloud_platform_libraries_bom sets version |
| google_http_client_jackson : "com.google.http-client:google-http-client-jackson:1.29.2", |
| google_http_client_jackson2 : "com.google.http-client:google-http-client-jackson2", // google_cloud_platform_libraries_bom sets version |
| google_http_client_protobuf : "com.google.http-client:google-http-client-protobuf", // google_cloud_platform_libraries_bom sets version |
| google_oauth_client : "com.google.oauth-client:google-oauth-client:$google_oauth_clients_version", |
| google_oauth_client_java6 : "com.google.oauth-client:google-oauth-client-java6:$google_oauth_clients_version", |
| // Don't use grpc_all, it can cause issues in Bazel builds. Reference the gRPC libraries you need individually instead. |
| grpc_alts : "io.grpc:grpc-alts", // google_cloud_platform_libraries_bom sets version |
| grpc_api : "io.grpc:grpc-api", // google_cloud_platform_libraries_bom sets version |
| grpc_auth : "io.grpc:grpc-auth", // google_cloud_platform_libraries_bom sets version |
| // Once grpc-xds is added to google_cloud_platform_libraries_bom, use google_cloud_platform_libraries_bom instead. |
| grpc_census : "io.grpc:grpc-census:$grpc_version", |
| grpc_context : "io.grpc:grpc-context", // google_cloud_platform_libraries_bom sets version |
| grpc_core : "io.grpc:grpc-core", // google_cloud_platform_libraries_bom sets version |
| grpc_google_cloud_firestore_v1 : "com.google.api.grpc:grpc-google-cloud-firestore-v1", // google_cloud_platform_libraries_bom sets version |
| grpc_google_cloud_pubsub_v1 : "com.google.api.grpc:grpc-google-cloud-pubsub-v1", // google_cloud_platform_libraries_bom sets version |
| grpc_google_cloud_pubsublite_v1 : "com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.5.0", // TODO(dpcollins-google): Let google_cloud_platform_libraries_bom set version once high enough |
| grpc_google_common_protos : "com.google.api.grpc:grpc-google-common-protos", // google_cloud_platform_libraries_bom sets version |
| grpc_grpclb : "io.grpc:grpc-grpclb", // google_cloud_platform_libraries_bom sets version |
| grpc_protobuf : "io.grpc:grpc-protobuf", // google_cloud_platform_libraries_bom sets version |
| grpc_protobuf_lite : "io.grpc:grpc-protobuf-lite:$grpc_version", |
| grpc_netty : "io.grpc:grpc-netty", // google_cloud_platform_libraries_bom sets version |
| grpc_netty_shaded : "io.grpc:grpc-netty-shaded", // google_cloud_platform_libraries_bom sets version |
| grpc_stub : "io.grpc:grpc-stub", // google_cloud_platform_libraries_bom sets version |
| // Once grpc-xds is added to google_cloud_platform_libraries_bom, use google_cloud_platform_libraries_bom instead. |
| grpc_xds : "io.grpc:grpc-xds:$grpc_version", |
| guava : "com.google.guava:guava:$guava_version", |
| guava_testlib : "com.google.guava:guava-testlib:$guava_version", |
| hadoop_client : "org.apache.hadoop:hadoop-client:$hadoop_version", |
| hadoop_common : "org.apache.hadoop:hadoop-common:$hadoop_version", |
| hadoop_mapreduce_client_core : "org.apache.hadoop:hadoop-mapreduce-client-core:$hadoop_version", |
| hadoop_minicluster : "org.apache.hadoop:hadoop-minicluster:$hadoop_version", |
| hadoop_hdfs : "org.apache.hadoop:hadoop-hdfs:$hadoop_version", |
| hadoop_hdfs_client : "org.apache.hadoop:hadoop-hdfs-client:$hadoop_version", |
| hadoop_hdfs_tests : "org.apache.hadoop:hadoop-hdfs:$hadoop_version:tests", |
| hamcrest : "org.hamcrest:hamcrest:$hamcrest_version", |
| http_client : "org.apache.httpcomponents:httpclient:$httpclient_version", |
| http_core : "org.apache.httpcomponents:httpcore:$httpcore_version", |
| influxdb_library : "org.influxdb:influxdb-java:$influxdb_version", |
| jackson_annotations : "com.fasterxml.jackson.core:jackson-annotations:$jackson_version", |
| jackson_jaxb_annotations : "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:$jackson_version", |
| jackson_core : "com.fasterxml.jackson.core:jackson-core:$jackson_version", |
| jackson_databind : "com.fasterxml.jackson.core:jackson-databind:$jackson_version", |
| jackson_dataformat_cbor : "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:$jackson_version", |
| jackson_dataformat_csv : "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:$jackson_version", |
| jackson_dataformat_xml : "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:$jackson_version", |
| jackson_dataformat_yaml : "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:$jackson_version", |
| jackson_datatype_joda : "com.fasterxml.jackson.datatype:jackson-datatype-joda:$jackson_version", |
| jackson_datatype_jsr310 : "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jackson_version", |
| jackson_module_scala_2_11 : "com.fasterxml.jackson.module:jackson-module-scala_2.11:$jackson_version", |
| jackson_module_scala_2_12 : "com.fasterxml.jackson.module:jackson-module-scala_2.12:$jackson_version", |
| // Swap to use the officially published version of 0.4.x once available |
| // instead of relying on a community published copy. See |
| // https://github.com/jbellis/jamm/issues/44 for additional details. |
| jamm : 'io.github.stephankoelle:jamm:0.4.1', |
| jaxb_api : "jakarta.xml.bind:jakarta.xml.bind-api:$jaxb_api_version", |
| jaxb_impl : "com.sun.xml.bind:jaxb-impl:$jaxb_api_version", |
| joda_time : "joda-time:joda-time:2.10.10", |
| jsonassert : "org.skyscreamer:jsonassert:1.5.0", |
| jsr305 : "com.google.code.findbugs:jsr305:$jsr305_version", |
| junit : "junit:junit:4.13.1", |
| kafka : "org.apache.kafka:kafka_2.11:$kafka_version", |
| kafka_clients : "org.apache.kafka:kafka-clients:$kafka_version", |
| mockito_core : "org.mockito:mockito-core:3.7.7", |
| mockito_inline : "org.mockito:mockito-inline:4.5.1", |
| mongo_java_driver : "org.mongodb:mongo-java-driver:3.12.10", |
| nemo_compiler_frontend_beam : "org.apache.nemo:nemo-compiler-frontend-beam:$nemo_version", |
| netty_all : "io.netty:netty-all:$netty_version", |
| netty_handler : "io.netty:netty-handler:$netty_version", |
| netty_tcnative_boringssl_static : "io.netty:netty-tcnative-boringssl-static:2.0.47.Final", |
| netty_transport_native_epoll : "io.netty:netty-transport-native-epoll:$netty_version", |
| postgres : "org.postgresql:postgresql:$postgres_version", |
| powermock : "org.powermock:powermock-module-junit4:$powermock_version", |
| powermock_mockito : "org.powermock:powermock-api-mockito2:$powermock_version", |
| protobuf_java : "com.google.protobuf:protobuf-java:$protobuf_version", |
| protobuf_java_util : "com.google.protobuf:protobuf-java-util:$protobuf_version", |
| proto_google_cloud_bigquery_storage_v1 : "com.google.api.grpc:proto-google-cloud-bigquerystorage-v1", // google_cloud_platform_libraries_bom sets version |
| proto_google_cloud_bigtable_admin_v2 : "com.google.api.grpc:proto-google-cloud-bigtable-admin-v2", // google_cloud_platform_libraries_bom sets version |
| proto_google_cloud_bigtable_v2 : "com.google.api.grpc:proto-google-cloud-bigtable-v2", // google_cloud_platform_libraries_bom sets version |
| proto_google_cloud_datacatalog_v1beta1 : "com.google.api.grpc:proto-google-cloud-datacatalog-v1beta1", // google_cloud_platform_libraries_bom sets version |
| proto_google_cloud_datastore_v1 : "com.google.api.grpc:proto-google-cloud-datastore-v1:0.93.10", // google_cloud_platform_libraries_bom sets version |
| proto_google_cloud_firestore_v1 : "com.google.api.grpc:proto-google-cloud-firestore-v1", // google_cloud_platform_libraries_bom sets version |
| proto_google_cloud_pubsub_v1 : "com.google.api.grpc:proto-google-cloud-pubsub-v1", // google_cloud_platform_libraries_bom sets version |
| proto_google_cloud_pubsublite_v1 : "com.google.api.grpc:proto-google-cloud-pubsublite-v1", // google_cloud_platform_libraries_bom sets version |
| proto_google_cloud_spanner_v1 : "com.google.api.grpc:proto-google-cloud-spanner-v1", // google_cloud_platform_libraries_bom sets version |
| proto_google_cloud_spanner_admin_database_v1: "com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1", // google_cloud_platform_libraries_bom sets version |
| proto_google_common_protos : "com.google.api.grpc:proto-google-common-protos", // google_cloud_platform_libraries_bom sets version |
| sbe_tool : "uk.co.real-logic:sbe-tool:$sbe_tool_version", |
| slf4j_api : "org.slf4j:slf4j-api:$slf4j_version", |
| slf4j_simple : "org.slf4j:slf4j-simple:$slf4j_version", |
| slf4j_jdk14 : "org.slf4j:slf4j-jdk14:$slf4j_version", |
| slf4j_log4j12 : "org.slf4j:slf4j-log4j12:$slf4j_version", |
| snappy_java : "org.xerial.snappy:snappy-java:1.1.8.4", |
| spark_core : "org.apache.spark:spark-core_2.11:$spark2_version", |
| spark_network_common : "org.apache.spark:spark-network-common_2.11:$spark2_version", |
| spark_sql : "org.apache.spark:spark-sql_2.11:$spark2_version", |
| spark_streaming : "org.apache.spark:spark-streaming_2.11:$spark2_version", |
| spark3_core : "org.apache.spark:spark-core_2.12:$spark3_version", |
| spark3_network_common : "org.apache.spark:spark-network-common_2.12:$spark3_version", |
| spark3_sql : "org.apache.spark:spark-sql_2.12:$spark3_version", |
| spark3_streaming : "org.apache.spark:spark-streaming_2.12:$spark3_version", |
| stax2_api : "org.codehaus.woodstox:stax2-api:4.2.1", |
| tephra : "org.apache.tephra:tephra-api:0.15.0-incubating", |
| testcontainers_base : "org.testcontainers:testcontainers:$testcontainers_version", |
| testcontainers_clickhouse : "org.testcontainers:clickhouse:$testcontainers_version", |
| testcontainers_elasticsearch : "org.testcontainers:elasticsearch:$testcontainers_version", |
| testcontainers_kafka : "org.testcontainers:kafka:$testcontainers_version", |
| testcontainers_localstack : "org.testcontainers:localstack:$testcontainers_version", |
| testcontainers_postgresql : "org.testcontainers:postgresql:$testcontainers_version", |
| testcontainers_mysql : "org.testcontainers:mysql:$testcontainers_version", |
| testcontainers_gcloud : "org.testcontainers:gcloud:$testcontainers_version", |
| vendored_grpc_1_43_2 : "org.apache.beam:beam-vendor-grpc-1_43_2:0.1", |
| vendored_guava_26_0_jre : "org.apache.beam:beam-vendor-guava-26_0-jre:0.1", |
| vendored_calcite_1_28_0 : "org.apache.beam:beam-vendor-calcite-1_28_0:0.2", |
| woodstox_core_asl : "org.codehaus.woodstox:woodstox-core-asl:4.4.1", |
| zstd_jni : "com.github.luben:zstd-jni:1.5.2-1", |
| quickcheck_core : "com.pholser:junit-quickcheck-core:$quickcheck_version", |
| quickcheck_generators : "com.pholser:junit-quickcheck-generators:$quickcheck_version", |
| arrow_vector : "org.apache.arrow:arrow-vector:$arrow_version", |
| arrow_memory_core : "org.apache.arrow:arrow-memory-core:$arrow_version", |
| arrow_memory_netty : "org.apache.arrow:arrow-memory-netty:$arrow_version", |
| ], |
| groovy: [ |
| groovy_all: "org.codehaus.groovy:groovy-all:2.4.13", |
| ], |
| // For generating pom.xml from archetypes. |
| maven: [ |
| maven_compiler_plugin: "maven-plugins:maven-compiler-plugin:3.7.0", |
| maven_exec_plugin : "maven-plugins:maven-exec-plugin:1.6.0", |
| maven_jar_plugin : "maven-plugins:maven-jar-plugin:3.0.2", |
| maven_shade_plugin : "maven-plugins:maven-shade-plugin:3.1.0", |
| maven_surefire_plugin: "maven-plugins:maven-surefire-plugin:3.0.0-M5", |
| ], |
| ] |
| |
| /** ***********************************************************************************************/ |
| |
| // Returns a string representing the relocated path to be used with the shadow plugin when |
| // given a suffix such as "com.google.common". |
| project.ext.getJavaRelocatedPath = { String suffix -> |
| return ("org.apache.beam.repackaged." |
| + project.name.replace("-", "_") |
| + "." |
| + suffix) |
| } |
| |
| project.ext.repositories = { |
| maven { |
| name "testPublicationLocal" |
| url "file://${project.rootProject.projectDir}/testPublication/" |
| } |
| maven { |
| url(project.properties['distMgmtSnapshotsUrl'] ?: isRelease(project) |
| ? 'https://repository.apache.org/service/local/staging/deploy/maven2' |
| : 'https://repository.apache.org/content/repositories/snapshots') |
| name(project.properties['distMgmtServerId'] ?: isRelease(project) |
| ? 'apache.releases.https' : 'apache.snapshots.https') |
| // The maven settings plugin will load credentials from ~/.m2/settings.xml file that a user |
| // has configured with the Apache release and snapshot staging credentials. |
| // <settings> |
| // <servers> |
| // <server> |
| // <id>apache.releases.https</id> |
| // <username>USER_TOKEN</username> |
| // <password>PASS_TOKEN</password> |
| // </server> |
| // <server> |
| // <id>apache.snapshots.https</id> |
| // <username>USER_TOKEN</username> |
| // <password>PASS_TOKEN</password> |
| // </server> |
| // </servers> |
| // </settings> |
| } |
| } |
| |
| // Configures a project with a default set of plugins that should apply to all Java projects. |
| // |
| // Users should invoke this method using Groovy map syntax. For example: |
| // applyJavaNature(enableSpotbugs: true) |
| // |
| // See JavaNatureConfiguration for the set of accepted properties. |
| // |
| // The following plugins are enabled: |
| // * java |
| // * maven |
| // * net.ltgt.apt (plugin to configure annotation processing tool) |
| // * checkstyle |
| // * spotbugs |
| // * shadow (conditional on shadowClosure being specified) |
| // * com.diffplug.spotless (code style plugin) |
| // |
| // Dependency Management for Java Projects |
| // --------------------------------------- |
| // |
| // By default, the shadow plugin is not enabled. It is only enabled by specifying a shadowClosure |
| // as an argument. If no shadowClosure has been specified, dependencies should fall into the |
| // configurations as described within the Gradle documentation (https://docs.gradle.org/current/userguide/java_plugin.html#sec:java_plugin_and_dependency_management) |
| // |
| // When the shadowClosure argument is specified, the shadow plugin is enabled to perform shading |
| // of commonly found dependencies. Because of this it is important that dependencies are added |
| // to the correct configuration. Dependencies should fall into one of these four configurations: |
| // * implementation - Required during compilation or runtime of the main source set. |
| // This configuration represents all dependencies that must also be shaded away |
| // otherwise the generated Maven pom will be missing this dependency. |
| // * shadow - Required during compilation or runtime of the main source set. |
| // Will become a runtime dependency of the generated Maven pom. |
| // * testImplementation - Required during compilation or runtime of the test source set. |
| // This must be shaded away in the shaded test jar. |
| // * shadowTest - Required during compilation or runtime of the test source set. |
| // TODO: Figure out whether this should be a test scope dependency |
| // of the generated Maven pom. |
| // |
| // When creating a cross-project dependency between two Java projects, one should only rely on |
| // the shaded configurations if the project has a shadowClosure being specified. This allows |
| // for compilation/test execution to occur against the final artifact that will be provided to |
| // users. This is by done by referencing the "shadow" or "shadowTest" configuration as so: |
| // dependencies { |
| // shadow project(path: "other:java:project1", configuration: "shadow") |
| // shadowTest project(path: "other:java:project2", configuration: "shadowTest") |
| // } |
| // This will ensure the correct set of transitive dependencies from those projects are correctly |
| // added to the main and test source set runtimes. |
| |
| project.ext.applyJavaNature = { |
| // Use the implicit it parameter of the closure to handle zero argument or one argument map calls. |
| JavaNatureConfiguration configuration = it ? it as JavaNatureConfiguration : new JavaNatureConfiguration() |
| |
| // Validate configuration |
| if (configuration.enableJmh && configuration.publish) { |
| throw new GradleException("Publishing of a benchmark project is not allowed. Benchmark projects are not meant to be consumed as artifacts for end users."); |
| } |
| |
| if (configuration.archivesBaseName) { |
| project.archivesBaseName = configuration.archivesBaseName |
| } |
| |
| project.apply plugin: "java" |
| |
| // We create a testRuntimeMigration configuration here to extend |
| // testImplementation, testRuntimeOnly, and default (similar to what |
| // testRuntime did). |
| project.configurations { |
| testRuntimeMigration.extendsFrom(project.configurations.default) |
| testRuntimeMigration.extendsFrom(testImplementation) |
| testRuntimeMigration.extendsFrom(testRuntimeOnly) |
| } |
| |
| // Provided configuration to match Maven provided scope |
| project.configurations { |
| provided |
| compileOnly.extendsFrom(provided) |
| runtimeOnly.extendsFrom(provided) |
| } |
| |
| // Configure the Java compiler source language and target compatibility levels. Also ensure that |
| // we configure the Java compiler to use UTF-8. |
| project.sourceCompatibility = project.javaVersion |
| project.targetCompatibility = project.javaVersion |
| |
| def defaultLintSuppressions = [ |
| 'options', |
| 'cast', |
| // https://bugs.openjdk.java.net/browse/JDK-8190452 |
| 'classfile', |
| 'deprecation', |
| 'fallthrough', |
| 'processing', |
| 'serial', |
| 'try', |
| 'unchecked', |
| 'varargs', |
| ] |
| |
| project.tasks.withType(JavaCompile).configureEach { |
| options.encoding = "UTF-8" |
| // As we want to add '-Xlint:-deprecation' we intentionally remove '-Xlint:deprecation' from compilerArgs here, |
| // as intellij is adding this, see https://youtrack.jetbrains.com/issue/IDEA-196615 |
| options.compilerArgs -= [ |
| "-Xlint:deprecation", |
| ] |
| options.compilerArgs += ([ |
| '-parameters', |
| '-Xlint:all', |
| '-Werror' |
| ] |
| + (defaultLintSuppressions + configuration.disableLintWarnings).collect { "-Xlint:-${it}" }) |
| } |
| |
| project.tasks.withType(Jar).configureEach { |
| preserveFileTimestamps(false) |
| } |
| |
| if (project.hasProperty("compileAndRunTestsWithJava11")) { |
| def java11Home = project.findProperty("java11Home") |
| project.tasks.compileTestJava { |
| options.fork = true |
| options.forkOptions.javaHome = java11Home as File |
| options.compilerArgs += ['-Xlint:-path'] |
| options.compilerArgs.addAll(['--release', '11']) |
| } |
| project.tasks.withType(Test) { |
| useJUnit() |
| executable = "${java11Home}/bin/java" |
| } |
| } |
| |
| // Configure the default test tasks set of tests executed |
| // to match the equivalent set that is executed by the maven-surefire-plugin. |
| // See http://maven.apache.org/components/surefire/maven-surefire-plugin/test-mojo.html |
| project.test { |
| include "**/Test*.class" |
| include "**/*Test.class" |
| include "**/*Tests.class" |
| include "**/*TestCase.class" |
| // fixes issues with test filtering on multi-module project |
| // see https://discuss.gradle.org/t/multi-module-build-fails-with-tests-filter/25835 |
| filter { setFailOnNoMatchingTests(false) } |
| } |
| |
| project.tasks.withType(Test).configureEach { |
| // Configure all test tasks to use JUnit |
| useJUnit {} |
| // default maxHeapSize on gradle 5 is 512m, lets increase to handle more demanding tests |
| maxHeapSize = '2g' |
| } |
| |
| List<String> skipDefRegexes = [] |
| skipDefRegexes << "AutoValue_.*" |
| skipDefRegexes << "AutoOneOf_.*" |
| skipDefRegexes << ".*\\.jmh_generated\\..*" |
| skipDefRegexes += configuration.generatedClassPatterns |
| skipDefRegexes += configuration.classesTriggerCheckerBugs.keySet() |
| String skipDefCombinedRegex = skipDefRegexes.collect({ regex -> "(${regex})"}).join("|") |
| |
| // SLF4J logger handles null log message parameters |
| String skipUsesRegex = "^org\\.slf4j\\.Logger.*" |
| |
| project.apply plugin: 'org.checkerframework' |
| project.checkerFramework { |
| checkers = [ |
| 'org.checkerframework.checker.nullness.NullnessChecker' |
| ] |
| |
| if (parseBooleanProperty(project, 'enableCheckerFramework') || project.jenkins.isCIBuild) { |
| skipCheckerFramework = false |
| } else { |
| skipCheckerFramework = true |
| } |
| |
| // Always exclude checkerframework on tests. It's slow, and it often |
| // raises erroneous error because we don't have checker annotations for |
| // test libraries like junit and hamcrest. See BEAM-11436. |
| // Consider re-enabling if we can get annotations for the test libraries |
| // we use. |
| excludeTests = true |
| |
| extraJavacArgs = [ |
| "-AskipDefs=${skipDefCombinedRegex}", |
| "-AskipUses=${skipUsesRegex}", |
| "-AsuppressWarnings=annotation.not.completed", |
| ] |
| |
| project.dependencies { |
| checkerFramework("org.checkerframework:checker:$checkerframework_version") |
| } |
| project.configurations.all { |
| it.exclude(group:"org.checkerframework", module:"jdk8") |
| } |
| } |
| |
| def jacocoExcludes = [ |
| '**/org/apache/beam/gradle/**', |
| '**/org/apache/beam/model/**', |
| '**/org/apache/beam/runners/dataflow/worker/windmill/**', |
| '**/AutoValue_*' |
| ] |
| |
| project.test { |
| jacoco { |
| excludes = jacocoExcludes |
| } |
| } |
| |
| project.jacocoTestReport { |
| doFirst { |
| getClassDirectories().setFrom(project.files( |
| project.fileTree( |
| dir: "${project.rootDir}", |
| exclude: jacocoExcludes |
| ) |
| ) |
| ) |
| } |
| } |
| |
| |
| |
| if (configuration.shadowClosure) { |
| // Ensure that tests are packaged and part of the artifact set. |
| project.task('packageTests', type: Jar) { |
| classifier = 'tests-unshaded' |
| from project.sourceSets.test.output |
| } |
| project.artifacts.archives project.packageTests |
| } |
| |
| // Note that these plugins specifically use the compileOnly and testCompileOnly |
| // configurations because they are never required to be shaded or become a |
| // dependency of the output. |
| def compileOnlyAnnotationDeps = [ |
| "com.google.auto.service:auto-service-annotations:$autoservice_version", |
| "com.google.auto.value:auto-value-annotations:$autovalue_version", |
| "com.google.code.findbugs:jsr305:$jsr305_version", |
| "com.google.j2objc:j2objc-annotations:1.3", |
| // These dependencies are needed to avoid error-prone warnings on package-info.java files, |
| // also to include the annotations to suppress warnings. |
| // |
| // spotbugs-annotations artifact is licensed under LGPL and cannot be included in the |
| // Apache Beam distribution, but may be relied on during build. |
| // See: https://www.apache.org/legal/resolved.html#prohibited |
| // Special case for jsr305 (a transitive dependency of spotbugs-annotations): |
| // sdks/java/core's FieldValueTypeInformation needs javax.annotations.Nullable at runtime. |
| // Therefore, the java core module declares jsr305 dependency (BSD license) as "compile". |
| // https://github.com/findbugsproject/findbugs/blob/master/findbugs/licenses/LICENSE-jsr305.txt |
| "com.github.spotbugs:spotbugs-annotations:$spotbugs_version", |
| "net.jcip:jcip-annotations:1.0", |
| // This explicitly adds javax.annotation.Generated (SOURCE retention) |
| // as a compile time dependency since Java 9+ no longer includes common |
| // EE annotations: http://bugs.openjdk.java.net/browse/JDK-8152842. This |
| // is required for grpc: http://github.com/grpc/grpc-java/issues/5343. |
| // |
| // javax.annotation is licensed under GPL 2.0 with Classpath Exception |
| // and must not be included in the Apache Beam distribution, but may be |
| // relied on during build. |
| // See exception in: https://www.apache.org/legal/resolved.html#prohibited |
| // License: https://github.com/javaee/javax.annotation/blob/1.3.2/LICENSE |
| "javax.annotation:javax.annotation-api:1.3.2", |
| ] |
| |
| project.dependencies { |
| compileOnlyAnnotationDeps.each { dep -> |
| compileOnly dep |
| testCompileOnly dep |
| annotationProcessor dep |
| testAnnotationProcessor dep |
| } |
| |
| // Add common annotation processors to all Java projects |
| def annotationProcessorDeps = [ |
| "com.google.auto.value:auto-value:$autovalue_version", |
| "com.google.auto.service:auto-service:$autoservice_version", |
| ] |
| |
| annotationProcessorDeps.each { dep -> |
| annotationProcessor dep |
| testAnnotationProcessor dep |
| } |
| |
| // This contains many improved annotations beyond javax.annotations for enhanced static checking |
| // of the codebase. It is runtime so users can also take advantage of them. The annotations themselves |
| // are MIT licensed (checkerframework is GPL and cannot be distributed) |
| implementation "org.checkerframework:checker-qual:$checkerframework_version" |
| } |
| |
| // Defines Targets for sonarqube analysis reporting. |
| project.apply plugin: "org.sonarqube" |
| |
| // Configures a checkstyle plugin enforcing a set of rules and also allows for a set of |
| // suppressions. |
| project.apply plugin: 'checkstyle' |
| project.tasks.withType(Checkstyle) { |
| configFile = project.project(":").file("sdks/java/build-tools/src/main/resources/beam/checkstyle.xml") |
| configProperties = ["checkstyle.suppressions.file": project.project(":").file("sdks/java/build-tools/src/main/resources/beam/suppressions.xml")] |
| showViolations = true |
| maxErrors = 0 |
| } |
| project.checkstyle { toolVersion = "8.23" } |
| // CheckStyle can be removed from the 'check' task by passing -PdisableCheckStyle=true on the Gradle |
| // command-line. This is useful for pre-commit which runs checkStyle separately. |
| def disableCheckStyle = project.hasProperty('disableCheckStyle') && |
| project.disableCheckStyle == 'true' |
| project.checkstyleMain.enabled = !disableCheckStyle |
| project.checkstyleTest.enabled = !disableCheckStyle |
| |
| // Configures javadoc plugin and ensure check runs javadoc. |
| project.tasks.withType(Javadoc) { |
| options.encoding = 'UTF-8' |
| options.addBooleanOption('Xdoclint:-missing', true) |
| } |
| project.check.dependsOn project.javadoc |
| |
| // Enables a plugin which can apply code formatting to source. |
| project.apply plugin: "com.diffplug.spotless" |
| // scan CVE |
| project.apply plugin: "net.ossindex.audit" |
| project.audit { rateLimitAsError = false } |
| // Spotless can be removed from the 'check' task by passing -PdisableSpotlessCheck=true on the Gradle |
| // command-line. This is useful for pre-commit which runs spotless separately. |
| def disableSpotlessCheck = project.hasProperty('disableSpotlessCheck') && |
| project.disableSpotlessCheck == 'true' |
| project.spotless { |
| enforceCheck !disableSpotlessCheck |
| java { |
| licenseHeader javaLicenseHeader |
| googleJavaFormat('1.7') |
| target project.fileTree(project.projectDir) { include 'src/*/java/**/*.java' } |
| } |
| } |
| |
| // Enables a plugin which performs code analysis for common bugs. |
| // This plugin is configured to only analyze the "main" source set. |
| if (configuration.enableSpotbugs) { |
| project.tasks.whenTaskAdded {task -> |
| if(task.name.contains("spotbugsTest")) { |
| task.enabled = false |
| } |
| } |
| project.apply plugin: 'com.github.spotbugs' |
| project.dependencies { |
| spotbugs "com.github.spotbugs:spotbugs:$spotbugs_version" |
| spotbugs "com.google.auto.value:auto-value:$autovalue_version" |
| compileOnlyAnnotationDeps.each { dep -> spotbugs dep } |
| } |
| project.spotbugs { |
| excludeFilter = project.rootProject.file('sdks/java/build-tools/src/main/resources/beam/spotbugs-filter.xml') |
| jvmArgs = ['-Xmx12g'] |
| } |
| project.tasks.withType(com.github.spotbugs.snom.SpotBugsTask) { |
| reports { |
| html.enabled = !project.jenkins.isCIBuild |
| xml.enabled = project.jenkins.isCIBuild |
| } |
| } |
| } |
| |
| // Disregard unused but declared (test) compile only dependencies used |
| // for common annotation classes used during compilation such as annotation |
| // processing or post validation such as spotbugs. |
| project.dependencies { |
| compileOnlyAnnotationDeps.each { dep -> |
| permitUnusedDeclared dep |
| permitTestUnusedDeclared dep |
| } |
| permitUnusedDeclared "org.checkerframework:checker-qual:$checkerframework_version" |
| } |
| |
| if (configuration.enableStrictDependencies) { |
| project.tasks.analyzeClassesDependencies.enabled = true |
| project.tasks.analyzeDependencies.enabled = true |
| project.tasks.analyzeTestClassesDependencies.enabled = false |
| } else { |
| project.tasks.analyzeClassesDependencies.enabled = false |
| project.tasks.analyzeTestClassesDependencies.enabled = false |
| project.tasks.analyzeDependencies.enabled = false |
| } |
| |
| // Enable errorprone static analysis |
| project.apply plugin: 'net.ltgt.errorprone' |
| |
| project.dependencies { |
| errorprone("com.google.errorprone:error_prone_core:$errorprone_version") |
| errorprone("jp.skypencil.errorprone.slf4j:errorprone-slf4j:0.1.2") |
| // At least JDk 9 compiler is required, however JDK 8 still can be used but with additional errorproneJavac |
| // configuration. For more details please see https://github.com/tbroyer/gradle-errorprone-plugin#jdk-8-support |
| errorproneJavac("com.google.errorprone:javac:9+181-r4173-1") |
| } |
| |
| project.configurations.errorprone { resolutionStrategy.force "com.google.errorprone:error_prone_core:$errorprone_version" } |
| |
| project.tasks.withType(JavaCompile) { |
| options.errorprone.disableWarningsInGeneratedCode = true |
| options.errorprone.excludedPaths = '(.*/)?(build/generated-src|build/generated.*avro-java|build/generated)/.*' |
| |
| // TODO(https://github.com/apache/beam/issues/20955): Enable errorprone checks |
| options.errorprone.errorproneArgs.add("-Xep:AutoValueImmutableFields:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:AutoValueSubclassLeaked:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:BadImport:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:BadInstanceof:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:BigDecimalEquals:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:ComparableType:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:DoNotMockAutoValue:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:EmptyBlockTag:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:EmptyCatch:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:EqualsGetClass:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:EqualsUnsafeCast:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:EscapedEntity:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:ExtendsAutoValue:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:InlineFormatString:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:InlineMeSuggester:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:InvalidBlockTag:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:InvalidInlineTag:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:InvalidLink:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:InvalidParam:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:InvalidThrows:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:JavaTimeDefaultTimeZone:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:JavaUtilDate:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:JodaConstructors:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:MalformedInlineTag:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:MissingSummary:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:MixedMutabilityReturnType:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:PreferJavaTimeOverload:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:MutablePublicArray:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:NonCanonicalType:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:ProtectedMembersInFinalClass:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:Slf4jFormatShouldBeConst:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:Slf4jSignOnlyFormat:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:StaticAssignmentInConstructor:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:ThreadPriorityCheck:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:TimeUnitConversionChecker:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:UndefinedEquals:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:UnescapedEntity:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:UnnecessaryLambda:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:UnnecessaryMethodReference:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:UnnecessaryParentheses:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:UnrecognisedJavadocTag:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:UnsafeReflectiveConstructionCast:OFF") |
| options.errorprone.errorproneArgs.add("-Xep:UseCorrectAssertInTests:OFF") |
| |
| // Sometimes a static logger is preferred, which is the convention |
| // currently used in beam. See docs: |
| // https://github.com/KengoTODA/findbugs-slf4j#slf4j_logger_should_be_non_static |
| options.errorprone.errorproneArgs.add("-Xep:Slf4jLoggerShouldBeNonStatic:OFF") |
| } |
| |
| if (project.hasProperty("compileAndRunTestsWithJava17")) { |
| def java17Home = project.findProperty("java17Home") |
| project.tasks.compileTestJava { |
| options.fork = true |
| options.forkOptions.javaHome = java17Home as File |
| options.compilerArgs += ['-Xlint:-path'] |
| options.compilerArgs.addAll(['--release', '17']) |
| // Error prone requires some packages to be exported/opened for Java 17 |
| // Disabling checks since this property is only used for Jenkins tests |
| // https://github.com/tbroyer/gradle-errorprone-plugin#jdk-16-support |
| options.errorprone.errorproneArgs.add("-XepDisableAllChecks") |
| options.forkOptions.jvmArgs += [ |
| "-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", |
| "-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", |
| "-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", |
| "-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", |
| "-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", |
| "-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", |
| "-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", |
| "-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", |
| "-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", |
| "-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" |
| ] |
| } |
| project.tasks.withType(Test) { |
| useJUnit() |
| executable = "${java17Home}/bin/java" |
| } |
| } |
| if (configuration.shadowClosure) { |
| // Enables a plugin which can perform shading of classes. See the general comments |
| // above about dependency management for Java projects and how the shadow plugin |
| // is expected to be used for the different Gradle configurations. |
| // |
| // TODO: Enforce all relocations are always performed to: |
| // getJavaRelocatedPath(package_suffix) where package_suffix is something like "com.google.commmon" |
| project.apply plugin: 'com.github.johnrengelman.shadow' |
| |
| // Create a new configuration 'shadowTest' like 'shadow' for the test scope |
| project.configurations { |
| shadow { description = "Dependencies for shaded source set 'main'" } |
| implementation.extendsFrom shadow |
| shadowTest { |
| description = "Dependencies for shaded source set 'test'" |
| extendsFrom shadow |
| } |
| testImplementation.extendsFrom shadowTest |
| } |
| |
| // Ensure build task depends on shadowJar (required for shadow plugin >=4.0.4) |
| project.tasks.named('build').configure { |
| dependsOn project.tasks.named('shadowJar') |
| } |
| } |
| |
| project.jar { |
| setAutomaticModuleNameHeader(configuration, project) |
| |
| zip64 true |
| into("META-INF/") { |
| from "${project.rootProject.projectDir}/LICENSE" |
| from "${project.rootProject.projectDir}/NOTICE" |
| } |
| } |
| |
| // Always configure the shadowJar classifier and merge service files. |
| if (configuration.shadowClosure) { |
| // Only set the classifer on the unshaded classes if we are shading. |
| project.jar { classifier = "unshaded" } |
| |
| project.shadowJar({ |
| classifier = null |
| mergeServiceFiles() |
| zip64 true |
| into("META-INF/") { |
| from "${project.rootProject.projectDir}/LICENSE" |
| from "${project.rootProject.projectDir}/NOTICE" |
| } |
| } << configuration.shadowClosure) |
| |
| // Always configure the shadowTestJar classifier and merge service files. |
| project.task('shadowTestJar', type: ShadowJar, { |
| group = "Shadow" |
| description = "Create a combined JAR of project and test dependencies" |
| classifier = "tests" |
| from project.sourceSets.test.output |
| configurations = [ |
| project.configurations.testRuntimeMigration |
| ] |
| zip64 true |
| exclude "META-INF/INDEX.LIST" |
| exclude "META-INF/*.SF" |
| exclude "META-INF/*.DSA" |
| exclude "META-INF/*.RSA" |
| } << configuration.shadowClosure) |
| |
| // Ensure that shaded jar and test-jar are part of their own configuration artifact sets |
| project.artifacts.shadow project.shadowJar |
| project.artifacts.shadowTest project.shadowTestJar |
| |
| if (configuration.testShadowJar) { |
| // Use a configuration and dependency set which represents the execution classpath using shaded artifacts for tests. |
| project.configurations { shadowTestRuntimeClasspath } |
| |
| project.dependencies { |
| shadowTestRuntimeClasspath it.project(path: project.path, configuration: "shadowTest") |
| shadowTestRuntimeClasspath it.project(path: project.path) |
| } |
| |
| project.test { classpath = project.configurations.shadowTestRuntimeClasspath } |
| } |
| |
| if (configuration.validateShadowJar) { |
| def validateShadedJarDoesntLeakNonProjectClasses = project.tasks.register('validateShadedJarDoesntLeakNonProjectClasses') { |
| dependsOn 'shadowJar' |
| ext.outFile = project.file("${project.reportsDir}/${name}.out") |
| inputs.files(project.configurations.shadow.artifacts.files) |
| .withPropertyName("shadowArtifactsFiles") |
| .withPathSensitivity(PathSensitivity.RELATIVE) |
| outputs.files outFile |
| doLast { |
| project.configurations.shadow.artifacts.files.each { |
| FileTree exposedClasses = project.zipTree(it).matching { |
| include "**/*.class" |
| // BEAM-5919: Exclude paths for Java 9 multi-release jars. |
| exclude "META-INF/versions/*/module-info.class" |
| configuration.shadowJarValidationExcludes.each { |
| exclude "$it" |
| exclude "META-INF/versions/*/$it" |
| } |
| } |
| outFile.text = exposedClasses.files |
| if (exposedClasses.files) { |
| throw new GradleException("$it exposed classes outside of ${configuration.shadowJarValidationExcludes}: ${exposedClasses.files}") |
| } |
| } |
| } |
| } |
| project.tasks.check.dependsOn validateShadedJarDoesntLeakNonProjectClasses |
| } |
| } else { |
| project.tasks.register("testJar", Jar) { |
| group = "Jar" |
| description = "Create a JAR of test classes" |
| classifier = "tests" |
| from project.sourceSets.test.output |
| zip64 true |
| exclude "META-INF/INDEX.LIST" |
| exclude "META-INF/*.SF" |
| exclude "META-INF/*.DSA" |
| exclude "META-INF/*.RSA" |
| } |
| project.artifacts.testRuntimeMigration project.testJar |
| } |
| |
| if (configuration.enableJmh) { |
| project.dependencies { |
| annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$jmh_version" |
| implementation "org.openjdk.jmh:jmh-core:$jmh_version" |
| } |
| |
| project.compileJava { |
| // Always exclude checkerframework on JMH generated code. It's slow, |
| // and it often raises erroneous error because we don't have checker |
| // annotations for generated code and test libraries. |
| // |
| // Consider re-enabling if we can get annotations for the generated |
| // code and test libraries we use. |
| checkerFramework { |
| skipCheckerFramework = true |
| } |
| } |
| |
| project.tasks.register("jmh", JavaExec) { |
| dependsOn project.classes |
| mainClass = "org.openjdk.jmh.Main" |
| classpath = project.sourceSets.main.runtimeClasspath |
| // For a list of arguments, see |
| // https://github.com/guozheng/jmh-tutorial/blob/master/README.md |
| // |
| // Enumerate available benchmarks and exit (uncomment below and disable other args) |
| // args '-l' |
| // |
| // Enable connecting a debugger by disabling forking (uncomment below and disable other args) |
| // Useful for debugging via an IDE such as Intellij |
| // args '-f=0' |
| // Specify -Pbenchmark=ProcessBundleBenchmark.testTinyBundle on the command |
| // line to enable running a single benchmark. |
| |
| // Enable Google Cloud Profiler and upload the benchmarks to GCP. |
| if (project.hasProperty("benchmark")) { |
| args project.getProperty("benchmark") |
| // Add JVM arguments allowing one to additionally use Google's Java Profiler |
| // Agent: (see https://cloud.google.com/profiler/docs/profiling-java#installing-profiler for instructions on how to install) |
| if (project.file("/opt/cprof/profiler_java_agent.so").exists()) { |
| def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing' |
| def userName = System.getProperty("user.name").toLowerCase().replaceAll(" ", "_") |
| jvmArgs '-agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=' + userName + "_" + project.getProperty("benchmark").toLowerCase() + '_' + System.currentTimeMillis() + ',-cprof_project_id=' + gcpProject + ',-cprof_zone_name=us-central1-a' |
| } |
| } else { |
| // We filter for only Apache Beam benchmarks to ensure that we aren't |
| // running benchmarks that may have been packaged from another library |
| // that ends up on the runtime classpath. |
| args 'org.apache.beam' |
| } |
| args '-foe=true' |
| } |
| |
| // Single shot of JMH benchmarks ensures that they can execute. |
| // |
| // Note that these tests will fail on JVMs that JMH doesn't support. |
| def jmhTest = project.tasks.register("jmhTest", JavaExec) { |
| dependsOn project.classes |
| mainClass = "org.openjdk.jmh.Main" |
| classpath = project.sourceSets.main.runtimeClasspath |
| |
| // We filter for only Apache Beam benchmarks to ensure that we aren't |
| // running benchmarks that may have been packaged from another library |
| // that ends up on the runtime classpath. |
| args 'org.apache.beam' |
| args '-bm=ss' |
| args '-i=1' |
| args '-f=0' |
| args '-wf=0' |
| args '-foe=true' |
| } |
| project.check.dependsOn jmhTest |
| } |
| |
| project.ext.includeInJavaBom = configuration.publish |
| project.ext.exportJavadoc = configuration.exportJavadoc |
| |
| if ((isRelease(project) || project.hasProperty('publishing')) && |
| configuration.publish) { |
| project.apply plugin: "maven-publish" |
| |
| // plugin to support repository authentication via ~/.m2/settings.xml |
| // https://github.com/mark-vieira/gradle-maven-settings-plugin/ |
| project.apply plugin: 'net.linguica.maven-settings' |
| |
| // Create a task which emulates the maven-archiver plugin in generating a |
| // pom.properties file. |
| def pomPropertiesFile = "${project.buildDir}/publications/mavenJava/pom.properties" |
| project.task('generatePomPropertiesFileForMavenJavaPublication') { |
| outputs.file "${pomPropertiesFile}" |
| doLast { |
| new File("${pomPropertiesFile}").text = |
| """version=${project.version} |
| groupId=${project.mavenGroupId} |
| artifactId=${project.archivesBaseName}""" |
| } |
| } |
| |
| // Have the main artifact jar include both the generate pom.xml and its properties file |
| // emulating the behavior of the maven-archiver plugin. |
| project.(configuration.shadowClosure ? 'shadowJar' : 'jar') { |
| def pomFile = "${project.buildDir}/publications/mavenJava/pom-default.xml" |
| |
| // Validate that the artifacts exist before copying them into the jar. |
| doFirst { |
| if (!project.file("${pomFile}").exists()) { |
| throw new GradleException("Expected ${pomFile} to have been generated by the 'generatePomFileForMavenJavaPublication' task.") |
| } |
| if (!project.file("${pomPropertiesFile}").exists()) { |
| throw new GradleException("Expected ${pomPropertiesFile} to have been generated by the 'generatePomPropertiesFileForMavenJavaPublication' task.") |
| } |
| } |
| |
| dependsOn 'generatePomFileForMavenJavaPublication' |
| into("META-INF/maven/${project.mavenGroupId}/${project.archivesBaseName}") { |
| from "${pomFile}" |
| rename('.*', 'pom.xml') |
| } |
| |
| dependsOn project.generatePomPropertiesFileForMavenJavaPublication |
| into("META-INF/maven/${project.mavenGroupId}/${project.archivesBaseName}") { from "${pomPropertiesFile}" } |
| } |
| |
| // Only build artifacts for archives if we are publishing |
| if (configuration.shadowClosure) { |
| project.artifacts.archives project.shadowJar |
| project.artifacts.archives project.shadowTestJar |
| } else { |
| project.artifacts.archives project.jar |
| project.artifacts.archives project.testJar |
| } |
| |
| project.task('sourcesJar', type: Jar) { |
| from project.sourceSets.main.allSource |
| classifier = 'sources' |
| } |
| project.artifacts.archives project.sourcesJar |
| |
| project.task('testSourcesJar', type: Jar) { |
| from project.sourceSets.test.allSource |
| classifier = 'test-sources' |
| } |
| project.artifacts.archives project.testSourcesJar |
| |
| project.task('javadocJar', type: Jar, dependsOn: project.javadoc) { |
| classifier = 'javadoc' |
| from project.javadoc.destinationDir |
| } |
| project.artifacts.archives project.javadocJar |
| |
| project.publishing { |
| repositories project.ext.repositories |
| |
| publications { |
| mavenJava(MavenPublication) { |
| if (configuration.shadowClosure) { |
| artifact project.shadowJar |
| artifact project.shadowTestJar |
| } else { |
| artifact project.jar |
| artifact project.testJar |
| } |
| artifact project.sourcesJar |
| artifact project.testSourcesJar |
| artifact project.javadocJar |
| |
| artifactId = project.archivesBaseName |
| groupId = project.mavenGroupId |
| |
| pom { |
| name = project.description |
| if (project.hasProperty("summary")) { |
| description = project.summary |
| } |
| url = "http://beam.apache.org" |
| inceptionYear = "2016" |
| licenses { |
| license { |
| name = "Apache License, Version 2.0" |
| url = "http://www.apache.org/licenses/LICENSE-2.0.txt" |
| distribution = "repo" |
| } |
| } |
| scm { |
| connection = "scm:git:https://gitbox.apache.org/repos/asf/beam.git" |
| developerConnection = "scm:git:https://gitbox.apache.org/repos/asf/beam.git" |
| url = "https://gitbox.apache.org/repos/asf?p=beam.git;a=summary" |
| } |
| issueManagement { |
| system = "github" |
| url = "https://github.com/apache/beam/issues" |
| } |
| mailingLists { |
| mailingList { |
| name = "Beam Dev" |
| subscribe = "dev-subscribe@beam.apache.org" |
| unsubscribe = "dev-unsubscribe@beam.apache.org" |
| post = "dev@beam.apache.org" |
| archive = "http://www.mail-archive.com/dev%beam.apache.org" |
| } |
| mailingList { |
| name = "Beam User" |
| subscribe = "user-subscribe@beam.apache.org" |
| unsubscribe = "user-unsubscribe@beam.apache.org" |
| post = "user@beam.apache.org" |
| archive = "http://www.mail-archive.com/user%beam.apache.org" |
| } |
| mailingList { |
| name = "Beam Commits" |
| subscribe = "commits-subscribe@beam.apache.org" |
| unsubscribe = "commits-unsubscribe@beam.apache.org" |
| post = "commits@beam.apache.org" |
| archive = "http://www.mail-archive.com/commits%beam.apache.org" |
| } |
| } |
| developers { |
| developer { |
| name = "The Apache Beam Team" |
| email = "dev@beam.apache.org" |
| url = "http://beam.apache.org" |
| organization = "Apache Software Foundation" |
| organizationUrl = "http://www.apache.org" |
| } |
| } |
| } |
| |
| pom.withXml { |
| def root = asNode() |
| |
| // Add "repositories" section if it was configured |
| if (configuration.mavenRepositories && configuration.mavenRepositories.size() > 0) { |
| def repositoriesNode = root.appendNode('repositories') |
| configuration.mavenRepositories.each { |
| def repositoryNode = repositoriesNode.appendNode('repository') |
| repositoryNode.appendNode('id', it.get('id')) |
| repositoryNode.appendNode('url', it.get('url')) |
| } |
| } |
| |
| def dependenciesNode = root.appendNode('dependencies') |
| |
| // BOMs, declared with 'platform' or 'enforced-platform', appear in <dependencyManagement> section |
| def boms = [] |
| |
| def generateDependenciesFromConfiguration = { param -> |
| project.configurations."${param.configuration}".allDependencies.each { |
| String category = it.getAttributes().getAttribute(Category.CATEGORY_ATTRIBUTE) |
| if (Category.ENFORCED_PLATFORM == category || Category.REGULAR_PLATFORM == category) { |
| boms.add(it) |
| return |
| } |
| |
| def dependencyNode = dependenciesNode.appendNode('dependency') |
| def appendClassifier = { dep -> |
| dep.artifacts.each { art -> |
| if (art.hasProperty('classifier')) { |
| dependencyNode.appendNode('classifier', art.classifier) |
| } |
| } |
| } |
| |
| if (it instanceof ProjectDependency) { |
| dependencyNode.appendNode('groupId', it.getDependencyProject().mavenGroupId) |
| dependencyNode.appendNode('artifactId', it.getDependencyProject().archivesBaseName) |
| dependencyNode.appendNode('version', it.version) |
| dependencyNode.appendNode('scope', param.scope) |
| appendClassifier(it) |
| } else { |
| dependencyNode.appendNode('groupId', it.group) |
| dependencyNode.appendNode('artifactId', it.name) |
| if (it.version != null) { // bom-managed artifacts do not have their versions |
| dependencyNode.appendNode('version', it.version) |
| } |
| dependencyNode.appendNode('scope', param.scope) |
| appendClassifier(it) |
| } |
| |
| // Start with any exclusions that were added via configuration exclude rules. |
| // Then add all the exclusions that are specific to the dependency (if any |
| // were declared). Finally build the node that represents all exclusions. |
| def exclusions = [] |
| exclusions += project.configurations."${param.configuration}".excludeRules |
| if (it.hasProperty('excludeRules')) { |
| exclusions += it.excludeRules |
| } |
| if (!exclusions.empty) { |
| def exclusionsNode = dependencyNode.appendNode('exclusions') |
| exclusions.each { exclude -> |
| def exclusionNode = exclusionsNode.appendNode('exclusion') |
| exclusionNode.appendNode('groupId', exclude.group) |
| exclusionNode.appendNode('artifactId', exclude.module) |
| } |
| } |
| } |
| } |
| |
| // TODO: Should we use the runtime scope instead of the compile scope |
| // which forces all our consumers to declare what they consume? |
| generateDependenciesFromConfiguration( |
| configuration: (configuration.shadowClosure ? 'shadow' : 'implementation'), scope: 'compile') |
| generateDependenciesFromConfiguration(configuration: 'provided', scope: 'provided') |
| |
| if (!boms.isEmpty()) { |
| def dependencyManagementNode = root.appendNode('dependencyManagement') |
| def dependencyManagementDependencies = dependencyManagementNode.appendNode('dependencies') |
| |
| // Resolve linkage error with guava jre vs android caused by Google Cloud libraries BOM |
| // https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM#guava-versions--jre-or--android |
| def guavaDependencyNode = dependencyManagementDependencies.appendNode('dependency') |
| guavaDependencyNode.appendNode('groupId', 'com.google.guava') |
| guavaDependencyNode.appendNode('artifactId', 'guava') |
| guavaDependencyNode.appendNode('version', "$guava_version") |
| |
| boms.each { |
| def dependencyNode = dependencyManagementDependencies.appendNode('dependency') |
| dependencyNode.appendNode('groupId', it.group) |
| dependencyNode.appendNode('artifactId', it.name) |
| dependencyNode.appendNode('version', it.version) |
| dependencyNode.appendNode('type', 'pom') |
| dependencyNode.appendNode('scope', 'import') |
| } |
| } |
| // NB: This must come after asNode() logic, as it seems asNode() |
| // removes XML comments. |
| // TODO: Load this from file? |
| def elem = asElement() |
| def hdr = elem.getOwnerDocument().createComment( |
| ''' |
| Licensed to the Apache Software Foundation (ASF) under one or more |
| contributor license agreements. See the NOTICE file distributed with |
| this work for additional information regarding copyright ownership. |
| The ASF licenses this file to You under the Apache License, Version 2.0 |
| (the "License"); you may not use this file except in compliance with |
| the License. You may obtain a copy of the License at |
| http://www.apache.org/licenses/LICENSE-2.0 |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| ''') |
| elem.insertBefore(hdr, elem.getFirstChild()) |
| } |
| } |
| } |
| } |
| // Only sign artifacts if we are performing a release |
| if (isRelease(project) && !project.hasProperty('noSigning')) { |
| project.apply plugin: "signing" |
| project.signing { |
| useGpgCmd() |
| sign project.publishing.publications |
| } |
| } |
| } |
| |
| // Ban these dependencies from all configurations |
| project.configurations.all { |
| // guava-jdk5 brings in classes which conflict with guava |
| exclude group: "com.google.guava", module: "guava-jdk5" |
| // Ban the usage of the JDK tools as a library as this is system dependent |
| exclude group: "jdk.tools", module: "jdk.tools" |
| // protobuf-lite duplicates classes which conflict with protobuf-java |
| exclude group: "com.google.protobuf", module: "protobuf-lite" |
| // Exclude these test dependencies because they bundle other common |
| // test libraries classes causing version conflicts. Users should rely |
| // on using the yyy-core package instead of the yyy-all package. |
| exclude group: "org.hamcrest", module: "hamcrest-all" |
| } |
| |
| // Force usage of the libraries defined within our common set found in the root |
| // build.gradle instead of using Gradles default dependency resolution mechanism |
| // which chooses the latest version available. |
| // |
| // TODO: Figure out whether we should force all dependency conflict resolution |
| // to occur in the "shadow" and "shadowTest" configurations. |
| project.configurations.all { config -> |
| // When running beam_Dependency_Check, resolutionStrategy should not be used; otherwise |
| // gradle-versions-plugin does not report the latest versions of the dependencies. |
| def startTasks = project.gradle.startParameter.taskNames |
| def inDependencyUpdates = 'dependencyUpdates' in startTasks || 'runBeamDependencyCheck' in startTasks |
| |
| // The "errorprone" configuration controls the classpath used by errorprone static analysis, which |
| // has different dependencies than our project. |
| if (config.getName() != "errorprone" && !inDependencyUpdates) { |
| config.resolutionStrategy { |
| // Filtering versionless coordinates that depend on BOM. Beam project needs to set the |
| // versions for only handful libraries when building the project (BEAM-9542). |
| def librariesWithVersion = project.library.java.values().findAll { it.split(':').size() > 2 } |
| force librariesWithVersion |
| |
| // hamcrest-core and hamcrest-library have been superseded by hamcrest. |
| // We force their versions here to ensure that any resolved version provides |
| // the same classes as hamcrest. |
| force "org.hamcrest:hamcrest-core:$hamcrest_version" |
| force "org.hamcrest:hamcrest-library:$hamcrest_version" |
| } |
| } |
| } |
| } |
| def cleanUpTask = project.tasks.register('cleanUp') { |
| dependsOn ':runners:google-cloud-dataflow-java:cleanUpDockerJavaImages' |
| } |
| |
| // When applied in a module's build.gradle file, this closure provides task for running |
| // IO integration tests. |
| project.ext.enableJavaPerformanceTesting = { |
| |
| // Use the implicit it parameter of the closure to handle zero argument or one argument map calls. |
| // See: http://groovy-lang.org/closures.html#implicit-it |
| JavaPerformanceTestConfiguration configuration = it ? it as JavaPerformanceTestConfiguration : new JavaPerformanceTestConfiguration() |
| |
| // Task for running integration tests |
| def itTask = project.task('integrationTest', type: Test) { |
| |
| // Disable Gradle cache (it should not be used because the IT's won't run). |
| outputs.upToDateWhen { false } |
| |
| include "**/*IT.class" |
| |
| def pipelineOptionsString = configuration.integrationTestPipelineOptions |
| def pipelineOptionsStringFormatted |
| def allOptionsList |
| |
| if(pipelineOptionsString) { |
| allOptionsList = (new JsonSlurper()).parseText(pipelineOptionsString) |
| } |
| |
| if (pipelineOptionsString && configuration.runner?.equalsIgnoreCase('dataflow')) { |
| if (pipelineOptionsString.contains('use_runner_v2')) { |
| dependsOn ':runners:google-cloud-dataflow-java:buildAndPushDockerJavaContainer' |
| } |
| } |
| |
| // We construct the pipeline options during task execution time in order to get dockerJavaImageName. |
| doFirst { |
| if (pipelineOptionsString && configuration.runner?.equalsIgnoreCase('dataflow')) { |
| def dataflowRegion = project.findProperty('dataflowRegion') ?: 'us-central1' |
| if (pipelineOptionsString.contains('use_runner_v2')) { |
| def dockerJavaImageName = project.project(':runners:google-cloud-dataflow-java').ext.dockerJavaImageName |
| allOptionsList.addAll([ |
| "--sdkContainerImage=${dockerJavaImageName}", |
| "--region=${dataflowRegion}" |
| ]) |
| } else { |
| project.evaluationDependsOn(":runners:google-cloud-dataflow-java:worker:legacy-worker") |
| def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: |
| project.project(":runners:google-cloud-dataflow-java:worker:legacy-worker").shadowJar.archivePath |
| allOptionsList.addAll([ |
| // Keep as legacy flag to ensure via test this flag works for |
| // legacy pipeline. |
| '--workerHarnessContainerImage=', |
| "--dataflowWorkerJar=${dataflowWorkerJar}", |
| "--region=${dataflowRegion}" |
| ]) |
| } |
| } |
| |
| // Windows handles quotation marks differently |
| if (pipelineOptionsString && System.properties['os.name'].toLowerCase().contains('windows')) { |
| def allOptionsListFormatted = allOptionsList.collect { "\"$it\"" } |
| pipelineOptionsStringFormatted = JsonOutput.toJson(allOptionsListFormatted) |
| } else if (pipelineOptionsString) { |
| pipelineOptionsStringFormatted = JsonOutput.toJson(allOptionsList) |
| } |
| |
| systemProperties.beamTestPipelineOptions = pipelineOptionsStringFormatted ?: pipelineOptionsString |
| } |
| } |
| project.afterEvaluate { |
| // Ensure all tasks which use published docker images run before they are cleaned up |
| project.tasks.each { t -> |
| if (t.dependsOn.contains(":runners:google-cloud-dataflow-java:buildAndPushDockerJavaContainer")) { |
| t.finalizedBy cleanUpTask |
| } |
| } |
| } |
| } |
| |
| // When applied in a module's build.gradle file, this closure adds task providing |
| // additional dependencies that might be needed while running integration tests. |
| project.ext.provideIntegrationTestingDependencies = { |
| |
| // Use the implicit it parameter of the closure to handle zero argument or one argument map calls. |
| // See: http://groovy-lang.org/closures.html#implicit-it |
| JavaPerformanceTestConfiguration configuration = it ? it as JavaPerformanceTestConfiguration : new JavaPerformanceTestConfiguration() |
| |
| project.dependencies { |
| def runner = configuration.runner |
| def filesystem = configuration.filesystem |
| |
| /* include dependencies required by runners */ |
| //if (runner?.contains('dataflow')) { |
| if (runner?.equalsIgnoreCase('dataflow')) { |
| testRuntimeOnly it.project(path: ":runners:google-cloud-dataflow-java", configuration: "testRuntimeMigration") |
| testRuntimeOnly it.project(path: ":runners:google-cloud-dataflow-java:worker:legacy-worker", configuration: 'shadow') |
| } |
| |
| if (runner?.equalsIgnoreCase('direct')) { |
| testRuntimeOnly it.project(path: ":runners:direct-java", configuration: 'shadowTest') |
| } |
| |
| if (runner?.equalsIgnoreCase('flink')) { |
| testRuntimeOnly it.project(path: ":runners:flink:${project.ext.latestFlinkVersion}", configuration: "testRuntimeMigration") |
| } |
| |
| if (runner?.equalsIgnoreCase('spark')) { |
| testRuntimeOnly it.project(path: ":runners:spark:2", configuration: "testRuntimeMigration") |
| testRuntimeOnly project.library.java.spark_core |
| testRuntimeOnly project.library.java.spark_streaming |
| |
| // Testing the Spark runner causes a StackOverflowError if slf4j-jdk14 is on the classpath |
| project.configurations.testRuntimeClasspath { |
| exclude group: "org.slf4j", module: "slf4j-jdk14" |
| } |
| } |
| |
| /* include dependencies required by filesystems */ |
| if (filesystem?.equalsIgnoreCase('hdfs')) { |
| testRuntimeOnly it.project(path: ":sdks:java:io:hadoop-file-system", configuration: "testRuntimeMigration") |
| testRuntimeOnly project.library.java.hadoop_client |
| } |
| |
| /* include dependencies required by AWS S3 */ |
| if (filesystem?.equalsIgnoreCase('s3')) { |
| testRuntimeOnly it.project(path: ":sdks:java:io:amazon-web-services", configuration: "testRuntimeMigration") |
| } |
| } |
| project.task('packageIntegrationTests', type: Jar) |
| } |
| |
| /** ***********************************************************************************************/ |
| |
| project.ext.applyGoNature = { |
| // Define common lifecycle tasks and artifact types |
| project.apply plugin: 'base' |
| |
| // For some reason base doesn't define a test task so we define it below and make |
| // check depend on it. This makes the Go project similar to the task layout like |
| // Java projects, see https://docs.gradle.org/4.2.1/userguide/img/javaPluginTasks.png |
| if (project.tasks.findByName('test') == null) { |
| project.task('test') {} |
| } |
| project.check.dependsOn project.test |
| |
| def goRootDir = "${project.rootDir}/sdks/go" |
| |
| // This sets the whole project Go version. |
| project.ext.goVersion = "go1.18.1" |
| |
| // Minor TODO: Figure out if we can pull out the GOCMD env variable after goPrepare script |
| // completion, and avoid this GOBIN substitution. |
| project.ext.goCmd = "${goRootDir}/run_with_go_version.sh --gocmd GOBIN/${project.ext.goVersion}" |
| |
| def goPrepare = project.tasks.register("goPrepare") { |
| description "Prepare ${project.ext.goVersion} for builds and tests." |
| project.exec { |
| executable 'sh' |
| args '-c', "${goRootDir}/prepare_go_version.sh --version ${project.ext.goVersion}" |
| } |
| } |
| |
| def goBuild = project.tasks.register("goBuild") { |
| dependsOn goPrepare |
| ext.goTargets = './...' |
| ext.outputLocation = './build/bin/${GOOS}_${GOARCH}/' |
| doLast { |
| project.exec { |
| // Set these so the substitutions work. |
| // May cause issues for the folks running gradle commands on other architectures |
| // and operating systems. |
| environment "GOOS", "linux" |
| environment "GOARCH", "amd64" |
| |
| executable 'sh' |
| args '-c', "${project.ext.goCmd} build -o "+ ext.outputLocation + ' ' + ext.goTargets |
| } |
| } |
| } |
| |
| project.tasks.register("goTest") { |
| dependsOn goBuild |
| doLast { |
| project.exec { |
| executable 'sh' |
| args '-c', "${project.ext.goCmd} test ./..." |
| } |
| } |
| } |
| } |
| |
| /** ***********************************************************************************************/ |
| |
| project.ext.applyDockerNature = { |
| project.apply plugin: "com.palantir.docker" |
| project.docker { noCache true } |
| project.tasks.create(name: "copyLicenses", type: Copy) { |
| from "${project.rootProject.projectDir}/LICENSE" |
| from "${project.rootProject.projectDir}/LICENSE.python" |
| from "${project.rootProject.projectDir}/NOTICE" |
| into "build/target" |
| } |
| project.tasks.dockerPrepare.dependsOn project.tasks.copyLicenses |
| } |
| |
| project.ext.applyDockerRunNature = { |
| project.apply plugin: "com.palantir.docker-run" |
| } |
| /** ***********************************************************************************************/ |
| |
| project.ext.applyGroovyNature = { |
| project.apply plugin: "groovy" |
| |
| project.apply plugin: "com.diffplug.spotless" |
| def disableSpotlessCheck = project.hasProperty('disableSpotlessCheck') && |
| project.disableSpotlessCheck == 'true' |
| project.spotless { |
| enforceCheck !disableSpotlessCheck |
| def grEclipseConfig = project.project(":").file("buildSrc/greclipse.properties") |
| groovy { |
| greclipse().configFile(grEclipseConfig) |
| target project.fileTree(project.projectDir) { include '**/*.groovy' } |
| } |
| groovyGradle { greclipse().configFile(grEclipseConfig) } |
| } |
| } |
| |
| // containerImageName returns a configurable container image name, by default a |
| // development image at docker.io (see sdks/CONTAINERS.md): |
| // |
| // format: apache/beam_$NAME_sdk:latest |
| // ie: apache/beam_python3.7_sdk:latest apache/beam_java8_sdk:latest apache/beam_go_sdk:latest |
| // |
| // Both the root and tag can be defined using properties or explicitly provided. |
| project.ext.containerImageName = { |
| // Use the implicit it parameter of the closure to handle zero argument or one argument map calls. |
| ContainerImageNameConfiguration configuration = it ? it as ContainerImageNameConfiguration : new ContainerImageNameConfiguration() |
| |
| if (configuration.root == null) { |
| if (project.rootProject.hasProperty(["docker-repository-root"])) { |
| configuration.root = project.rootProject["docker-repository-root"] |
| } else { |
| configuration.root = "${System.properties["user.name"]}-docker-apache.bintray.io/beam" |
| } |
| } |
| if (configuration.tag == null) { |
| if (project.rootProject.hasProperty(["docker-tag"])) { |
| configuration.tag = project.rootProject["docker-tag"] |
| } else { |
| configuration.tag = 'latest' |
| } |
| } |
| return "${configuration.root}/${configuration.name}:${configuration.tag}" |
| } |
| |
| project.ext.containerImageTags = { |
| String[] tags |
| if (project.rootProject.hasProperty(["docker-tag-list"])) { |
| tags = project.rootProject["docker-tag-list"].split(',') |
| } else { |
| tags = [ |
| project.rootProject.hasProperty(["docker-tag"]) ? |
| project.rootProject["docker-tag"] : project.sdk_version |
| ] |
| } |
| return tags |
| } |
| |
| /** ***********************************************************************************************/ |
| |
| // applyGrpcNature should only be applied to projects who wish to use |
| // unvendored gRPC / protobuf dependencies. |
| project.ext.applyGrpcNature = { |
| project.apply plugin: "com.google.protobuf" |
| project.protobuf { |
| protoc { |
| // The artifact spec for the Protobuf Compiler |
| artifact = "com.google.protobuf:protoc:$protobuf_version" } |
| |
| // Configure the codegen plugins |
| plugins { |
| // An artifact spec for a protoc plugin, with "grpc" as |
| // the identifier, which can be referred to in the "plugins" |
| // container of the "generateProtoTasks" closure. |
| grpc { artifact = "io.grpc:protoc-gen-grpc-java:$grpc_version" } |
| } |
| |
| generateProtoTasks { |
| ofSourceSet("main")*.plugins { |
| // Apply the "grpc" plugin whose spec is defined above, without |
| // options. Note the braces cannot be omitted, otherwise the |
| // plugin will not be added. This is because of the implicit way |
| // NamedDomainObjectContainer binds the methods. |
| grpc {} |
| } |
| } |
| } |
| |
| def generatedProtoMainJavaDir = "${project.buildDir}/generated/source/proto/main/java" |
| def generatedProtoTestJavaDir = "${project.buildDir}/generated/source/proto/test/java" |
| def generatedGrpcMainJavaDir = "${project.buildDir}/generated/source/proto/main/grpc" |
| def generatedGrpcTestJavaDir = "${project.buildDir}/generated/source/proto/test/grpc" |
| project.idea { |
| module { |
| sourceDirs += project.file(generatedProtoMainJavaDir) |
| generatedSourceDirs += project.file(generatedProtoMainJavaDir) |
| |
| testSourceDirs += project.file(generatedProtoTestJavaDir) |
| generatedSourceDirs += project.file(generatedProtoTestJavaDir) |
| |
| sourceDirs += project.file(generatedGrpcMainJavaDir) |
| generatedSourceDirs += project.file(generatedGrpcMainJavaDir) |
| |
| testSourceDirs += project.file(generatedGrpcTestJavaDir) |
| generatedSourceDirs += project.file(generatedGrpcTestJavaDir) |
| } |
| } |
| } |
| |
| /** ***********************************************************************************************/ |
| |
| // applyPortabilityNature should only be applied to projects that want to use |
| // vendored gRPC / protobuf dependencies. |
| project.ext.applyPortabilityNature = { |
| PortabilityNatureConfiguration configuration = it ? it as PortabilityNatureConfiguration : new PortabilityNatureConfiguration() |
| |
| if (configuration.archivesBaseName) { |
| project.archivesBaseName = configuration.archivesBaseName |
| } |
| |
| project.ext.applyJavaNature( |
| enableStrictDependencies: false, |
| exportJavadoc: false, |
| enableSpotbugs: false, |
| publish: configuration.publish, |
| generatedClassPatterns: configuration.generatedClassPatterns, |
| archivesBaseName: configuration.archivesBaseName, |
| automaticModuleName: configuration.automaticModuleName, |
| shadowJarValidationExcludes: it.shadowJarValidationExcludes, |
| shadowClosure: GrpcVendoring_1_43_2.shadowClosure() << { |
| // We perform all the code relocations but don't include |
| // any of the actual dependencies since they will be supplied |
| // by org.apache.beam:beam-vendor-grpc-v1p43p2 |
| dependencies { |
| include(dependency { return false }) |
| } |
| }) |
| |
| // Don't force modules here because we don't want to take the shared declarations in build_rules.gradle |
| // because we would like to have the freedom to choose which versions of dependencies we |
| // are using for the portability APIs separate from what is being used inside other modules such as GCP. |
| project.configurations.all { config -> |
| config.resolutionStrategy { forcedModules = []} |
| } |
| |
| project.apply plugin: "com.google.protobuf" |
| project.protobuf { |
| protoc { |
| // The artifact spec for the Protobuf Compiler |
| artifact = "com.google.protobuf:protoc:${GrpcVendoring_1_43_2.protobuf_version}" } |
| |
| // Configure the codegen plugins |
| plugins { |
| // An artifact spec for a protoc plugin, with "grpc" as |
| // the identifier, which can be referred to in the "plugins" |
| // container of the "generateProtoTasks" closure. |
| grpc { artifact = "io.grpc:protoc-gen-grpc-java:${GrpcVendoring_1_43_2.grpc_version}" } |
| } |
| |
| generateProtoTasks { |
| ofSourceSet("main")*.plugins { |
| // Apply the "grpc" plugin whose spec is defined above, without |
| // options. Note the braces cannot be omitted, otherwise the |
| // plugin will not be added. This is because of the implicit way |
| // NamedDomainObjectContainer binds the methods. |
| grpc { } |
| } |
| } |
| } |
| |
| project.dependencies GrpcVendoring_1_43_2.dependenciesClosure() << { shadow project.ext.library.java.vendored_grpc_1_43_2 } |
| } |
| |
| /** ***********************************************************************************************/ |
| |
| // TODO: Decide whether this should be inlined into the one project that relies on it |
| // or be left here. |
| project.ext.applyAvroNature = { project.apply plugin: "com.commercehub.gradle.plugin.avro" } |
| |
| project.ext.applyAntlrNature = { |
| project.apply plugin: 'antlr' |
| project.idea { |
| module { |
| // mark antlrs output folders as generated |
| generatedSourceDirs += project.generateGrammarSource.outputDirectory |
| generatedSourceDirs += project.generateTestGrammarSource.outputDirectory |
| } |
| } |
| } |
| |
| // Creates a task to run the quickstart for a runner. |
| // Releases version and URL, can be overriden for a RC release with |
| // ./gradlew :release:runJavaExamplesValidationTask -Pver=2.3.0 -Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-1027 |
| project.ext.createJavaExamplesArchetypeValidationTask = { |
| JavaExamplesArchetypeValidationConfiguration config = it as JavaExamplesArchetypeValidationConfiguration |
| |
| def taskName = "run${config.type}Java${config.runner}" |
| def releaseVersion = project.findProperty('ver') ?: project.version |
| def releaseRepo = project.findProperty('repourl') ?: 'https://repository.apache.org/content/repositories/snapshots' |
| def argsNeeded = [ |
| "--ver=${releaseVersion}", |
| "--repourl=${releaseRepo}" |
| ] |
| if (config.gcpProject) { |
| argsNeeded.add("--gcpProject=${config.gcpProject}") |
| } |
| if (config.gcpRegion) { |
| argsNeeded.add("--gcpRegion=${config.gcpRegion}") |
| } |
| if (config.gcsBucket) { |
| argsNeeded.add("--gcsBucket=${config.gcsBucket}") |
| } |
| if (config.bqDataset) { |
| argsNeeded.add("--bqDataset=${config.bqDataset}") |
| } |
| if (config.pubsubTopic) { |
| argsNeeded.add("--pubsubTopic=${config.pubsubTopic}") |
| } |
| project.evaluationDependsOn(':release') |
| project.task(taskName, dependsOn: ':release:classes', type: JavaExec) { |
| group = "Verification" |
| description = "Run the Beam ${config.type} with the ${config.runner} runner" |
| mainClass = "${config.type}-java-${config.runner}".toLowerCase() |
| classpath = project.project(':release').sourceSets.main.runtimeClasspath |
| args argsNeeded |
| } |
| } |
| |
| |
| /** ***********************************************************************************************/ |
| |
| // Method to create the PortableValidatesRunnerTask. |
| // The method takes PortableValidatesRunnerConfiguration as parameter. |
| project.ext.createPortableValidatesRunnerTask = { |
| /* |
| * We need to rely on manually specifying these evaluationDependsOn to ensure that |
| * the following projects are evaluated before we evaluate this project. This is because |
| * we are attempting to reference the "sourceSets.test.output" directly. |
| */ |
| project.evaluationDependsOn(":sdks:java:core") |
| project.evaluationDependsOn(":runners:core-java") |
| def config = it ? it as PortableValidatesRunnerConfiguration : new PortableValidatesRunnerConfiguration() |
| |
| def name = config.name |
| def beamTestPipelineOptions = [ |
| "--runner=org.apache.beam.runners.portability.testing.TestPortableRunner", |
| "--jobServerDriver=${config.jobServerDriver}", |
| "--environmentCacheMillis=10000", |
| "--experiments=beam_fn_api", |
| ] |
| beamTestPipelineOptions.addAll(config.pipelineOpts) |
| if (config.environment == PortableValidatesRunnerConfiguration.Environment.EMBEDDED) { |
| beamTestPipelineOptions += "--defaultEnvironmentType=EMBEDDED" |
| } |
| if (config.jobServerConfig) { |
| beamTestPipelineOptions.add("--jobServerConfig=${config.jobServerConfig}") |
| } |
| config.systemProperties.put("beamTestPipelineOptions", JsonOutput.toJson(beamTestPipelineOptions)) |
| project.tasks.register(name, Test) { |
| group = "Verification" |
| description = "Validates the PortableRunner with JobServer ${config.jobServerDriver}" |
| systemProperties config.systemProperties |
| classpath = config.testClasspathConfiguration |
| testClassesDirs = project.files(project.project(":sdks:java:core").sourceSets.test.output.classesDirs, project.project(":runners:core-java").sourceSets.test.output.classesDirs) |
| maxParallelForks config.numParallelTests |
| useJUnit(config.testCategories) |
| filter(config.testFilter) |
| // increase maxHeapSize as this is directly correlated to direct memory, |
| // see https://issues.apache.org/jira/browse/BEAM-6698 |
| maxHeapSize = '4g' |
| if (config.environment == PortableValidatesRunnerConfiguration.Environment.DOCKER) { |
| dependsOn ':sdks:java:container:java8:docker' |
| } |
| } |
| } |
| |
| /** ***********************************************************************************************/ |
| |
| // Method to create the crossLanguageValidatesRunnerTask. |
| // The method takes crossLanguageValidatesRunnerConfiguration as parameter. |
| project.ext.createCrossLanguageValidatesRunnerTask = { |
| // This task won't work if the python build file doesn't exist. |
| if (!project.project(":sdks:python").buildFile.exists()) { |
| System.err.println 'Python build file not found. Skipping createCrossLanguageValidatesRunnerTask.' |
| return |
| } |
| def config = it ? it as CrossLanguageValidatesRunnerConfiguration : new CrossLanguageValidatesRunnerConfiguration() |
| |
| project.evaluationDependsOn(":sdks:python") |
| project.evaluationDependsOn(":sdks:java:testing:expansion-service") |
| project.evaluationDependsOn(":runners:core-construction-java") |
| project.evaluationDependsOn(":sdks:java:extensions:python") |
| project.evaluationDependsOn(":sdks:go:test") |
| |
| // Task for launching expansion services |
| def envDir = project.project(":sdks:python").envdir |
| def pythonDir = project.project(":sdks:python").projectDir |
| def javaPort = getRandomPort() |
| def pythonPort = getRandomPort() |
| def expansionJar = project.project(':sdks:java:testing:expansion-service').buildTestExpansionServiceJar.archivePath |
| def javaClassLookupAllowlistFile = project.project(":sdks:java:testing:expansion-service").projectDir.getPath() + "/src/test/resources/test_expansion_service_allowlist.yaml" |
| def expansionServiceOpts = [ |
| "group_id": project.name, |
| "java_expansion_service_jar": expansionJar, |
| "java_port": javaPort, |
| "java_expansion_service_allowlist_file": javaClassLookupAllowlistFile, |
| "python_expansion_service_allowlist_glob": "\\*", |
| "python_virtualenv_dir": envDir, |
| "python_expansion_service_module": "apache_beam.runners.portability.expansion_service_test", |
| "python_port": pythonPort |
| ] |
| def serviceArgs = project.project(':sdks:python').mapToArgString(expansionServiceOpts) |
| def pythonContainerSuffix = project.project(':sdks:python').pythonVersion == '2.7' ? '2' : project.project(':sdks:python').pythonVersion.replace('.', '') |
| def javaContainerSuffix |
| if (JavaVersion.current() == JavaVersion.VERSION_1_8) { |
| javaContainerSuffix = 'java8' |
| } else if (JavaVersion.current() == JavaVersion.VERSION_11) { |
| javaContainerSuffix = 'java11' |
| } else if (JavaVersion.current() == JavaVersion.VERSION_17) { |
| javaContainerSuffix = 'java17' |
| } else { |
| throw new GradleException("unsupported java version.") |
| } |
| def setupTask = project.tasks.register(config.name+"Setup", Exec) { |
| dependsOn ':sdks:java:container:'+javaContainerSuffix+':docker' |
| dependsOn ':sdks:python:container:py'+pythonContainerSuffix+':docker' |
| dependsOn ':sdks:java:testing:expansion-service:buildTestExpansionServiceJar' |
| dependsOn ":sdks:python:installGcpTest" |
| // setup test env |
| executable 'sh' |
| args '-c', "$pythonDir/scripts/run_expansion_services.sh stop --group_id ${project.name} && $pythonDir/scripts/run_expansion_services.sh start $serviceArgs" |
| } |
| |
| def mainTask = project.tasks.register(config.name) { |
| group = "Verification" |
| description = "Validates cross-language capability of runner" |
| } |
| |
| def cleanupTask = project.tasks.register(config.name+'Cleanup', Exec) { |
| // teardown test env |
| executable 'sh' |
| args '-c', "$pythonDir/scripts/run_expansion_services.sh stop --group_id ${project.name}" |
| } |
| setupTask.configure {finalizedBy cleanupTask} |
| config.startJobServer.configure {finalizedBy config.cleanupJobServer} |
| |
| def sdkLocationOpt = [] |
| if (config.needsSdkLocation) { |
| setupTask.configure {dependsOn ':sdks:python:sdist'} |
| sdkLocationOpt = [ |
| "--sdk_location=${pythonDir}/build/apache-beam.tar.gz" |
| ] |
| } |
| |
| ['Java': javaPort, 'Python': pythonPort].each { sdk, port -> |
| // Task for running testcases in Java SDK |
| def javaTask = project.tasks.register(config.name+"JavaUsing"+sdk, Test) { |
| group = "Verification" |
| description = "Validates runner for cross-language capability of using ${sdk} transforms from Java SDK" |
| systemProperty "beamTestPipelineOptions", JsonOutput.toJson(config.javaPipelineOptions) |
| systemProperty "expansionJar", expansionJar |
| systemProperty "expansionPort", port |
| systemProperty "semiPersistDir", config.semiPersistDir |
| classpath = config.classpath + project.files( |
| project.project(":runners:core-construction-java").sourceSets.test.runtimeClasspath, |
| project.project(":sdks:java:extensions:python").sourceSets.test.runtimeClasspath |
| ) |
| testClassesDirs = project.files( |
| project.project(":runners:core-construction-java").sourceSets.test.output.classesDirs, |
| project.project(":sdks:java:extensions:python").sourceSets.test.output.classesDirs |
| ) |
| maxParallelForks config.numParallelTests |
| if (sdk == "Java") { |
| useJUnit{ includeCategories 'org.apache.beam.sdk.testing.UsesJavaExpansionService' } |
| } else if (sdk == "Python") { |
| useJUnit{ includeCategories 'org.apache.beam.sdk.testing.UsesPythonExpansionService' } |
| } else { |
| throw new GradleException("unsupported expansion service for Java validate runner tests.") |
| } |
| // increase maxHeapSize as this is directly correlated to direct memory, |
| // see https://issues.apache.org/jira/browse/BEAM-6698 |
| maxHeapSize = '4g' |
| dependsOn setupTask |
| dependsOn config.startJobServer |
| } |
| if (sdk != "Java") { |
| mainTask.configure {dependsOn javaTask} |
| } |
| cleanupTask.configure {mustRunAfter javaTask} |
| config.cleanupJobServer.configure {mustRunAfter javaTask} |
| |
| // Task for running testcases in Python SDK |
| def beamPythonTestPipelineOptions = [ |
| "pipeline_opts": config.pythonPipelineOptions + sdkLocationOpt, |
| "test_opts": config.pytestOptions, |
| "suite": "xlangValidateRunner", |
| ] |
| if (sdk == "Java") { |
| beamPythonTestPipelineOptions["collect"] = "uses_java_expansion_service" |
| } else if (sdk == "Python") { |
| beamPythonTestPipelineOptions["collect"] = "uses_python_expansion_service" |
| } else { |
| throw new GradleException("unsupported expansion service for Python validate runner tests.") |
| } |
| def cmdArgs = project.project(':sdks:python').mapToArgString(beamPythonTestPipelineOptions) |
| def pythonTask = project.tasks.register(config.name+"PythonUsing"+sdk, Exec) { |
| group = "Verification" |
| description = "Validates runner for cross-language capability of using ${sdk} transforms from Python SDK" |
| environment "EXPANSION_JAR", expansionJar |
| environment "EXPANSION_PORT", port |
| executable 'sh' |
| args '-c', ". $envDir/bin/activate && cd $pythonDir && ./scripts/run_integration_test.sh $cmdArgs" |
| dependsOn setupTask |
| dependsOn config.startJobServer |
| } |
| if (sdk != "Python") { |
| mainTask.configure{dependsOn pythonTask} |
| } |
| cleanupTask.configure{mustRunAfter pythonTask} |
| config.cleanupJobServer.configure{mustRunAfter pythonTask} |
| } |
| |
| // Task for running SQL testcases in Python SDK |
| def beamPythonTestPipelineOptions = [ |
| "pipeline_opts": config.pythonPipelineOptions + sdkLocationOpt, |
| "test_opts": config.pytestOptions, |
| "suite": "xlangSqlValidateRunner", |
| "collect": "xlang_sql_expansion_service" |
| ] |
| def cmdArgs = project.project(':sdks:python').mapToArgString(beamPythonTestPipelineOptions) |
| def pythonSqlTask = project.tasks.register(config.name+"PythonUsingSql", Exec) { |
| group = "Verification" |
| description = "Validates runner for cross-language capability of using Java's SqlTransform from Python SDK" |
| executable 'sh' |
| args '-c', ". $envDir/bin/activate && cd $pythonDir && ./scripts/run_integration_test.sh $cmdArgs" |
| dependsOn setupTask |
| dependsOn config.startJobServer |
| dependsOn ':sdks:java:extensions:sql:expansion-service:shadowJar' |
| } |
| mainTask.configure{dependsOn pythonSqlTask} |
| cleanupTask.configure{mustRunAfter pythonSqlTask} |
| config.cleanupJobServer.configure{mustRunAfter pythonSqlTask} |
| |
| // Task for running Java testcases in Go SDK. |
| def pipelineOpts = [ |
| "--expansion_addr=test:localhost:${javaPort}", |
| ] |
| def goTask = project.project(":sdks:go:test:").goIoValidatesRunnerTask(project, config.name+"GoUsingJava", config.goScriptOptions, pipelineOpts) |
| goTask.configure { |
| description = "Validates runner for cross-language capability of using Java transforms from Go SDK" |
| dependsOn setupTask |
| dependsOn config.startJobServer |
| } |
| mainTask.configure{dependsOn goTask} |
| cleanupTask.configure{mustRunAfter goTask} |
| config.cleanupJobServer.configure{mustRunAfter goTask} |
| } |
| |
| /** ***********************************************************************************************/ |
| |
| project.ext.applyPythonNature = { |
| |
| // Define common lifecycle tasks and artifact types |
| project.apply plugin: "base" |
| |
| // For some reason base doesn't define a test task so we define it below and make |
| // check depend on it. This makes the Python project similar to the task layout like |
| // Java projects, see https://docs.gradle.org/4.2.1/userguide/img/javaPluginTasks.png |
| if (project.tasks.findByName('test') == null) { |
| project.task('test') {} |
| } |
| project.check.dependsOn project.test |
| |
| // Due to Beam-4256, we need to limit the length of virtualenv path to make the |
| // virtualenv activated properly. So instead of include project name in the path, |
| // we use the hash value. |
| project.ext.envdir = "${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}" |
| def pythonRootDir = "${project.rootDir}/sdks/python" |
| |
| // Python interpreter version for virtualenv setup and test run. This value can be |
| // set from commandline with -PpythonVersion, or in build script of certain project. |
| // If none of them applied, version set here will be used as default value. |
| // TODO(BEAM-12000): Move default value to Py3.9. |
| project.ext.pythonVersion = project.hasProperty('pythonVersion') ? |
| project.pythonVersion : '3.8' |
| |
| def setupVirtualenv = project.tasks.register('setupVirtualenv') { |
| doLast { |
| def virtualenvCmd = [ |
| "python${project.ext.pythonVersion}", |
| "-m", |
| "venv", |
| "--clear", |
| "${project.ext.envdir}", |
| ] |
| project.exec { |
| executable 'sh' |
| args '-c', virtualenvCmd.join(' ') |
| } |
| project.exec { |
| executable 'sh' |
| args '-c', ". ${project.ext.envdir}/bin/activate && " + |
| "pip install --retries 10 --upgrade pip && " + |
| "pip install --retries 10 --upgrade tox==3.20.1 -r ${project.rootDir}/sdks/python/build-requirements.txt" |
| } |
| } |
| // Gradle will delete outputs whenever it thinks they are stale. Putting a |
| // specific binary here could make gradle delete it while pip will believe |
| // the package is fully installed. |
| outputs.dirs(project.ext.envdir) |
| } |
| |
| project.ext.pythonSdkDeps = project.files( |
| project.fileTree( |
| dir: "${project.rootDir}", |
| include: ['model/**', 'sdks/python/**'], |
| // Exclude temporary directories and files that are generated |
| // during build and test. |
| exclude: [ |
| '**/build/**', |
| '**/dist/**', |
| '**/target/**', |
| '**/.gogradle/**', |
| '**/*.pyc', |
| 'sdks/python/*.egg*/**', |
| 'sdks/python/test-suites/**', |
| 'sdks/python/__pycache__', |
| '**/reports/test/index.html', |
| ]) |
| ) |
| def copiedSrcRoot = "${project.buildDir}/srcs" |
| |
| // Create new configuration distTarBall which represents Python source |
| // distribution tarball generated by :sdks:python:sdist. |
| project.configurations { distTarBall } |
| |
| def installGcpTest = project.tasks.register('installGcpTest') { |
| dependsOn setupVirtualenv |
| dependsOn ':sdks:python:sdist' |
| doLast { |
| def distTarBall = "${pythonRootDir}/build/apache-beam.tar.gz" |
| project.exec { |
| executable 'sh' |
| args '-c', ". ${project.ext.envdir}/bin/activate && pip install --retries 10 ${distTarBall}[gcp,test,aws,azure,dataframe]" |
| } |
| } |
| } |
| |
| def cleanPython = project.tasks.register('cleanPython') { |
| doLast { |
| def activate = "${project.ext.envdir}/bin/activate" |
| project.exec { |
| executable 'sh' |
| args '-c', "if [ -e ${activate} ]; then " + |
| ". ${activate} && cd ${pythonRootDir} && python setup.py clean; " + |
| "fi" |
| } |
| project.delete project.buildDir // Gradle build directory |
| project.delete project.ext.envdir // virtualenv directory |
| project.delete "$project.projectDir/target" // tox work directory |
| } |
| } |
| project.clean.dependsOn cleanPython |
| // Force this subproject's clean to run before the main :clean, to avoid |
| // racing on deletes. |
| project.rootProject.clean.dependsOn project.clean |
| |
| // Return a joined String from a Map that contains all commandline args of |
| // IT test. |
| project.ext.mapToArgString = { argMap -> |
| def argList = [] |
| argMap.each { k, v -> |
| if (v in List) { |
| v = "\"${v.join(' ')}\"" |
| } else if (v in String && v.contains(' ')) { |
| // We should use double quote around the arg value if it contains series |
| // of flags joined with space. Otherwise, commandline parsing of the |
| // shell script will be broken. |
| v = "\"${v.replace('"', '')}\"" |
| } |
| argList.add("--$k $v") |
| } |
| return argList.join(' ') |
| } |
| |
| project.ext.toxTask = { name, tox_env, posargs='' -> |
| project.tasks.register(name) { |
| dependsOn setupVirtualenv |
| dependsOn ':sdks:python:sdist' |
| |
| doLast { |
| // Python source directory is also tox execution workspace, We want |
| // to isolate them per tox suite to avoid conflict when running |
| // multiple tox suites in parallel. |
| project.copy { from project.pythonSdkDeps; into copiedSrcRoot } |
| |
| def copiedPyRoot = "${copiedSrcRoot}/sdks/python" |
| def distTarBall = "${pythonRootDir}/build/apache-beam.tar.gz" |
| project.exec { |
| executable 'sh' |
| args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env $distTarBall '$posargs'" |
| } |
| } |
| inputs.files project.pythonSdkDeps |
| outputs.files project.fileTree(dir: "${pythonRootDir}/target/.tox/${tox_env}/log/") |
| } |
| } |
| |
| // Run single or a set of integration tests with provided test options and pipeline options. |
| project.ext.enablePythonPerformanceTest = { |
| |
| // Use the implicit it parameter of the closure to handle zero argument or one argument map calls. |
| // See: http://groovy-lang.org/closures.html#implicit-it |
| def config = it ? it as PythonPerformanceTestConfiguration : new PythonPerformanceTestConfiguration() |
| |
| project.tasks.register('integrationTest') { |
| dependsOn installGcpTest |
| dependsOn ':sdks:python:sdist' |
| |
| doLast { |
| def argMap = [:] |
| |
| // Build test options that configures test environment and framework |
| def testOptions = [] |
| if (config.tests) |
| testOptions += "$config.tests" |
| if (config.attribute) |
| testOptions += "-m=$config.attribute" |
| testOptions.addAll(config.extraTestOptions) |
| argMap["test_opts"] = testOptions |
| |
| // Build pipeline options that configures pipeline job |
| if (config.pipelineOptions) |
| argMap["pipeline_opts"] = config.pipelineOptions |
| if (config.kmsKeyName) |
| argMap["kms_key_name"] = config.kmsKeyName |
| argMap["suite"] = "integrationTest-perf" |
| |
| def cmdArgs = project.mapToArgString(argMap) |
| def runScriptsDir = "${pythonRootDir}/scripts" |
| project.exec { |
| executable 'sh' |
| args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh ${cmdArgs}" |
| } |
| } |
| } |
| } |
| |
| def addPortableWordCountTask = { boolean isStreaming, String runner -> |
| def taskName = 'portableWordCount' + runner + (isStreaming ? 'Streaming' : 'Batch') |
| def flinkJobServerProject = ":runners:flink:${project.ext.latestFlinkVersion}:job-server" |
| project.tasks.register(taskName) { |
| dependsOn = [installGcpTest] |
| mustRunAfter = [ |
| ":runners:flink:${project.ext.latestFlinkVersion}:job-server:shadowJar", |
| ':runners:spark:2:job-server:shadowJar', |
| ':sdks:python:container:py37:docker', |
| ':sdks:python:container:py38:docker', |
| ':sdks:python:container:py39:docker', |
| ] |
| doLast { |
| // TODO: Figure out GCS credentials and use real GCS input and output. |
| def outputDir = File.createTempDir(taskName, '') |
| def options = [ |
| "--input=/etc/profile", |
| "--output=${outputDir}/out.txt", |
| "--runner=${runner}", |
| "--parallelism=2", |
| "--sdk_worker_parallelism=1", |
| "--flink_job_server_jar=${project.project(flinkJobServerProject).shadowJar.archivePath}", |
| "--spark_job_server_jar=${project.project(':runners:spark:2:job-server').shadowJar.archivePath}", |
| ] |
| if (isStreaming) |
| options += [ |
| "--streaming" |
| ] |
| else |
| // workaround for local file output in docker container |
| options += [ |
| "--environment_cache_millis=60000" |
| ] |
| if (project.hasProperty("jobEndpoint")) |
| options += [ |
| "--job_endpoint=${project.property('jobEndpoint')}" |
| ] |
| if (project.hasProperty("environmentType")) { |
| options += [ |
| "--environment_type=${project.property('environmentType')}" |
| ] |
| } |
| if (project.hasProperty("environmentConfig")) { |
| options += [ |
| "--environment_config=${project.property('environmentConfig')}" |
| ] |
| } |
| project.exec { |
| executable 'sh' |
| args '-c', ". ${project.ext.envdir}/bin/activate && python -m apache_beam.examples.wordcount ${options.join(' ')}" |
| // TODO: Check that the output file is generated and runs. |
| } |
| } |
| } |
| } |
| project.ext.addPortableWordCountTasks = { |
| -> |
| addPortableWordCountTask(false, "FlinkRunner") |
| addPortableWordCountTask(true, "FlinkRunner") |
| addPortableWordCountTask(false, "SparkRunner") |
| } |
| |
| project.ext.getVersionSuffix = { String version -> |
| return version == '2.7' ? '2' : version.replace('.', '') |
| } |
| |
| project.ext.getVersionsAsList = { String propertyName -> |
| return project.getProperty(propertyName).split(',') |
| } |
| } |
| } |
| |
| private void setAutomaticModuleNameHeader(JavaNatureConfiguration configuration, Project project) { |
| if (configuration.publish && !configuration.automaticModuleName) { |
| throw new GradleException("Expected automaticModuleName to be set for the module that is published to maven repository.") |
| } else if (configuration.automaticModuleName) { |
| project.jar.manifest { |
| attributes 'Automatic-Module-Name': configuration.automaticModuleName |
| } |
| } |
| } |
| } |