blob: b947b9eee364edb28e70d8f7a42a5b863130c57f [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
plugins {
id 'base'
// This plugin provides a task to determine which dependencies have updates.
// Additionally, the plugin checks for updates to Gradle itself.
//
// See https://github.com/ben-manes/gradle-versions-plugin for further details.
id 'com.github.ben-manes.versions' version '0.33.0'
// Apply one top level rat plugin to perform any required license enforcement analysis
id 'org.nosphere.apache.rat' version '0.7.0'
// Enable gradle-based release management
id 'net.researchgate.release' version '2.8.1'
id 'org.apache.beam.module'
id "org.sonarqube" version "3.0"
}
/*************************************************************************************************/
// Configure the root project
rat {
// Set input directory to that of the root project instead of the CWD. This
// makes .gitignore rules (added below) work properly.
inputDir = project.rootDir
def exclusions = [
// Ignore files we track but do not distribute
"**/.github/**/*",
"gradlew",
"gradlew.bat",
"gradle/wrapper/gradle-wrapper.properties",
"**/package-list",
"**/test.avsc",
"**/user.avsc",
"**/test/resources/**/*.txt",
"**/test/**/.placeholder",
// Default eclipse excludes neglect subprojects
// Proto/grpc generated wrappers
"**/apache_beam/portability/api/*_pb2*.py",
"**/go/pkg/beam/**/*.pb.go",
// Ignore go.sum files, which don't permit headers
"**/go.sum",
// Ignore Go test data files
"**/go/data/**",
// VCF test files
"**/apache_beam/testing/data/vcf/*",
// JDBC package config files
"**/META-INF/services/java.sql.Driver",
// Website build files
"**/Gemfile.lock",
"**/Rakefile",
"**/.htaccess",
"website/www/site/assets/scss/_bootstrap.scss",
"website/www/site/assets/scss/bootstrap/**/*",
"website/www/site/static/images/mascot/*.ai",
"website/www/site/static/js/bootstrap*.js",
"website/www/site/static/js/bootstrap/**/*",
"website/www/site/themes",
"website/www/yarn.lock",
"website/www/package.json",
// Ignore ownership files
"ownership/**/*",
"**/OWNERS",
// Json doesn't support comments.
"**/*.json",
// Katas files
"learning/katas/**/course-remote-info.yaml",
"learning/katas/**/section-remote-info.yaml",
"learning/katas/**/lesson-remote-info.yaml",
"learning/katas/**/task-remote-info.yaml",
"learning/katas/**/*.txt",
// test p8 file for SnowflakeIO
"sdks/java/io/snowflake/src/test/resources/invalid_test_rsa_key.p8",
"sdks/java/io/snowflake/src/test/resources/valid_test_rsa_key.p8",
// Mockito extensions
"sdks/java/io/amazon-web-services2/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",
"sdks/java/io/azure/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",
"sdks/java/extensions/ml/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",
// JupyterLab extensions
"sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/yarn.lock",
// Sample text file for Java quickstart
"sdks/java/maven-archetypes/examples/sample.txt"
]
// Add .gitignore excludes to the Apache Rat exclusion list. We re-create the behavior
// of the Apache Maven Rat plugin since the Apache Ant Rat plugin doesn't do this
// automatically.
def gitIgnore = project(':').file('.gitignore')
if (gitIgnore.exists()) {
def gitIgnoreExcludes = gitIgnore.readLines().findAll { !it.isEmpty() && !it.startsWith('#') }
exclusions.addAll(gitIgnoreExcludes)
}
failOnError = true
excludes = exclusions
}
check.dependsOn rat
// Define root pre/post commit tasks simplifying what is needed
// to be specified on the commandline when executing locally.
// This indirection also makes Jenkins use the branch of the PR
// for the test definitions.
task javaPreCommit() {
// We need to list the model/* builds since sdks/java/core doesn't
// depend on any of the model.
dependsOn ":model:pipeline:build"
dependsOn ":model:job-management:build"
dependsOn ":model:fn-execution:build"
dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:build"
dependsOn ":sdks:java:core:buildNeeded"
dependsOn ":sdks:java:core:buildDependents"
dependsOn ":examples:java:preCommit"
dependsOn ":sdks:java:extensions:sql:jdbc:preCommit"
dependsOn ":sdks:java:javadoc:allJavadoc"
dependsOn ":runners:direct-java:needsRunnerTests"
dependsOn ":sdks:java:container:java8:docker"
}
task sqlPreCommit() {
dependsOn ":sdks:java:extensions:sql:runBasicExample"
dependsOn ":sdks:java:extensions:sql:runPojoExample"
dependsOn ":sdks:java:extensions:sql:build"
dependsOn ":sdks:java:extensions:sql:buildDependents"
}
task javaPreCommitPortabilityApi() {
dependsOn ":runners:google-cloud-dataflow-java:worker:build"
}
task javaPostCommit() {
dependsOn ":runners:google-cloud-dataflow-java:postCommit"
dependsOn ":runners:google-cloud-dataflow-java:postCommitRunnerV2"
dependsOn ":sdks:java:extensions:google-cloud-platform-core:postCommit"
dependsOn ":sdks:java:extensions:zetasketch:postCommit"
dependsOn ":sdks:java:io:google-cloud-platform:postCommit"
dependsOn ":sdks:java:io:kinesis:integrationTest"
dependsOn ":sdks:java:extensions:ml:postCommit"
dependsOn ":javaHadoopVersionsTest"
dependsOn ":sdks:java:io:kafka:kafkaVersionsCompatibilityTest"
}
task javaHadoopVersionsTest() {
dependsOn ":sdks:java:io:hadoop-common:hadoopVersionsTest"
dependsOn ":sdks:java:io:hadoop-file-system:hadoopVersionsTest"
dependsOn ":sdks:java:io:hadoop-format:hadoopVersionsTest"
dependsOn ":sdks:java:io:hcatalog:hadoopVersionsTest"
dependsOn ":sdks:java:io:parquet:hadoopVersionsTest"
dependsOn ":sdks:java:extensions:sorter:hadoopVersionsTest"
dependsOn ":runners:spark:hadoopVersionsTest"
}
task sqlPostCommit() {
dependsOn ":sdks:java:extensions:sql:postCommit"
dependsOn ":sdks:java:extensions:sql:jdbc:postCommit"
dependsOn ":sdks:java:extensions:sql:datacatalog:postCommit"
dependsOn ":sdks:java:extensions:sql:hadoopVersionsTest"
}
task goPreCommit() {
dependsOn ":sdks:go:goBuild"
dependsOn ":sdks:go:goTest"
dependsOn ":sdks:go:examples:goBuild"
dependsOn ":sdks:go:test:goBuild"
// Ensure all container Go boot code builds as well.
dependsOn ":sdks:java:container:goBuild"
dependsOn ":sdks:python:container:goBuild"
dependsOn ":sdks:go:container:goBuild"
}
task goPostCommit() {
dependsOn ":goIntegrationTests"
}
task goIntegrationTests() {
doLast {
exec {
executable 'sh'
args '-c', './sdks/go/test/run_integration_tests.sh'
}
}
dependsOn ":sdks:go:test:build"
dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
}
task pythonPreCommit() {
dependsOn ":sdks:python:test-suites:tox:pycommon:preCommitPyCommon"
dependsOn ":sdks:python:test-suites:tox:py36:preCommitPy36"
dependsOn ":sdks:python:test-suites:tox:py37:preCommitPy37"
dependsOn ":sdks:python:test-suites:tox:py38:preCommitPy38"
dependsOn ":sdks:python:test-suites:dataflow:preCommitIT"
dependsOn ":sdks:python:test-suites:dataflow:preCommitIT_V2"
}
task pythonDocsPreCommit() {
dependsOn ":sdks:python:test-suites:tox:pycommon:docs"
}
task pythonDockerBuildPreCommit() {
dependsOn ":sdks:python:container:py36:docker"
dependsOn ":sdks:python:container:py37:docker"
dependsOn ":sdks:python:container:py38:docker"
}
task pythonLintPreCommit() {
dependsOn ":sdks:python:test-suites:tox:py37:lint"
}
task pythonFormatterPreCommit() {
dependsOn 'sdks:python:test-suites:tox:py38:formatter'
}
task python36PostCommit() {
dependsOn ":sdks:python:test-suites:dataflow:py36:postCommitIT"
dependsOn ":sdks:python:test-suites:direct:py36:postCommitIT"
dependsOn ":sdks:python:test-suites:portable:py36:postCommitPy36"
}
task python37PostCommit() {
dependsOn ":sdks:python:test-suites:dataflow:py37:postCommitIT"
dependsOn ":sdks:python:test-suites:direct:py37:postCommitIT"
dependsOn ":sdks:python:test-suites:direct:py37:directRunnerIT"
dependsOn ":sdks:python:test-suites:direct:py37:hdfsIntegrationTest"
dependsOn ":sdks:python:test-suites:direct:py37:mongodbioIT"
dependsOn ":sdks:python:test-suites:portable:py37:postCommitPy37"
}
task python38PostCommit() {
dependsOn ":sdks:python:test-suites:dataflow:py38:postCommitIT"
dependsOn ":sdks:python:test-suites:direct:py38:postCommitIT"
dependsOn ":sdks:python:test-suites:direct:py38:hdfsIntegrationTest"
dependsOn ":sdks:python:test-suites:portable:py38:postCommitPy38"
}
task portablePythonPreCommit() {
dependsOn ":sdks:python:test-suites:portable:py36:preCommitPy36"
dependsOn ":sdks:python:test-suites:portable:py37:preCommitPy37"
}
task pythonSparkPostCommit() {
dependsOn ":sdks:python:test-suites:portable:py36:sparkValidatesRunner"
dependsOn ":sdks:python:test-suites:portable:py37:sparkValidatesRunner"
dependsOn ":sdks:python:test-suites:portable:py38:sparkValidatesRunner"
}
task websitePreCommit() {
dependsOn ":website:preCommit"
}
task communityMetricsPreCommit() {
dependsOn ":beam-test-infra-metrics:preCommit"
}
task communityMetricsProber() {
dependsOn ":beam-test-infra-metrics:checkProber"
}
task javaExamplesDataflowPrecommit() {
dependsOn ":runners:google-cloud-dataflow-java:examples:preCommit"
dependsOn ":runners:google-cloud-dataflow-java:examples-streaming:preCommit"
}
task runBeamDependencyCheck() {
dependsOn ":dependencyUpdates"
dependsOn ":sdks:python:dependencyUpdates"
}
task whitespacePreCommit() {
dependsOn ":sdks:python:test-suites:tox:py38:archiveFilesToLint"
dependsOn ":sdks:python:test-suites:tox:py38:unpackFilesToLint"
dependsOn ":sdks:python:test-suites:tox:py38:whitespacelint"
}
task typescriptPreCommit() {
dependsOn ":sdks:python:test-suites:tox:py38:eslint"
dependsOn ":sdks:python:test-suites:tox:py38:jest"
}
task pushAllDockerImages() {
dependsOn ":runners:spark:job-server:container:dockerPush"
dependsOn ":sdks:java:container:pushAll"
dependsOn ":sdks:python:container:pushAll"
for (version in project.ext.allFlinkVersions) {
dependsOn ":runners:flink:${version}:job-server-container:dockerPush"
}
}
// Configure the release plugin to do only local work; the release manager determines what, if
// anything, to push. On failure, the release manager can reset the branch without pushing.
release {
revertOnFail = false
tagTemplate = 'v${version}'
git {
requireBranch = 'release-.*|master'
pushToRemote = ''
}
}
// Reports linkage errors across multiple Apache Beam artifact ids.
//
// To use (from the root of project):
// ./gradlew -Ppublishing -PjavaLinkageArtifactIds=artifactId1,artifactId2,... :checkJavaLinkage
//
// For example:
// ./gradlew -Ppublishing -PjavaLinkageArtifactIds=beam-sdks-java-core,beam-sdks-java-io-jdbc :checkJavaLinkage
//
// Note that this task publishes artifacts into your local Maven repository.
if (project.hasProperty('javaLinkageArtifactIds')) {
if (!project.hasProperty('publishing')) {
throw new GradleException('You can only check linkage of Java artifacts if you specify -Ppublishing on the command line as well.')
}
configurations { linkageCheckerJava }
dependencies {
linkageCheckerJava "com.google.cloud.tools:dependencies:1.5.4"
}
// We need to evaluate all the projects first so that we can find depend on all the
// publishMavenJavaPublicationToMavenLocal tasks below.
for (p in rootProject.subprojects) {
if (!p.path.equals(project.path)) {
evaluationDependsOn(p.path)
}
}
project.task('checkJavaLinkage', type: JavaExec) {
dependsOn project.getTasksByName('publishMavenJavaPublicationToMavenLocal', true /* recursively */)
classpath = project.configurations.linkageCheckerJava
main = 'com.google.cloud.tools.opensource.classpath.LinkageCheckerMain'
def arguments = ['-a', project.javaLinkageArtifactIds.split(',').collect({
if (it.contains(':')) {
"${project.ext.mavenGroupId}:${it}"
} else {
// specify the version if not provided
"${project.ext.mavenGroupId}:${it}:${project.version}"
}
}).join(',')]
// Exclusion file filters out existing linkage errors before a change
if (project.hasProperty('javaLinkageWriteBaseline')) {
arguments += ['--output-exclusion-file', project.getProperty('javaLinkageWriteBaseline')]
} else if (project.hasProperty('javaLinkageReadBaseline')) {
arguments += ['--exclusion-file', project.getProperty('javaLinkageReadBaseline')]
}
args arguments
doLast {
println "NOTE: This task published artifacts into your local Maven repository. You may want to remove them manually."
}
}
}
if (project.hasProperty('compileAndRunTestsWithJava11')) {
project.javaPreCommitPortabilityApi.dependsOn ':sdks:java:testing:test-utils:verifyJavaVersion'
project.javaExamplesDataflowPrecommit.dependsOn ':sdks:java:testing:test-utils:verifyJavaVersion'
project.sqlPreCommit.dependsOn ':sdks:java:testing:test-utils:verifyJavaVersion'
} else {
allprojects {
tasks.withType(Test).configureEach {
exclude '**/JvmVerification.class'
}
}
}