| /* | |
| * Licensed to the Apache Software Foundation (ASF) under one | |
| * or more contributor license agreements. See the NOTICE file | |
| * distributed with this work for additional information | |
| * regarding copyright ownership. The ASF licenses this file | |
| * to you under the Apache License, Version 2.0 (the | |
| * "License"); you may not use this file except in compliance | |
| * with the License. You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, software | |
| * distributed under the License is distributed on an "AS IS" BASIS, | |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| * See the License for the specific language governing permissions and | |
| * limitations under the License. | |
| */ | |
| plugins { id 'org.apache.beam.module' } | |
| // Set a specific version of 'com.google.apis:google-api-services-dataflow' | |
| // by adding -Pdataflow.version=<version> in Gradle command. Otherwise, | |
| // 'google_clients_version' defined in BeamModulePlugin will be used as default. | |
| def DATAFLOW_VERSION = "dataflow.version" | |
| // Get full dependency of 'com.google.apis:google-api-services-dataflow' | |
| def google_api_services_dataflow = project.hasProperty(DATAFLOW_VERSION) ? "com.google.apis:google-api-services-dataflow:" + getProperty(DATAFLOW_VERSION) : library.java.google_api_services_dataflow | |
| // Returns a string representing the relocated path to be used with the shadow | |
| // plugin when given a suffix such as "com.". | |
| def getWorkerRelocatedPath = { String suffix -> | |
| return ("org.apache.beam.runners.dataflow.worker.repackaged." | |
| + suffix) | |
| } | |
| def sdk_provided_dependencies = [ | |
| google_api_services_dataflow, | |
| library.java.avro, | |
| library.java.google_http_client, | |
| library.java.google_http_client_gson, | |
| library.java.jackson_annotations, | |
| library.java.jackson_core, | |
| library.java.jackson_databind, | |
| library.java.joda_time, | |
| library.java.slf4j_api, | |
| library.java.vendored_grpc_1_69_0, | |
| ] | |
| def sdk_provided_shaded_project_dependencies = [ | |
| ":model:pipeline", | |
| ":sdks:java:core", | |
| ] | |
| def sdk_provided_project_dependencies = [ | |
| ":runners:google-cloud-dataflow-java", | |
| ":sdks:java:extensions:avro", | |
| ":sdks:java:extensions:google-cloud-platform-core", | |
| ":sdks:java:io:google-cloud-platform", | |
| ] | |
| // Exclude unneeded dependencies when building jar | |
| def excluded_dependencies = [ | |
| "com.google.auto.service:auto-service", // Provided scope added from applyJavaNature | |
| "com.google.auto.value:auto-value", // Provided scope added from applyJavaNature | |
| "org.codehaus.jackson:jackson-core-asl", // Exclude an old version of jackson-core-asl introduced by google-http-client-jackson | |
| "org.objenesis:objenesis", // Transitive dependency introduced from Beam | |
| library.java.commons_compress, // Transitive dependency introduced from Beam | |
| library.java.error_prone_annotations, // Provided scope added in worker | |
| library.java.hamcrest, // Test only | |
| library.java.junit, // Test only | |
| library.java.jsonassert, // Test only | |
| library.java.truth // Test only | |
| ] | |
| applyJavaNature( | |
| automaticModuleName: 'org.apache.beam.runners.dataflow.worker', | |
| archivesBaseName: 'beam-runners-google-cloud-dataflow-java-legacy-worker', | |
| classesTriggerCheckerBugs: [ | |
| 'BatchGroupAlsoByWindowAndCombineFn' : 'TODO: file a bug report', | |
| 'AssignWindowsParDoFnFactory' : 'TODO: file a bug report', | |
| 'FetchAndFilterStreamingSideInputsOperation': 'https://github.com/typetools/checker-framework/issues/5436', | |
| ], | |
| exportJavadoc: false, | |
| enableSpotbugs: false /* TODO(BEAM-5658): enable spotbugs */, | |
| shadowJarValidationExcludes: [ | |
| "org/apache/beam/runners/dataflow/worker/**", | |
| "org/apache/beam/repackaged/beam_runners_google_cloud_dataflow_java_legacy_worker/**", | |
| // TODO(https://github.com/apache/beam/issues/19114): Move DataflowRunnerHarness class under org.apache.beam.runners.dataflow.worker namespace | |
| "com/google/cloud/dataflow/worker/DataflowRunnerHarness.class", | |
| // Allow slf4j implementation worker for logging during pipeline execution | |
| "org/slf4j/jul/**" | |
| ], | |
| generatedClassPatterns: [ | |
| /^org\.apache\.beam\.runners\.dataflow\.worker\.windmill\..*AutoBuilder.*/, | |
| /^org\.apache\.beam\.runners\.dataflow\.worker\..*AutoBuilder.*/, | |
| ], | |
| shadowClosure: { | |
| // Each included dependency must also include all of its necessary transitive dependencies | |
| // or have them provided by the users pipeline during job submission. Typically a users | |
| // pipeline includes :runners:google-cloud-dataflow-java and its transitive dependencies | |
| // so those dependencies don't need to be shaded (bundled and relocated) away. All other | |
| // dependencies needed to run the worker must be shaded (bundled and relocated) to prevent | |
| // ClassNotFound and/or MethodNotFound errors during pipeline execution. | |
| // | |
| // Each included dependency should have a matching relocation rule below that ensures | |
| // that the shaded jar is correctly built. | |
| dependencies { | |
| include(dependency(library.java.slf4j_jdk14)) | |
| } | |
| dependencies { | |
| include(project(path: ":model:fn-execution", configuration: "shadow")) | |
| } | |
| relocate("org.apache.beam.model.fnexecution.v1", getWorkerRelocatedPath("org.apache.beam.model.fnexecution.v1")) | |
| dependencies { | |
| include(project(":runners:core-java")) | |
| } | |
| relocate("org.apache.beam.runners.core", getWorkerRelocatedPath("org.apache.beam.runners.core")) | |
| relocate("org.apache.beam.repackaged.beam_runners_core_construction_java", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_core_construction_java")) | |
| relocate("org.apache.beam.repackaged.beam_runners_core_java", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_core_java")) | |
| dependencies { | |
| include(project(":runners:java-fn-execution")) | |
| } | |
| relocate("org.apache.beam.runners.fnexecution", getWorkerRelocatedPath("org.apache.beam.runners.fnexecution")) | |
| relocate("org.apache.beam.repackaged.beam_runners_java_fn_execution", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_runners_java_fn_execution")) | |
| relocate("org.apache.beam.repackaged.beam_sdks_java_fn_execution", getWorkerRelocatedPath("org.apache.beam.repackaged.beam_sdks_java_fn_execution")) | |
| dependencies { | |
| // We have to include jetty-server/jetty-servlet and all of its transitive dependencies | |
| // which includes several org.eclipse.jetty artifacts + servlet-api | |
| include(dependency("org.eclipse.jetty:.*:9.4.58.v20250814")) | |
| include(dependency("javax.servlet:javax.servlet-api:3.1.0")) | |
| } | |
| relocate("org.eclipse.jetty", getWorkerRelocatedPath("org.eclipse.jetty")) | |
| relocate("javax.servlet", getWorkerRelocatedPath("javax.servlet")) | |
| // We don't relocate windmill since it is already underneath the org.apache.beam.runners.dataflow.worker namespace and never | |
| // expect a user pipeline to include it. There is also a JNI component that windmill server relies on which makes | |
| // arbitrary relocation more difficult. | |
| dependencies { | |
| include(project(path: ":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow")) | |
| } | |
| // Include original source files extracted under | |
| // '$buildDir/original_sources_to_package' to jar | |
| from "$buildDir/original_sources_to_package" | |
| exclude "META-INF/LICENSE.txt" | |
| exclude "META-INF/maven/org.eclipse.jetty/**" | |
| exclude "about.html" | |
| }) | |
| /******************************************************************************/ | |
| // Configure the worker root project | |
| tasks.withType(Test).configureEach { | |
| // WindmillStateTestUtils.assertNoReference walks every object reachable from the Windmill | |
| // state cache (Guava Cache / ConcurrentHashMap) to ensure no per-work-item WindmillStateReader | |
| // leaks into the global cache. It uses reflection (Field.setAccessible) into JDK internals | |
| // (e.g. Integer.value, AtomicReferenceArray.array, ReentrantLock.sync, ReferenceQueue.head), | |
| // which requires --add-opens on Java 17+. | |
| jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED', | |
| '--add-opens=java.base/java.lang.ref=ALL-UNNAMED', | |
| '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', | |
| '--add-opens=java.base/java.io=ALL-UNNAMED', | |
| '--add-opens=java.base/java.util=ALL-UNNAMED', | |
| '--add-opens=java.base/java.util.concurrent=ALL-UNNAMED', | |
| '--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED', | |
| '--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED' | |
| } | |
| configurations { | |
| sourceFile | |
| // Ban these dependencies from all configurations | |
| all { | |
| // Ban the usage of AppleJavaExtensions in spotbugs. | |
| exclude group: "com.apple", module: "AppleJavaExtensions" | |
| } | |
| } | |
| dependencies { | |
| implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) | |
| implementation library.java.gax | |
| implementation library.java.google_cloud_core | |
| implementation library.java.guava | |
| permitUnusedDeclared library.java.guava | |
| implementation library.java.protobuf_java | |
| // Note that any dependency that is modified here should also be modified within | |
| // runners/google-cloud-dataflow-java/worker/build.gradle using the rules provided | |
| // there. | |
| sdk_provided_dependencies.each { | |
| provided(it) | |
| testImplementation(it) | |
| } | |
| sdk_provided_shaded_project_dependencies.each { | |
| provided project(path: it, configuration: "shadow") | |
| } | |
| sdk_provided_project_dependencies.each { | |
| provided project(path: it) | |
| testImplementation project(path: it) | |
| } | |
| implementation project(path: ":model:fn-execution", configuration: "shadow") | |
| implementation project(path: ":model:pipeline", configuration: "shadow") | |
| implementation project(path: ":sdks:java:core", configuration: "shadow") | |
| implementation project(":runners:core-java") | |
| implementation project(":runners:java-fn-execution") | |
| implementation project(path: ":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow") | |
| shadow library.java.vendored_guava_32_1_2_jre | |
| implementation library.java.google_auth_library_credentials | |
| implementation library.java.proto_google_common_protos | |
| implementation library.java.google_cloud_logging | |
| // Conscrypt shouldn't be included here because Conscrypt won't work when being shaded. | |
| // (Context: https://github.com/apache/beam/pull/13846) | |
| // Conscrypt will be added to runtime dependencies by GrpcVendoring so compileOnly works for now. | |
| compileOnly "org.conscrypt:conscrypt-openjdk-uber:2.5.1" | |
| implementation "javax.servlet:javax.servlet-api:3.1.0" | |
| implementation "org.eclipse.jetty:jetty-server:9.4.58.v20250814" | |
| implementation "org.eclipse.jetty:jetty-servlet:9.4.58.v20250814" | |
| implementation library.java.avro | |
| implementation library.java.jackson_annotations | |
| implementation library.java.jackson_core | |
| implementation library.java.jackson_databind | |
| implementation library.java.joda_time | |
| implementation library.java.opentelemetry_context | |
| testImplementation library.java.opentelemetry_sdk_testing | |
| implementation library.java.opentelemetry_api | |
| implementation library.java.slf4j_api | |
| implementation library.java.vendored_grpc_1_69_0 | |
| implementation library.java.error_prone_annotations | |
| permitUnusedDeclared library.java.error_prone_annotations | |
| runtimeOnly library.java.slf4j_jdk14 | |
| // Any test dependency which intersects with our relocation rules above needs to be relocated | |
| // as well and placed within the testImplementation configuration. Otherwise we can place it within | |
| // the shadowTest configuration. | |
| testImplementation project(path: ":runners:core-java", configuration: "testRuntimeMigration") | |
| // TODO: excluding Guava until Truth updates it to >32.1.x | |
| testImplementation(library.java.truth) { | |
| exclude group: 'com.google.guava', module: 'guava' | |
| } | |
| shadowTest project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "testRuntimeMigration") | |
| shadowTest project(path: ":runners:direct-java", configuration: "shadow") | |
| shadowTest project(path: ":sdks:java:harness", configuration: "shadowTest") | |
| shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") | |
| shadowTest library.java.jsonassert | |
| shadowTest library.java.junit | |
| shadowTest library.java.mockito_core | |
| } | |
| project.task('validateShadedJarContainsSlf4jJdk14', dependsOn: 'shadowJar') { | |
| ext.outFile = project.file("${project.reportsDir}/${name}.out") | |
| inputs.files(project.configurations.shadow.artifacts.files) | |
| .withPropertyName("shadowArtifactsFiles") | |
| .withPathSensitivity(PathSensitivity.RELATIVE) | |
| outputs.files outFile | |
| doLast { | |
| project.configurations.shadow.artifacts.files.each { | |
| FileTree slf4jImpl = project.zipTree(it).matching { | |
| include "org/slf4j/jul/JDK14LoggerAdapter.class" | |
| } | |
| outFile.text = slf4jImpl.files | |
| if (slf4jImpl.files.isEmpty()) { | |
| throw new GradleException("Did not find slf4j-jdk14 in Dataflow Worker uber jar") | |
| } | |
| } | |
| } | |
| } | |
| tasks.check.dependsOn project.tasks.validateShadedJarContainsSlf4jJdk14 | |
| //TODO(https://github.com/apache/beam/issues/19115): checktyle task should be enabled in the future. | |
| checkstyleMain.enabled = false | |
| checkstyleTest.enabled = false | |
| //TODO(https://github.com/apache/beam/issues/19119): javadoc task should be enabled in the future. | |
| javadoc.enabled = false |