| /* | |
| * Licensed to the Apache Software Foundation (ASF) under one | |
| * or more contributor license agreements. See the NOTICE file | |
| * distributed with this work for additional information | |
| * regarding copyright ownership. The ASF licenses this file | |
| * to you under the Apache License, Version 2.0 (the | |
| * License); you may not use this file except in compliance | |
| * with the License. You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, software | |
| * distributed under the License is distributed on an AS IS BASIS, | |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| * See the License for the specific language governing permissions and | |
| * limitations under the License. | |
| */ | |
| import groovy.json.JsonOutput | |
| import java.util.stream.Collectors | |
| plugins { | |
| id 'org.apache.beam.module' | |
| id 'org.javacc.javacc' | |
| } | |
| applyJavaNature( | |
| generatedClassPatterns: [ | |
| /^org\.apache\.beam\.sdk\.extensions\.sql\.impl\.parser\.impl.*/, | |
| ], | |
| automaticModuleName: 'org.apache.beam.sdk.extensions.sql', | |
| classesTriggerCheckerBugs: [ | |
| // TODO(https://github.com/apache/beam/issues/21068): This currently crashes with checkerframework 3.10.0 | |
| // when compiling :sdks:java:extensions:sql:compileJava with: | |
| // message: class file for com.google.datastore.v1.Entity not found | |
| // ; The Checker Framework crashed. Please report the crash. | |
| // Compilation unit: /usr/local/google/home/lcwik/git/beam/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datastore/DataStoreV1TableProvider.java | |
| // Last visited tree at line 49 column 1: | |
| // @AutoService(TableProvider.class) | |
| // Exception: com.sun.tools.javac.code.Symbol$CompletionFailure: class file for com.google.datastore.v1.Entity not found; com.sun.tools.javac.code.Symbol$CompletionFailure: class file for com.google.datastore.v1.Entity not found | |
| 'DataStoreV1TableProvider': 'TODO(https://github.com/apache/beam/issues/21068): Report the crash if still occurring on newest version', | |
| ], | |
| // javacc generated code produces lint warnings | |
| disableLintWarnings: ['dep-ann', 'rawtypes'], | |
| // Disable SpotBugs due to ASM bytecode analysis issue with BeamCalcRel class | |
| enableSpotbugs: false, | |
| ) | |
| description = "Apache Beam :: SDKs :: Java :: Extensions :: SQL" | |
| ext.summary = "Beam SQL provides a new interface to generate a Beam pipeline from SQL statement" | |
| configurations { | |
| // Create an fmppTask configuration representing the dependencies | |
| // required to define and execute the Ant FMPP task. | |
| // TODO: Migrate to a FMPP plugin once one exists | |
| fmppTask | |
| fmppTemplates | |
| } | |
| def hadoopVersions = [ | |
| "285": "2.8.5", | |
| "292": "2.9.2", | |
| "2102": "2.10.2", | |
| "324": "3.2.4", | |
| ] | |
| hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")} | |
| dependencies { | |
| implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) | |
| // TODO(https://github.com/apache/beam/issues/21156): Determine how to build without this dependency | |
| provided "org.immutables:value:2.8.8" | |
| permitUnusedDeclared "org.immutables:value:2.8.8" | |
| javacc "net.java.dev.javacc:javacc:4.0" | |
| fmppTask "com.googlecode.fmpp-maven-plugin:fmpp-maven-plugin:1.0" | |
| fmppTask "org.freemarker:freemarker:2.3.31" | |
| fmppTemplates library.java.vendored_calcite_1_40_0 | |
| implementation project(path: ":sdks:java:core", configuration: "shadow") | |
| implementation project(":sdks:java:extensions:avro") | |
| implementation project(":sdks:java:extensions:join-library") | |
| permitUnusedDeclared project(":sdks:java:extensions:join-library") // BEAM-11761 | |
| implementation project(":sdks:java:extensions:sql:udf") | |
| implementation project(path: ":runners:direct-java", configuration: "shadow") | |
| implementation library.java.commons_codec | |
| implementation library.java.commons_csv | |
| implementation library.java.jackson_databind | |
| implementation library.java.joda_time | |
| implementation library.java.vendored_calcite_1_40_0 | |
| implementation "org.codehaus.janino:janino:3.0.11" | |
| implementation "org.codehaus.janino:commons-compiler:3.0.11" | |
| implementation library.java.jackson_core | |
| implementation library.java.mongo_java_driver | |
| permitUnusedDeclared library.java.mongo_java_driver | |
| implementation library.java.mongo_bson | |
| implementation library.java.mongodb_driver_core | |
| implementation library.java.slf4j_api | |
| implementation library.java.joda_time | |
| implementation library.java.vendored_guava_32_1_2_jre | |
| provided project(":sdks:java:io:kafka") | |
| implementation project(":sdks:java:extensions:google-cloud-platform-core") | |
| permitUnusedDeclared project(":sdks:java:extensions:google-cloud-platform-core") | |
| implementation project(":sdks:java:io:google-cloud-platform") | |
| implementation library.java.proto_google_cloud_bigtable_v2 | |
| implementation library.java.google_api_services_bigquery | |
| permitUnusedDeclared library.java.google_api_services_bigquery | |
| implementation project(":sdks:java:io:mongodb") | |
| implementation library.java.avro | |
| provided project(":sdks:java:io:parquet") | |
| provided library.java.jackson_dataformat_xml | |
| permitUnusedDeclared library.java.jackson_dataformat_xml | |
| provided library.java.hadoop_client | |
| permitUnusedDeclared library.java.hadoop_client | |
| provided library.java.kafka_clients | |
| testImplementation library.java.vendored_calcite_1_40_0 | |
| testImplementation library.java.vendored_guava_32_1_2_jre | |
| testImplementation library.java.junit | |
| testImplementation library.java.quickcheck_core | |
| testImplementation library.java.testcontainers_kafka | |
| testImplementation library.java.google_cloud_bigtable | |
| testImplementation library.java.google_cloud_bigtable_client_core_config | |
| testImplementation library.java.google_cloud_bigtable_emulator | |
| testImplementation library.java.proto_google_cloud_bigtable_admin_v2 | |
| testImplementation library.java.proto_google_cloud_datastore_v1 | |
| testImplementation library.java.google_cloud_datastore_v1_proto_client | |
| testImplementation library.java.kafka_clients | |
| testImplementation project(":sdks:java:io:kafka") | |
| testImplementation project(path: ":sdks:java:io:mongodb", configuration: "testRuntimeMigration") | |
| testImplementation library.java.mongo_java_driver | |
| testImplementation project(path: ":sdks:java:io:thrift", configuration: "testRuntimeMigration") | |
| testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration") | |
| testCompileOnly project(":sdks:java:extensions:sql:udf-test-provider") | |
| testRuntimeOnly library.java.slf4j_jdk14 | |
| hadoopVersions.each {kv -> | |
| "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-client:$kv.value" | |
| } | |
| } | |
| hadoopVersions.each {kv -> | |
| configurations."hadoopVersion$kv.key" { | |
| resolutionStrategy { | |
| force "org.apache.hadoop:hadoop-client:$kv.value" | |
| } | |
| } | |
| } | |
| // Copy Calcite templates and our own template into the build directory | |
| // so we have one location for the FMPP task to parse. | |
| task copyFmppTemplatesFromSrc(type: Copy) { | |
| from "src/main/codegen" | |
| into "${project.buildDir}/templates-fmpp/codegen" | |
| } | |
| task copyFmppTemplatesFromCalciteCore(type: Copy) { | |
| dependsOn configurations.fmppTemplates | |
| File calciteCoreJar = files(configurations.fmppTemplates.files).filter { | |
| it.name.startsWith("beam-vendor-calcite") | |
| }.singleFile | |
| from zipTree(calciteCoreJar) | |
| include "**/Parser.jj" | |
| into "${project.buildDir}/templates-fmpp" | |
| filter{ | |
| line -> | |
| line.replace('import org.apache.calcite.', 'import org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.') | |
| } | |
| filter{ | |
| line -> | |
| line.replace('import static org.apache.calcite.', 'import static org.apache.beam.vendor.calcite.v1_40_0.org.apache.calcite.') | |
| } | |
| filter{ | |
| line -> | |
| line.replace('import com.google.common.', 'import org.apache.beam.vendor.calcite.v1_40_0.com.google.common.') | |
| } | |
| } | |
| // Generate the FMPP sources from the FMPP templates. | |
| def generateFmppOutputDir = "${project.buildDir}/generated/fmpp" | |
| task generateFmppSources { | |
| dependsOn configurations.fmppTask | |
| dependsOn copyFmppTemplatesFromSrc | |
| dependsOn copyFmppTemplatesFromCalciteCore | |
| doLast { | |
| ant.taskdef(name: "fmpp", classname: "fmpp.tools.AntTask", classpath: configurations.fmppTask.asPath) | |
| ant.fmpp(configuration: "src/main/codegen/config.fmpp", sourceRoot: "${project.buildDir}/templates-fmpp/codegen/templates", outputRoot: generateFmppOutputDir) | |
| } | |
| } | |
| // Match the output directory for generated code with the package, to be more tool-friendly | |
| def generateFmppJavaccRoot = "${generateFmppOutputDir}/javacc" | |
| def generatedJavaccSourceDir = "${project.buildDir}/generated/javacc" | |
| def generatedJavaccPackageDir = "${generatedJavaccSourceDir}/org/apache/beam/sdk/extensions/sql/impl/parser/impl" | |
| compileJavacc { | |
| dependsOn generateFmppSources | |
| inputDirectory = file(generateFmppJavaccRoot) | |
| outputDirectory = file(generatedJavaccPackageDir) | |
| arguments = [static: "false", lookahead: "2"] | |
| } | |
| // Help IntelliJ find the fmpp bits | |
| idea { | |
| module { | |
| sourceDirs += file(generateFmppOutputDir) | |
| generatedSourceDirs += file(generateFmppOutputDir) | |
| sourceDirs += file(generatedJavaccSourceDir) | |
| generatedSourceDirs += file(generatedJavaccSourceDir) | |
| } | |
| } | |
| // Run basic SQL example | |
| task runBasicExample(type: JavaExec) { | |
| description = "Run basic SQL example" | |
| mainClass = "org.apache.beam.sdk.extensions.sql.example.BeamSqlExample" | |
| classpath = sourceSets.main.runtimeClasspath | |
| args = ["--runner=DirectRunner"] | |
| } | |
| // Run SQL example on POJO inputs | |
| task runPojoExample(type: JavaExec) { | |
| description = "Run SQL example for PCollections of POJOs" | |
| mainClass = "org.apache.beam.sdk.extensions.sql.example.BeamSqlPojoExample" | |
| classpath = sourceSets.main.runtimeClasspath | |
| args = ["--runner=DirectRunner"] | |
| } | |
| task integrationTest(type: Test) { | |
| def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing' | |
| def gcsTempRoot = project.findProperty('gcsTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests/' | |
| // Disable Gradle cache (it should not be used because the IT's won't run). | |
| outputs.upToDateWhen { false } | |
| def pipelineOptions = [ | |
| "--project=${gcpProject}", | |
| "--tempLocation=${gcsTempRoot}", | |
| "--blockOnRun=false"] | |
| systemProperty "beamTestPipelineOptions", JsonOutput.toJson(pipelineOptions) | |
| include '**/*IT.class' | |
| maxParallelForks 4 | |
| classpath = project(":sdks:java:extensions:sql") | |
| .sourceSets | |
| .test | |
| .runtimeClasspath | |
| testClassesDirs = files(project(":sdks:java:extensions:sql").sourceSets.test.output.classesDirs) | |
| useJUnit { } | |
| } | |
| task preCommit { | |
| dependsOn build | |
| dependsOn runBasicExample | |
| dependsOn runPojoExample | |
| if (project.hasProperty("testJavaVersion")) { | |
| var testVer = project.property("testJavaVersion") | |
| dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer") | |
| } | |
| } | |
| task postCommit { | |
| group = "Verification" | |
| description = "Various integration tests" | |
| dependsOn integrationTest | |
| } | |
| task emptyJar(type: Jar) { | |
| archiveBaseName = "${project.archivesBaseName}-empty-jar" | |
| from fileTree(dir: getTemporaryDir().createNewFile().toString()) | |
| } | |
| task hadoopVersionsTest(group: "Verification") { | |
| description = "Runs SQL tests with different Hadoop versions" | |
| def taskNames = hadoopVersions.keySet().stream() | |
| .map{num -> "hadoopVersion${num}Test"} | |
| .collect(Collectors.toList()) | |
| dependsOn taskNames | |
| } | |
| hadoopVersions.each { kv -> | |
| task "hadoopVersion${kv.key}Test"(type: Test, group: "Verification") { | |
| description = "Runs SQL tests with Hadoop version $kv.value" | |
| classpath = configurations."hadoopVersion$kv.key" + sourceSets.test.runtimeClasspath | |
| include '**/*Test.class' | |
| dependsOn emptyJar | |
| // Pass jars used by Java UDF tests via system properties. | |
| evaluationDependsOn(":sdks:java:extensions:sql:udf-test-provider") // Needed to resolve jarPath. | |
| systemProperty "beam.sql.udf.test.jar_path", project(":sdks:java:extensions:sql:udf-test-provider").jarPath | |
| systemProperty "beam.sql.udf.test.empty_jar_path", emptyJar.archivePath | |
| } | |
| } | |
| test { | |
| dependsOn emptyJar | |
| // Pass jars used by Java UDF tests via system properties. | |
| evaluationDependsOn(":sdks:java:extensions:sql:udf-test-provider") // Needed to resolve jarPath. | |
| systemProperty "beam.sql.udf.test.jar_path", project(":sdks:java:extensions:sql:udf-test-provider").jarPath | |
| systemProperty "beam.sql.udf.test.empty_jar_path", emptyJar.archivePath | |
| } |