| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * License); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an AS IS BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| import groovy.json.JsonOutput |
| import java.util.stream.Collectors |
| |
| plugins { |
| id 'org.apache.beam.module' |
| id 'org.javacc.javacc' |
| } |
| applyJavaNature( |
| generatedClassPatterns: [ |
| /^org\.apache\.beam\.sdk\.extensions\.sql\.impl\.parser\.impl.*/, |
| ], |
| automaticModuleName: 'org.apache.beam.sdk.extensions.sql', |
| classesTriggerCheckerBugs: [ |
| // TODO(https://github.com/apache/beam/issues/21068): This currently crashes with checkerframework 3.10.0 |
| // when compiling :sdks:java:extensions:sql:compileJava with: |
| // message: class file for com.google.datastore.v1.Entity not found |
| // ; The Checker Framework crashed. Please report the crash. |
| // Compilation unit: /usr/local/google/home/lcwik/git/beam/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datastore/DataStoreV1TableProvider.java |
| // Last visited tree at line 49 column 1: |
| // @AutoService(TableProvider.class) |
| // Exception: com.sun.tools.javac.code.Symbol$CompletionFailure: class file for com.google.datastore.v1.Entity not found; com.sun.tools.javac.code.Symbol$CompletionFailure: class file for com.google.datastore.v1.Entity not found |
| 'DataStoreV1TableProvider': 'TODO(https://github.com/apache/beam/issues/21068): Report the crash if still occurring on newest version', |
| ], |
| // javacc generated code produces lint warnings |
| disableLintWarnings: ['dep-ann', 'rawtypes'], |
| ) |
| |
| description = "Apache Beam :: SDKs :: Java :: Extensions :: SQL" |
| ext.summary = "Beam SQL provides a new interface to generate a Beam pipeline from SQL statement" |
| |
| configurations { |
| // Create an fmppTask configuration representing the dependencies |
| // required to define and execute the Ant FMPP task. |
| // TODO: Migrate to a FMPP plugin once one exists |
| fmppTask |
| fmppTemplates |
| } |
| |
| def hadoopVersions = [ |
| "285": "2.8.5", |
| "292": "2.9.2", |
| "2102": "2.10.2", |
| "324": "3.2.4", |
| ] |
| |
| hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")} |
| |
| dependencies { |
| implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) |
| |
| // TODO(https://github.com/apache/beam/issues/21156): Determine how to build without this dependency |
| provided "org.immutables:value:2.8.8" |
| permitUnusedDeclared "org.immutables:value:2.8.8" |
| javacc "net.java.dev.javacc:javacc:4.0" |
| fmppTask "com.googlecode.fmpp-maven-plugin:fmpp-maven-plugin:1.0" |
| fmppTask "org.freemarker:freemarker:2.3.31" |
| fmppTemplates library.java.vendored_calcite_1_28_0 |
| implementation project(path: ":sdks:java:core", configuration: "shadow") |
| implementation project(":sdks:java:extensions:avro") |
| implementation project(":sdks:java:extensions:join-library") |
| permitUnusedDeclared project(":sdks:java:extensions:join-library") // BEAM-11761 |
| implementation project(":sdks:java:extensions:sql:udf") |
| implementation project(path: ":runners:direct-java", configuration: "shadow") |
| implementation library.java.commons_codec |
| implementation library.java.commons_csv |
| implementation library.java.jackson_databind |
| implementation library.java.joda_time |
| implementation library.java.vendored_calcite_1_28_0 |
| implementation "org.codehaus.janino:janino:3.0.11" |
| implementation "org.codehaus.janino:commons-compiler:3.0.11" |
| implementation library.java.jackson_core |
| implementation library.java.mongo_java_driver |
| implementation library.java.slf4j_api |
| implementation library.java.joda_time |
| implementation library.java.vendored_guava_32_1_2_jre |
| provided project(":sdks:java:io:kafka") |
| implementation project(":sdks:java:extensions:google-cloud-platform-core") |
| permitUnusedDeclared project(":sdks:java:extensions:google-cloud-platform-core") |
| implementation project(":sdks:java:io:google-cloud-platform") |
| implementation library.java.proto_google_cloud_bigtable_v2 |
| implementation library.java.google_api_services_bigquery |
| permitUnusedDeclared library.java.google_api_services_bigquery |
| implementation library.java.proto_google_cloud_pubsublite_v1 |
| implementation library.java.google_cloud_pubsublite |
| implementation project(":sdks:java:io:mongodb") |
| implementation library.java.avro |
| implementation library.java.protobuf_java |
| implementation library.java.protobuf_java_util |
| provided project(":sdks:java:io:parquet") |
| provided library.java.jackson_dataformat_xml |
| permitUnusedDeclared library.java.jackson_dataformat_xml |
| provided library.java.hadoop_client |
| permitUnusedDeclared library.java.hadoop_client |
| provided library.java.kafka_clients |
| testImplementation library.java.vendored_calcite_1_28_0 |
| testImplementation library.java.vendored_guava_32_1_2_jre |
| testImplementation library.java.junit |
| testImplementation library.java.quickcheck_core |
| testImplementation library.java.testcontainers_kafka |
| testImplementation library.java.google_cloud_bigtable |
| testImplementation library.java.google_cloud_bigtable_client_core_config |
| testImplementation library.java.google_cloud_bigtable_emulator |
| testImplementation library.java.proto_google_cloud_bigtable_admin_v2 |
| testImplementation library.java.proto_google_cloud_datastore_v1 |
| testImplementation library.java.google_cloud_datastore_v1_proto_client |
| testImplementation library.java.kafka_clients |
| testImplementation project(":sdks:java:io:kafka") |
| testImplementation project(path: ":sdks:java:io:mongodb", configuration: "testRuntimeMigration") |
| testImplementation project(path: ":sdks:java:io:thrift", configuration: "testRuntimeMigration") |
| testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration") |
| testCompileOnly project(":sdks:java:extensions:sql:udf-test-provider") |
| testRuntimeOnly library.java.slf4j_jdk14 |
| hadoopVersions.each {kv -> |
| "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-client:$kv.value" |
| } |
| } |
| |
| hadoopVersions.each {kv -> |
| configurations."hadoopVersion$kv.key" { |
| resolutionStrategy { |
| force "org.apache.hadoop:hadoop-client:$kv.value" |
| } |
| } |
| } |
| |
| // Copy Calcite templates and our own template into the build directory |
| // so we have one location for the FMPP task to parse. |
| task copyFmppTemplatesFromSrc(type: Copy) { |
| from "src/main/codegen" |
| into "${project.buildDir}/templates-fmpp/codegen" |
| } |
| task copyFmppTemplatesFromCalciteCore(type: Copy) { |
| dependsOn configurations.fmppTemplates |
| File calciteCoreJar = files(configurations.fmppTemplates.files).filter { |
| it.name.startsWith("beam-vendor-calcite") |
| }.singleFile |
| from zipTree(calciteCoreJar) |
| include "**/Parser.jj" |
| into "${project.buildDir}/templates-fmpp" |
| filter{ |
| line -> |
| line.replace('import org.apache.calcite.', 'import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.') |
| } |
| filter{ |
| line -> |
| line.replace('import static org.apache.calcite.', 'import static org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.') |
| } |
| } |
| |
| // Generate the FMPP sources from the FMPP templates. |
| def generateFmppOutputDir = "${project.buildDir}/generated/fmpp" |
| task generateFmppSources { |
| dependsOn configurations.fmppTask |
| dependsOn copyFmppTemplatesFromSrc |
| dependsOn copyFmppTemplatesFromCalciteCore |
| doLast { |
| ant.taskdef(name: "fmpp", classname: "fmpp.tools.AntTask", classpath: configurations.fmppTask.asPath) |
| ant.fmpp(configuration: "src/main/codegen/config.fmpp", sourceRoot: "${project.buildDir}/templates-fmpp/codegen/templates", outputRoot: generateFmppOutputDir) |
| } |
| } |
| |
| // Match the output directory for generated code with the package, to be more tool-friendly |
| def generateFmppJavaccRoot = "${generateFmppOutputDir}/javacc" |
| def generatedJavaccSourceDir = "${project.buildDir}/generated/javacc" |
| def generatedJavaccPackageDir = "${generatedJavaccSourceDir}/org/apache/beam/sdk/extensions/sql/impl/parser/impl" |
| compileJavacc { |
| dependsOn generateFmppSources |
| inputDirectory = file(generateFmppJavaccRoot) |
| outputDirectory = file(generatedJavaccPackageDir) |
| arguments = [static: "false", lookahead: "2"] |
| } |
| |
| // Help IntelliJ find the fmpp bits |
| idea { |
| module { |
| sourceDirs += file(generateFmppOutputDir) |
| generatedSourceDirs += file(generateFmppOutputDir) |
| |
| sourceDirs += file(generatedJavaccSourceDir) |
| generatedSourceDirs += file(generatedJavaccSourceDir) |
| } |
| } |
| |
| // Run basic SQL example |
| task runBasicExample(type: JavaExec) { |
| description = "Run basic SQL example" |
| mainClass = "org.apache.beam.sdk.extensions.sql.example.BeamSqlExample" |
| classpath = sourceSets.main.runtimeClasspath |
| args = ["--runner=DirectRunner"] |
| } |
| |
| // Run SQL example on POJO inputs |
| task runPojoExample(type: JavaExec) { |
| description = "Run SQL example for PCollections of POJOs" |
| mainClass = "org.apache.beam.sdk.extensions.sql.example.BeamSqlPojoExample" |
| classpath = sourceSets.main.runtimeClasspath |
| args = ["--runner=DirectRunner"] |
| } |
| |
| task integrationTest(type: Test) { |
| def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing' |
| def gcsTempRoot = project.findProperty('gcsTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests/' |
| |
| // Disable Gradle cache (it should not be used because the IT's won't run). |
| outputs.upToDateWhen { false } |
| |
| def pipelineOptions = [ |
| "--project=${gcpProject}", |
| "--tempLocation=${gcsTempRoot}", |
| "--blockOnRun=false"] |
| |
| systemProperty "beamTestPipelineOptions", JsonOutput.toJson(pipelineOptions) |
| |
| include '**/*IT.class' |
| |
| maxParallelForks 4 |
| classpath = project(":sdks:java:extensions:sql") |
| .sourceSets |
| .test |
| .runtimeClasspath |
| testClassesDirs = files(project(":sdks:java:extensions:sql").sourceSets.test.output.classesDirs) |
| useJUnit { } |
| } |
| |
| task preCommit { |
| dependsOn build |
| dependsOn runBasicExample |
| dependsOn runPojoExample |
| |
| if (project.hasProperty("testJavaVersion")) { |
| var testVer = project.property("testJavaVersion") |
| dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer") |
| } |
| } |
| |
| task postCommit { |
| group = "Verification" |
| description = "Various integration tests" |
| dependsOn integrationTest |
| } |
| |
| task emptyJar(type: Jar) { |
| archiveBaseName = "${project.archivesBaseName}-empty-jar" |
| from fileTree(dir: getTemporaryDir().createNewFile().toString()) |
| } |
| |
| task hadoopVersionsTest(group: "Verification") { |
| description = "Runs SQL tests with different Hadoop versions" |
| def taskNames = hadoopVersions.keySet().stream() |
| .map{num -> "hadoopVersion${num}Test"} |
| .collect(Collectors.toList()) |
| dependsOn taskNames |
| } |
| |
| hadoopVersions.each { kv -> |
| task "hadoopVersion${kv.key}Test"(type: Test, group: "Verification") { |
| description = "Runs SQL tests with Hadoop version $kv.value" |
| classpath = configurations."hadoopVersion$kv.key" + sourceSets.test.runtimeClasspath |
| include '**/*Test.class' |
| dependsOn emptyJar |
| // Pass jars used by Java UDF tests via system properties. |
| evaluationDependsOn(":sdks:java:extensions:sql:udf-test-provider") // Needed to resolve jarPath. |
| systemProperty "beam.sql.udf.test.jar_path", project(":sdks:java:extensions:sql:udf-test-provider").jarPath |
| systemProperty "beam.sql.udf.test.empty_jar_path", emptyJar.archivePath |
| } |
| } |
| |
| test { |
| dependsOn emptyJar |
| // Pass jars used by Java UDF tests via system properties. |
| evaluationDependsOn(":sdks:java:extensions:sql:udf-test-provider") // Needed to resolve jarPath. |
| systemProperty "beam.sql.udf.test.jar_path", project(":sdks:java:extensions:sql:udf-test-provider").jarPath |
| systemProperty "beam.sql.udf.test.empty_jar_path", emptyJar.archivePath |
| } |
