it/google-cloud-platform/build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import org.apache.beam.gradle.IoPerformanceTestUtilities

plugins { id 'org.apache.beam.module' }
applyJavaNature(
    automaticModuleName: 'org.apache.beam.it.gcp',
    exportJavadoc: false,
)

description = "Apache Beam :: IT :: Google Cloud Platform"
ext.summary = "Integration test utilities for Google Cloud Platform."

evaluationDependsOn(":it:common")

dependencies {
    implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
    implementation project(path: ":it:common")
    implementation project(path: ":it:conditions")
    implementation project(path: ":it:truthmatchers")
    implementation library.java.failsafe
    implementation library.java.google_api_services_dataflow
    implementation library.java.junit
    implementation library.java.slf4j_api
    implementation library.java.vendored_guava_32_1_2_jre
    implementation library.java.jackson_core
    implementation library.java.jackson_databind
    implementation 'org.apache.commons:commons-lang3:3.9'
    implementation library.java.gax
    implementation library.java.google_api_common
    implementation library.java.google_api_client
    implementation library.java.google_auth_library_credentials
    implementation library.java.google_auth_library_oauth2_http
    implementation library.java.guava
    implementation library.java.protobuf_java
    implementation library.java.threetenbp
    implementation 'org.awaitility:awaitility:4.2.0'
    // Google Cloud Dependencies
    implementation library.java.google_cloud_core
    implementation 'com.google.cloud:google-cloud-bigquery'
    implementation 'com.google.cloud:google-cloud-bigtable'
    implementation 'com.google.cloud:google-cloud-spanner'
    implementation 'com.google.cloud:google-cloud-pubsub'
    provided 'com.google.api.grpc:proto-google-cloud-pubsub-v1'
    implementation 'com.google.cloud:google-cloud-datastore'
    implementation 'com.google.cloud:google-cloud-datastream'
    provided 'com.google.api.grpc:proto-google-cloud-datastream-v1'
    implementation 'com.google.cloud:google-cloud-kms'
    provided 'com.google.api.grpc:proto-google-cloud-kms-v1'
    implementation 'com.google.cloud:google-cloud-dlp'
    provided 'com.google.api.grpc:proto-google-cloud-dlp-v2'
    implementation 'com.google.cloud:google-cloud-secretmanager'
    provided 'com.google.api.grpc:proto-google-cloud-secretmanager-v1'

    testImplementation project(path: ":it:common", configuration: "testRuntimeMigration")
    testImplementation project(path: ":sdks:java:io:google-cloud-platform")
    testImplementation project(path: ":sdks:java:io:parquet", configuration: "testRuntimeMigration")
    // CompressionCodecName is needed to configure the ParquetIO sink. parquet-hadoop is only an
    // "implementation" dependency of :sdks:java:io:parquet, so it is not exposed transitively.
    // Keep the version in sync with sdks/java/io/parquet/build.gradle.
    testImplementation "org.apache.parquet:parquet-hadoop:1.15.2"
    testImplementation project(path: ":sdks:java:extensions:avro", configuration: "testRuntimeMigration")
    testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration")
    testImplementation project(path: ":sdks:java:io:synthetic")
    testImplementation library.java.mockito_inline
    testImplementation project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "testRuntimeMigration")
    testImplementation project(path: ":runners:direct-java", configuration: "shadow")
    testRuntimeOnly library.java.slf4j_simple
}

tasks.register(
        "GCSPerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'TextIOLT',
        ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
// Note: the task always passes 'configuration' down as a system property, so the "local" default in
// ParquetIOLT itself is never reached through gradle. The default here is therefore the cheap local
// preset: an accidental `./gradlew :it:google-cloud-platform:ParquetPerformanceTest` must not launch
// a full scale Dataflow job. Pass an explicit preset (or a json override) for real runs, e.g.
// -Dconfiguration=large or -Dconfiguration='{"preset":"f100_s16","totalBytes":"10GB"}'.
tasks.register(
        "ParquetPerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'ParquetIOLT',
        ['configuration':'local','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
tasks.named("ParquetPerformanceTest") {
    // A DirectRunner configuration runs the pipeline inside this JVM, and the DirectRunner clones
    // every element to check that DoFns do not mutate their input. With wide records that doubles
    // the live set, so the default test heap is not enough. Dataflow runs are unaffected, their
    // work happens on the workers.
    maxHeapSize = '8g'
    // Metrics are printed to stdout (not exported), so make sure they show up in the console.
    testLogging {
        showStandardStreams = true
    }
}
tasks.register(
        "BigTablePerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigTableIOLT',
        ['configuration':'large', 'project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
tasks.register(
        "BigQueryPerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigQueryIOLT',
        ['configuration':'medium', 'project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
tasks.register(
        "BigQueryStressTestMedium", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigQueryIOST',
        ['configuration':'medium', 'project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
tasks.register(
        "BigQueryStressTestLarge", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigQueryIOST',
        ['configuration':'large', 'project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
tasks.register(
        "BigQueryStorageApiStreamingPerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigQueryStreamingLT',
        ['configuration':'large', 'project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
tasks.register(
        "PubsubLoadTestMedium", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'PubsubIOLT',
        ['configuration':'medium', 'project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
tasks.register(
        "PubsubLoadTestLarge", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'PubsubIOLT',
        ['configuration':'large', 'project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
                + System.properties
)
tasks.register(
        "WordCountIntegrationTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'WordCountIT',
        ['project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
tasks.register(
        "BigTableStressTestMedium", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigTableIOST',
        ['configuration':'medium','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
tasks.register(
        "BigTableStressTestLarge", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigTableIOST',
        ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
tasks.register(
        "SpannerStressTestMedium", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'SpannerIOST',
        ['configuration':'medium','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
tasks.register(
        "SpannerStressTestLarge", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'SpannerIOST',
        ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties
)
tasks.register(
        "PubSubStressTestMedium", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'PubSubIOST',
        ['configuration':'medium','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties)
tasks.register(
        "PubSubStressTestLarge", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'PubSubIOST',
        ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
        + System.properties)