runners/jet/build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import groovy.json.JsonOutput

plugins { id 'org.apache.beam.module' }
applyJavaNature(
  automaticModuleName: 'org.apache.beam.runners.jet',
)

description = "Apache Beam :: Runners :: Hazelcast Jet"

evaluationDependsOn(":sdks:java:core")
evaluationDependsOn(":runners:core-java")
evaluationDependsOn(":runners:core-construction-java")

project.ext {
    jet_version = '4.0'
    hazelcast_version = '4.0'
}

configurations {
    needsRunner
    validatesRunner
}

dependencies {
    compile project(path: ":sdks:java:core", configuration: "shadow")
    compile project(":runners:core-java")
    compile project(path: ":model:pipeline", configuration: "shadow")
    compile project(":runners:core-construction-java")
    compile "com.hazelcast.jet:hazelcast-jet:$jet_version"
    compile library.java.joda_time
    compile library.java.vendored_guava_26_0_jre
    compile library.java.slf4j_api

    testCompile project(path: ":sdks:java:core", configuration: "shadowTest")
    testCompile project(path: ":runners:core-java", configuration: "testRuntime")
    testCompile project(path: ":runners:core-construction-java", configuration: "testRuntime")
    testCompile library.java.joda_time
    testCompile "com.hazelcast.jet:hazelcast-jet-core:$jet_version:tests"
    testCompile "com.hazelcast:hazelcast:$hazelcast_version:tests"

    needsRunner project(path: ":sdks:java:core", configuration: "shadowTest")
    needsRunner project(path: ":runners:core-java", configuration: "testRuntime")
    needsRunner project(path: ":runners:core-construction-java", configuration: "testRuntime")
    needsRunner project(path: project.path, configuration: "testRuntime")

    validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
    validatesRunner project(path: ":runners:core-java", configuration: "testRuntime")
    validatesRunner project(path: ":runners:core-construction-java", configuration: "testRuntime")
    validatesRunner project(path: project.path, configuration: "testRuntime")
}

task validatesRunnerBatch(type: Test) {
    group = "Verification"
    systemProperty "beamTestPipelineOptions", JsonOutput.toJson(["--runner=TestJetRunner"])

    classpath = configurations.validatesRunner
    testClassesDirs = files(project(":sdks:java:core").sourceSets.test.output.classesDirs)
    useJUnit {
        includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
        excludeCategories "org.apache.beam.sdk.testing.LargeKeys\$Above100MB"
        excludeCategories 'org.apache.beam.sdk.testing.UsesTimerMap'
        excludeCategories 'org.apache.beam.sdk.testing.UsesOnWindowExpiration'
        excludeCategories 'org.apache.beam.sdk.testing.UsesOrderedListState'
        excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer'
        excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo'

        //Jet Runner doesn't current support @RequiresTimeSortedInput annotation.
        excludeCategories 'org.apache.beam.sdk.testing.UsesRequiresTimeSortedInput'

        //Event type not supported in TestStream: class org.apache.beam.sdk.testing.AutoValue_TestStream_ProcessingTimeEvent
        excludeCategories 'org.apache.beam.sdk.testing.UsesTestStreamWithProcessingTime'
    }
    filter {
        // There is an existing bug caused by Jet assuming that outputs emitted are Object.equals comparable which is not true for byte[]
        excludeTestsMatching 'org.apache.beam.sdk.transforms.SplittableDoFnTest.testOutputAfterCheckpointBounded'

        // There is an existing bug where we are trying to add additional outputs during flushing and don't restore state correctly
        excludeTestsMatching 'org.apache.beam.sdk.transforms.SplittableDoFnTest.testWindowedSideInputWithCheckpointsBounded'

        // unbounded streams created from bounded sources not supported by Jet Runner
        excludeTestsMatching 'org.apache.beam.sdk.transforms.ParDoTest$TimerTests.testTwoTimersSettingEachOtherWithCreateAsInputUnbounded'

        // timer output timestamps not supported by Jet Runner
        excludeTestsMatching 'org.apache.beam.sdk.transforms.ParDoTest$TimerTests.testOutputTimestamp'

        excludeTestsMatching 'org.apache.beam.sdk.testing.PAssertTest.testAssertionSiteIsCaptured*'
    }

    maxHeapSize = '4g'
}

task validatesRunner {
    group = "Verification"
    description "Validates Jet Runner"
    dependsOn validatesRunnerBatch
}

task needsRunnerTests(type: Test) {
    group = "Verification"
    description = "Runs tests that require a runner to validate that piplines/transforms work correctly"
    systemProperty "beamTestPipelineOptions", JsonOutput.toJson(["--runner=TestJetRunner"])

    classpath = configurations.needsRunner
    testClassesDirs += files(project(":runners:core-construction-java").sourceSets.test.output.classesDirs)
    testClassesDirs += files(project(":runners:core-java").sourceSets.test.output.classesDirs)
    testClassesDirs += files(project(":sdks:java:core").sourceSets.test.output.classesDirs)
    useJUnit {
        includeCategories "org.apache.beam.sdk.testing.NeedsRunner"
        excludeCategories "org.apache.beam.sdk.testing.LargeKeys\$Above100MB"
        excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer'
    }
}

task needsRunner {
    group = "Verification"
    description "Runs lower level tests with the Jet Runner"
    dependsOn needsRunnerTests
}