runners/spark/4/build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

def basePath = '..'
/* All properties required for loading the Spark build script */
project.ext {
  spark_major = '4'
  // Spark 4 version as defined in BeamModulePlugin; requires Scala 2.13 and Java 17
  spark_version = spark4_version
  spark_scala_version = '2.13'
  archives_base_name = 'beam-runners-spark-4'
}

// Load the main build script which contains all build logic.
// spark_runner.gradle handles the per-version source-overrides Copy:
// shared base (runners/spark/src/) + previous majors + this module's ./src/ are
// merged into build/source-overrides/src using DuplicatesStrategy.INCLUDE so the
// 11 files under runners/spark/4/src/.../structuredstreaming/ override the
// shared-base versions.
apply from: "$basePath/spark_runner.gradle"

// Spark 4 always requires Java 17, so unconditionally add the --add-opens flags
// required by Kryo and other libraries that use reflection on JDK internals.
tasks.withType(Test).configureEach {
  jvmArgs "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
          "--add-opens=java.base/java.nio=ALL-UNNAMED",
          "--add-opens=java.base/java.util=ALL-UNNAMED",
          "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED"
}

tasks.validatesStructuredStreamingRunnerBatch {
  filter {
    // TODO(https://github.com/apache/beam/issues/36841): currently failing with INTERNAL_ERROR]
    //  Couldn't find method fromByteArray with arguments
    excludeTestsMatching 'org.apache.beam.sdk.transforms.CombineTest$BasicTests.testHotKeyCombining'
  }
}

tasks.validatesRunner.dependsOn(validatesStructuredStreamingRunnerBatch)

// Exclude DStream-based streaming tests from the shared-base copy: the Spark 4 module
// supports only structured streaming (batch) and does not include legacy DStream support.
// Streaming test utilities also depend on kafka.server.KafkaServerStartable which was
// removed in Kafka 2.8.0 (the first Kafka version with a _2.13 artifact).
tasks.named("copyTestSourceOverrides") {
  exclude "**/translation/streaming/**"
}