build.gradle.kts

import java.util.TreeMap

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

plugins {
  base
  // Apply one top level rat plugin to perform any required license enforcement analysis
  id("org.nosphere.apache.rat") version "0.11.0"
  id("org.apache.beam.module")
  id("org.sonarqube") version "3.0"
}

/*************************************************************************************************/
// Configure the root project

tasks.rat {
  // Set input directory to that of the root project instead of the CWD. This
  // makes .gitignore rules (added below) work properly.
  inputDir.set(project.rootDir)

  val exclusions = mutableListOf(
    // Ignore files we track but do not distribute
    "**/.github/**/*",
    "**/.gitkeep",
    "gradlew",
    "gradlew.bat",
    "gradle/wrapper/gradle-wrapper.properties",

    "**/package-list",
    "**/test.avsc",
    "**/logical-types.avsc",
    "**/user.avsc",
    "**/*.avro",
    "**/test/resources/**/*.txt",
    "**/test/resources/**/*.csv",
    "**/test/**/.placeholder",

    // Default eclipse excludes neglect subprojects

    // Proto/grpc generated wrappers
    "**/apache_beam/portability/api/**/*_pb2*.py",
    "**/go/pkg/beam/**/*.pb.go",
    "**/mock-apis/**/*.pb.go",

    // Ignore go.sum files, which don't permit headers
    "**/go.sum",

    // Ignore Go test data files
    "**/go/data/**",

    // VCF test files
    "**/apache_beam/testing/data/vcf/*",

    // JDBC package config files
    "**/META-INF/services/java.sql.Driver",

    // Website build files
    "**/Gemfile.lock",
    "**/Rakefile",
    "**/.htaccess",
    "website/www/site/assets/css/**/*",
    "website/www/site/assets/scss/_bootstrap.scss",
    "website/www/site/assets/scss/bootstrap/**/*",
    "website/www/site/assets/js/**/*",
    "website/www/site/static/images/mascot/*.ai",
    "website/www/site/static/js/bootstrap*.js",
    "website/www/site/static/js/bootstrap/**/*",
    "website/www/site/themes",
    "website/www/yarn.lock",
    "website/www/package.json",
    "website/www/site/static/js/hero/lottie-light.min.js",
    "website/www/site/static/js/keen-slider.min.js",
    "website/www/site/assets/scss/_keen-slider.scss",

    // Release automation files
    "release/src/main/scripts/*.txt",

    // Ignore ownership files
    "ownership/**/*",
    "**/OWNERS",

    // Ignore CPython LICENSE file
    "LICENSE.python",

    // Ignore vendored cloudpickle files
    "sdks/python/apache_beam/internal/cloudpickle/**",
    "LICENCE.cloudpickle",

    // Json doesn't support comments.
    "**/*.json",

    // Katas files
    "learning/katas/**/course-info.yaml",
    "learning/katas/**/task-info.yaml",
    "learning/katas/**/course-remote-info.yaml",
    "learning/katas/**/section-remote-info.yaml",
    "learning/katas/**/lesson-remote-info.yaml",
    "learning/katas/**/task-remote-info.yaml",
    "learning/katas/**/*.txt",

    // Tour Of Beam learning-content metadata and its samples
    "learning/tour-of-beam/**/content-info.yaml",
    "learning/tour-of-beam/**/module-info.yaml",
    "learning/tour-of-beam/**/group-info.yaml",
    "learning/tour-of-beam/**/unit-info.yaml",
    "learning/tour-of-beam/backend/samples/**/*.md",

    // Tour Of Beam example logs
    "learning/tour-of-beam/learning-content/**/*.log",

    // Tour Of Beam example txt files
    "learning/tour-of-beam/learning-content/**/*.txt",

    // Tour Of Beam example csv files
    "learning/tour-of-beam/learning-content/**/*.csv",

    // Tour Of Beam backend autogenerated Datastore indexes
    "learning/tour-of-beam/backend/internal/storage/index.yaml",

    // Tour Of Beam backend autogenerated Playground GRPC API stubs and mocks
    "learning/tour-of-beam/backend/playground_api/api/v1/api.pb.go",
    "learning/tour-of-beam/backend/playground_api/api/v1/api_grpc.pb.go",
    "learning/tour-of-beam/backend/playground_api/api/v1/mock.go",

    // Playground backend autogenerated GRPC API stubs and mocks
    "playground/backend/internal/api/v1/api.pb.go",
    "playground/backend/internal/api/v1/api_grpc.pb.go",

    // Playground infrastructure autogenerated GRPC API stubs and mocks
    "playground/infrastructure/api/v1/api_pb2.py",
    "playground/infrastructure/api/v1/api_pb2.pyi",
    "playground/infrastructure/api/v1/api_pb2_grpc.py",

    // test p8 file for SnowflakeIO
    "sdks/java/io/snowflake/src/test/resources/invalid_test_rsa_key.p8",
    "sdks/java/io/snowflake/src/test/resources/valid_encrypted_test_rsa_key.p8",
    "sdks/java/io/snowflake/src/test/resources/valid_unencrypted_test_rsa_key.p8",

    // Mockito extensions
    "sdks/java/io/amazon-web-services2/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",
    "sdks/java/io/azure/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",
    "sdks/java/extensions/ml/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",

    // JupyterLab extensions
    "sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/yarn.lock",

    // Autogenerated apitools clients.
    "sdks/python/apache_beam/runners/dataflow/internal/clients/*/**/*.py",

    // Sample text file for Java quickstart
    "sdks/java/maven-archetypes/examples/sample.txt",

    // Ignore Flutter autogenerated files for Playground
    "playground/frontend/**/*.g.dart",
    "playground/frontend/**/*.g.yaml",
    "playground/frontend/**/*.gen.dart",
    "playground/frontend/**/*.golden.yaml",
    "playground/frontend/**/*.mocks.dart",
    "playground/frontend/.metadata",
    "playground/frontend/pubspec.lock",

    // Ignore Flutter autogenerated files for Playground Components
    "playground/frontend/**/*.pb.dart",
    "playground/frontend/**/*.pbenum.dart",
    "playground/frontend/**/*.pbgrpc.dart",
    "playground/frontend/**/*.pbjson.dart",
    "playground/frontend/playground_components/.metadata",
    "playground/frontend/playground_components/pubspec.lock",

    // Ignore Flutter autogenerated files for Tour of Beam
    "learning/tour-of-beam/frontend/**/*.g.dart",
    "learning/tour-of-beam/frontend/**/*.gen.dart",
    "learning/tour-of-beam/frontend/.metadata",
    "learning/tour-of-beam/frontend/pubspec.lock",
    "learning/tour-of-beam/frontend/lib/firebase_options.dart",

    // Ignore .gitkeep file
    "**/.gitkeep",

    // Ignore Flutter localization .arb files (doesn't support comments)
    "playground/frontend/lib/l10n/**/*.arb",

    // Ignore LICENSES copied onto containers
    "sdks/java/container/license_scripts/manual_licenses",
    "sdks/python/container/license_scripts/manual_licenses",

    // Ignore third-party notices bundled in resources
    "**/resources/NOTICES",

    // Ignore autogenrated proto files.
    "sdks/typescript/src/apache_beam/proto/**/*.ts",

    // Ignore typesciript package management.
    "sdks/typescript/package-lock.json",
    "sdks/typescript/node_modules/**/*",

    // Ignore buf autogenerated files.
    "**/buf.lock",

    // Ignore poetry autogenerated files.
    "**/poetry.lock",

    // DuetAI training prompts
    "learning/prompts/**/*.md",

    // Ignore terraform lock files
    "**/.terraform.lock.hcl",

    // Ignore pytest cache files
    "**/.pytest_cache/**"
  )

  // Add .gitignore excludes to the Apache Rat exclusion list. We re-create the behavior
  // of the Apache Maven Rat plugin since the Apache Ant Rat plugin doesn't do this
  // automatically.
  val gitIgnore = project(":").file(".gitignore")
  if (gitIgnore.exists()) {
    val gitIgnoreExcludes = gitIgnore.readLines().filter { it.isNotEmpty() && !it.startsWith("#") }
    exclusions.addAll(gitIgnoreExcludes)
  }

  verbose.set(true)
  failOnError.set(true)
  setExcludes(exclusions)
}
tasks.check.get().dependsOn(tasks.rat)

// Define root pre/post commit tasks simplifying what is needed
// to be specified on the commandline when executing locally.
// This indirection also makes Jenkins use the branch of the PR
// for the test definitions.
tasks.register("javaPreCommit") {
  // We need to list the model/* builds since sdks/java/core doesn't
  // depend on any of the model.
  dependsOn(":model:pipeline:build")
  dependsOn(":model:job-management:build")
  dependsOn(":model:fn-execution:build")
  dependsOn(":sdks:java:core:buildNeeded")

  // Inline :sdks:java:core:buildDependents so we can carve out pieces at a time
  dependsOn(":beam-validate-runner:build")
  dependsOn(":examples:java:build")
  dependsOn(":examples:java:preCommit")
  dependsOn(":examples:java:sql:build")
  dependsOn(":examples:java:sql:preCommit")
  dependsOn(":examples:java:twitter:build")
  dependsOn(":examples:java:twitter:preCommit")
  dependsOn(":examples:java:iceberg:build")
  dependsOn(":examples:multi-language:build")
  dependsOn(":model:fn-execution:build")
  dependsOn(":model:job-management:build")
  dependsOn(":model:pipeline:build")
  dependsOn(":runners:core-java:build")
  dependsOn(":runners:direct-java:build")
  dependsOn(":runners:direct-java:needsRunnerTests")
  dependsOn(":runners:extensions-java:metrics:build")
  // lowest supported flink version
  var flinkVersions = project.ext.get("allFlinkVersions") as Array<*>
  dependsOn(":runners:flink:${flinkVersions[0]}:build")
  dependsOn(":runners:flink:${flinkVersions[0]}:job-server:build")
  dependsOn(":runners:google-cloud-dataflow-java:build")
  dependsOn(":runners:google-cloud-dataflow-java:examples-streaming:build")
  dependsOn(":runners:google-cloud-dataflow-java:examples:build")
  dependsOn(":runners:java-fn-execution:build")
  dependsOn(":runners:java-job-service:build")
  dependsOn(":runners:jet:build")
  // Only when the opt-in flag put it in the build; see settings.gradle.kts.
  if (findProject(":runners:kafka-streams") != null) {
    dependsOn(":runners:kafka-streams:build")
  }
  dependsOn(":runners:local-java:build")
  dependsOn(":runners:portability:java:build")
  dependsOn(":runners:prism:java:build")
  dependsOn(":runners:spark:3:build")
  dependsOn(":runners:spark:3:job-server:build")
  dependsOn(":sdks:java:build-tools:build")
  dependsOn(":sdks:java:container:java11:docker")
  dependsOn(":sdks:java:core:build")
  dependsOn(":sdks:java:core:jmh:build")
  dependsOn(":sdks:java:expansion-service:build")
  dependsOn(":sdks:java:expansion-service:app:build")
  dependsOn(":sdks:java:extensions:arrow:build")
  dependsOn(":sdks:java:extensions:avro:build")
  dependsOn(":sdks:java:extensions:combiners:build")
  dependsOn(":sdks:java:extensions:euphoria:build")
  dependsOn(":sdks:java:extensions:google-cloud-platform-core:build")
  dependsOn(":sdks:java:extensions:jackson:build")
  dependsOn(":sdks:java:extensions:join-library:build")
  dependsOn(":sdks:java:extensions:kryo:build")
  dependsOn(":sdks:java:extensions:ml:build")
  dependsOn(":sdks:java:extensions:protobuf:build")
  dependsOn(":sdks:java:extensions:python:build")
  dependsOn(":sdks:java:extensions:sbe:build")
  dependsOn(":sdks:java:extensions:schemaio-expansion-service:build")
  dependsOn(":sdks:java:extensions:sketching:build")
  dependsOn(":sdks:java:extensions:sorter:build")
  dependsOn(":sdks:java:extensions:timeseries:build")
  dependsOn(":sdks:java:extensions:yaml:build")
  dependsOn(":sdks:java:extensions:zetasketch:build")
  dependsOn(":sdks:java:harness:build")
  dependsOn(":sdks:java:harness:jmh:build")
  dependsOn(":sdks:java:io:bigquery-io-perf-tests:build")
  dependsOn(":sdks:java:io:common:build")
  dependsOn(":sdks:java:io:contextualtextio:build")
  dependsOn(":sdks:java:io:expansion-service:build")
  dependsOn(":sdks:java:io:file-based-io-tests:build")
  dependsOn(":sdks:java:io:sparkreceiver:3:build")
  dependsOn(":sdks:java:io:synthetic:build")
  dependsOn(":sdks:java:io:xml:build")
  dependsOn(":sdks:java:managed:build")
  dependsOn("sdks:java:ml:inference:remote:build")
  dependsOn("sdks:java:ml:inference:openai:build")
  dependsOn(":sdks:java:testing:expansion-service:build")
  dependsOn(":sdks:java:testing:jpms-tests:build")
  dependsOn(":sdks:java:testing:junit:build")
  dependsOn(":sdks:java:testing:load-tests:build")
  dependsOn(":sdks:java:testing:nexmark:build")
  dependsOn(":sdks:java:testing:test-utils:build")
  dependsOn(":sdks:java:testing:tpcds:build")
  dependsOn(":sdks:java:testing:watermarks:build")
  dependsOn(":sdks:java:transform-service:build")
  dependsOn(":sdks:java:transform-service:app:build")
  dependsOn(":sdks:java:transform-service:launcher:build")
}

// a precommit task build multiple IOs (except those splitting into single jobs)
tasks.register("javaioPreCommit") {
  dependsOn(":sdks:java:io:amqp:build")
  dependsOn(":sdks:java:io:arrow-flight:build")
  // CassandraIO, HBaseIO and HCatalogIO do not support Java17+, test ran separately
  // dependsOn(":sdks:java:io:cassandra:build")
  dependsOn(":sdks:java:io:csv:build")
  dependsOn(":sdks:java:io:cdap:build")
  dependsOn(":sdks:java:io:clickhouse:build")
  dependsOn(":sdks:java:io:datadog:build")
  dependsOn(":sdks:java:io:debezium:expansion-service:build")
  dependsOn(":sdks:java:io:debezium:build")
  dependsOn(":sdks:java:io:elasticsearch:build")
  dependsOn(":sdks:java:io:file-schema-transform:build")
  dependsOn(":sdks:java:io:google-ads:build")
  // dependsOn(":sdks:java:io:hbase:build")
  // dependsOn(":sdks:java:io:hcatalog:build")
  dependsOn(":sdks:java:io:influxdb:build")
  dependsOn(":sdks:java:io:jdbc:build")
  dependsOn(":sdks:java:io:jms:build")
  dependsOn(":sdks:java:io:kafka:build")
  dependsOn(":sdks:java:io:kafka:jmh:build")
  dependsOn(":sdks:java:io:kafka:upgrade:build")
  dependsOn(":sdks:java:extensions:kafka-factories:build")
  dependsOn(":sdks:java:io:kudu:build")
  dependsOn(":sdks:java:io:mongodb:build")
  dependsOn(":sdks:java:io:mqtt:build")
  dependsOn(":sdks:java:io:neo4j:build")
  dependsOn(":sdks:java:io:parquet:build")
  dependsOn(":sdks:java:io:pulsar:build")
  dependsOn(":sdks:java:io:rabbitmq:build")
  dependsOn(":sdks:java:io:redis:build")
  dependsOn(":sdks:java:io:rrio:build")
  dependsOn(":sdks:java:io:singlestore:build")
  dependsOn(":sdks:java:io:solr:build")
  dependsOn(":sdks:java:io:splunk:build")
  dependsOn(":sdks:java:io:thrift:build")
  dependsOn(":sdks:java:io:tika:build")
}

// a precommit task testing additional supported flink versions not covered by
// the main Java PreCommit (lowest supported version)
tasks.register("flinkPreCommit") {
  var flinkVersions = project.ext.get("allFlinkVersions") as Array<*>
  for (version in flinkVersions.slice(1..flinkVersions.size - 1)) {
    dependsOn(":runners:flink:${version}:build")
    dependsOn(":runners:flink:${version}:job-server:build")
  }
}

tasks.register("sqlPreCommit") {
  dependsOn(":sdks:java:extensions:sql:preCommit")
  dependsOn(":sdks:java:extensions:sql:buildDependents")
  dependsOn(":sdks:java:extensions:sql:expansion-service:build")
  dependsOn(":sdks:java:extensions:sql:hcatalog:build")
  dependsOn(":sdks:java:extensions:sql:iceberg:build")
  dependsOn(":sdks:java:extensions:sql:delta:build")
  dependsOn(":sdks:java:extensions:sql:jdbc:build")
  dependsOn(":sdks:java:extensions:sql:jdbc:preCommit")
  dependsOn(":sdks:java:extensions:sql:perf-tests:build")
  dependsOn(":sdks:java:extensions:sql:udf-test-provider:build")
  dependsOn(":sdks:java:extensions:sql:udf:build")
}

tasks.register("javaPreCommitPortabilityApi") {
  dependsOn(":runners:google-cloud-dataflow-java:worker:build")
}

tasks.register("javaPostCommit") {
  dependsOn(":sdks:java:extensions:google-cloud-platform-core:postCommit")
  dependsOn(":sdks:java:extensions:zetasketch:postCommit")
  dependsOn(":sdks:java:extensions:ml:postCommit")
}

tasks.register("javaPostCommitSickbay") {
  for (version in project.ext.get("allFlinkVersions") as Array<*>) {
    dependsOn(":runners:flink:${version}:validatesRunnerSickbay")
  }
  dependsOn(":runners:spark:3:job-server:validatesRunnerSickbay")
  dependsOn(":runners:direct-java:validatesRunnerSickbay")
  dependsOn(":runners:portability:java:validatesRunnerSickbay")
}

tasks.register("javaHadoopVersionsTest") {
  dependsOn(":sdks:java:io:hadoop-common:hadoopVersionsTest")
  dependsOn(":sdks:java:io:hadoop-file-system:hadoopVersionsTest")
  dependsOn(":sdks:java:io:hadoop-format:hadoopVersionsTest")
  dependsOn(":sdks:java:io:hcatalog:hadoopVersionsTest")
  dependsOn(":sdks:java:io:iceberg:hadoopVersionsTest")
  dependsOn(":sdks:java:io:parquet:hadoopVersionsTest")
  dependsOn(":sdks:java:extensions:sorter:hadoopVersionsTest")
  dependsOn(":runners:spark:3:hadoopVersionsTest")
}

tasks.register("javaAvroVersionsTest") {
  dependsOn(":sdks:java:extensions:avro:avroVersionsTest")
}

tasks.register("sqlPostCommit") {
  dependsOn(":sdks:java:extensions:sql:postCommit")
  dependsOn(":sdks:java:extensions:sql:jdbc:postCommit")
  dependsOn(":sdks:java:extensions:sql:iceberg:integrationTest")
  dependsOn(":sdks:java:extensions:sql:hadoopVersionsTest")
}

tasks.register("goPreCommit") {
  // Ensure the Precommit builds run after the tests, in order to avoid the
  // flake described in BEAM-11918. This is done by splitting them into two
  // tasks and using "mustRunAfter" to enforce ordering.
  dependsOn(":goPrecommitTest")
  dependsOn(":goPrecommitBuild")
}

tasks.register("goPrecommitTest") {
  dependsOn(":sdks:go:goTest")
}

tasks.register("goPrecommitBuild") {
  mustRunAfter(":goPrecommitTest")

  dependsOn(":sdks:go:goBuild")
  dependsOn(":sdks:go:examples:goBuild")
  dependsOn(":sdks:go:test:goBuild")

  // Ensure all container Go boot code builds as well.
  dependsOn(":sdks:java:container:goBuild")
  dependsOn(":sdks:python:container:goBuild")
  dependsOn(":sdks:go:container:goBuild")
}

tasks.register("goPortablePreCommit") {
  dependsOn(":sdks:go:test:ulrValidatesRunner")
}

tasks.register("goPrismPreCommit") {
  dependsOn(":sdks:go:test:prismValidatesRunner")
}

tasks.register("goPostCommitDataflowARM") {
  dependsOn(":sdks:go:test:dataflowValidatesRunnerARM64")
}

tasks.register("goPostCommit") {
  dependsOn(":sdks:go:test:dataflowValidatesRunner")
}

tasks.register("playgroundPreCommit") {
  dependsOn(":playground:lintProto")
  dependsOn(":playground:backend:precommit")
  dependsOn(":playground:frontend:precommit")
}

tasks.register("pythonPreCommit") {
  dependsOn(":sdks:python:test-suites:tox:pycommon:preCommitPyCommon")
  dependsOn(":sdks:python:test-suites:tox:py311:preCommitPy311")
  dependsOn(":sdks:python:test-suites:tox:py312:preCommitPy312")
  dependsOn(":sdks:python:test-suites:tox:py313:preCommitPy313")
  dependsOn(":sdks:python:test-suites:tox:py314:preCommitPy314")
}

tasks.register("pythonPreCommitIT") {
  dependsOn(":sdks:python:test-suites:tox:pycommon:preCommitPyCommon")
  dependsOn(":sdks:python:test-suites:dataflow:preCommitIT")
}

tasks.register("pythonDocsPreCommit") {
  dependsOn(":sdks:python:test-suites:tox:pycommon:docs")
}

tasks.register("pythonDockerBuildPreCommit") {
  dependsOn(":sdks:python:container:py311:docker")
  dependsOn(":sdks:python:container:py312:docker")
  dependsOn(":sdks:python:container:py313:docker")
  dependsOn(":sdks:python:container:py314:docker")
}

tasks.register("pythonLintPreCommit") {
  dependsOn(":sdks:python:test-suites:tox:pycommon:linter")
}

tasks.register("pythonFormatterPreCommit") {
  dependsOn("sdks:python:test-suites:tox:pycommon:formatter")
}

val changesLanguagePattern = "Java|Python|Go|Kotlin|TypeScript|YAML"
val changesRunnerPattern = "Dataflow|Flink|Jet|Kafka-Stream|Prism|Spark"
val changesTagPattern = "$changesLanguagePattern|$changesRunnerPattern"

tasks.register("formatChanges") {
  group = "formatting"
  description = "Formats CHANGES.md according to the template structure"

  doLast {
    val changesFile = file("CHANGES.md")
    if (!changesFile.exists()) {
      throw GradleException("CHANGES.md file not found")
    }

    val content = changesFile.readText()
    val lines = content.lines().toMutableList()

    // Find template end (after --> that follows <!-- Template -->)
    var templateStartIndex = -1
    var templateEndIndex = -1

    for (i in lines.indices) {
      if (lines[i].trim() == "<!-- Template -->") {
        templateStartIndex = i
      } else if (templateStartIndex != -1 && lines[i].trim() == "-->") {
        templateEndIndex = i
        break
      }
    }

    if (templateEndIndex == -1) {
      throw GradleException("Template end marker not found in CHANGES.md")
    }

    // Process each release section
    var i = templateEndIndex + 1
    val formattedLines = mutableListOf<String>()

    // Keep header and template exactly as-is (lines 0 to templateEndIndex inclusive)
    formattedLines.addAll(lines.subList(0, templateEndIndex + 1))

    // Always add blank line after template
    formattedLines.add("")

    while (i < lines.size) {
      val line = lines[i]

      // Check if this is a release header
      if (line.startsWith("# [")) {
        formattedLines.add(line)
        i++

        // Expected sections in order (following template)
        val expectedSections = listOf(
          "## Beam 3.0.0 Development Highlights",
          "## Highlights",
          "## I/Os",
          "## New Features / Improvements",
          "## Improvements",
          "## Breaking Changes",
          "## Deprecations",
          "## Bugfixes",
          "## Security Fixes",
          "## Known Issues"
        )

        val sectionContent = mutableMapOf<String, MutableList<String>>()
        var currentSection = ""

        // Parse existing sections
        while (i < lines.size && !lines[i].startsWith("# [")) {
          val currentLine = lines[i]

          if (currentLine.startsWith("## ")) {
            currentSection = currentLine
            if (!sectionContent.containsKey(currentSection)) {
              sectionContent[currentSection] = mutableListOf()
            }
          } else if (currentSection.isNotEmpty()) {
            sectionContent[currentSection]!!.add(currentLine)
          }
          i++
        }

        // Only add sections that actually exist with content
        val bracketedTagRegex = Regex("\\[(?:$changesTagPattern)(?:/(?:$changesTagPattern))*\\]")
        for (section in expectedSections) {
          if (sectionContent.containsKey(section)) {
            formattedLines.add("")
            formattedLines.add(section)
            formattedLines.add("")

            // Remove empty lines at start and end
            val content = sectionContent[section]!!
            while (content.isNotEmpty() && content.first().trim().isEmpty()) {
              content.removeAt(0)
            }
            while (content.isNotEmpty() && content.last().trim().isEmpty()) {
              content.removeAt(content.size - 1)
            }

            // Format content according to template rules
            val formattedContent = content.map { line ->
              // Convert SDK language and runner references from [Language]/[Runner] to (Language)/(Runner)
              line.replace(bracketedTagRegex) { matchResult ->
                "(${matchResult.value.substring(1, matchResult.value.length - 1)})"
              }
            }

            formattedLines.addAll(formattedContent)
          }
        }

        if (i < lines.size) {
          formattedLines.add("")
        }
      } else {
        i++
      }
    }

    // Write formatted content back
    changesFile.writeText(formattedLines.joinToString("\n") + "\n")
    println("CHANGES.md has been formatted according to template structure")
  }
}

tasks.register("validateChanges") {
  group = "verification"
  description = "Validates CHANGES.md follows required formatting rules"

  doLast {
    val changesFile = file("CHANGES.md")
    if (!changesFile.exists()) {
      throw GradleException("CHANGES.md file not found")
    }

    val content = changesFile.readText()
    val lines = content.lines()
    val errors = mutableListOf<String>()

    // Find template section boundaries
    var templateStartIndex = -1
    var templateEndIndex = -1

    for (i in lines.indices) {
      if (lines[i].trim() == "<!-- Template -->") {
        templateStartIndex = i
        println("Found template start at line ${i+1}")
      } else if (templateStartIndex != -1 && lines[i].trim() == "-->") {
        templateEndIndex = i
        println("Found template end at line ${i+1}")
        break
      }
    }

    if (templateStartIndex == -1 || templateEndIndex == -1) {
      throw GradleException("Template section not found in CHANGES.md")
    }

    println("Template section: lines ${templateStartIndex+1} to ${templateEndIndex+1}")

    // Find unreleased section after the template section
    var unreleasedSectionStart = -1
    for (i in (templateEndIndex + 1) until lines.size) {
      if (lines[i].startsWith("# [") && lines[i].contains("Unreleased")) {
        unreleasedSectionStart = i
        println("Found unreleased section at line ${i+1}: ${lines[i]}")
        break
      }
    }

    if (unreleasedSectionStart == -1) {
      throw GradleException("Unreleased section not found in CHANGES.md")
    }

    // Check entries in the unreleased section
    var i = unreleasedSectionStart + 1
    val items = TreeMap<Int, String>()
    var lastline = 0
    var item = ""
    while (i < lines.size && !lines[i].startsWith("# [")) {
      val line = lines[i].trim()
      if (line.isEmpty()) {
        // skip
      } else if (line.startsWith("* ")) {
        items.put(lastline, item)
        lastline = i
        item = line
      } else if (line.startsWith("##")) {
        items.put(lastline, item)
        lastline = i
        item = ""
      } else {
        item += line
      }
      i++
    }
    items.put(lastline, item)
    println("Starting validation from line ${i+1}")

    val bracketedTagRegex = Regex("\\[(?:$changesTagPattern)(?:/(?:$changesTagPattern))*\\]")
    val linkRegex = Regex("\\(\\[#[0-9a-zA-Z]+\\]\\(https://github\\.com/apache/beam/(?:issues|pull)/[0-9a-zA-Z]+\\)\\)")

    items.forEach { (i, line) ->
      if (line.startsWith("* ")) {
        println("Checking line ${i+1}: $line")

        // Skip comment lines
        if (line.startsWith("* [comment]:")) {
          println("  Skipping comment line")
        } else {
          // Rule 1: Check if language or runner references use parentheses instead of brackets
          val matches = bracketedTagRegex.findAll(line).toList()
          if (matches.isNotEmpty()) {
            for (match in matches) {
              val matchText = match.value
              val matchPosition = match.range.first
              println("  Found bracketed language/runner reference: $matchText at position $matchPosition")

              // Check if this is part of an issue link or URL
              val beforeMatch = if (matchPosition > 0) line.substring(0, matchPosition) else ""
              val isPartOfLink = beforeMatch.contains("[#") ||
                                beforeMatch.contains("http") ||
                                line.contains("CVE-")

              println("  Is part of link: $isPartOfLink")

              if (!isPartOfLink) {
                val error = "Line ${i+1}: Language/runner references should use parentheses () instead of brackets []: $line"
                println("  Adding error: $error")
                errors.add(error)
              }
            }
          } else {
            println("  No bracketed language/runner reference found")
          }

          // Rule 2: Check if each entry links an issue or a PR
          val hasLink = linkRegex.containsMatchIn(line)
          println("  Has issue link: $hasLink")

          if (!hasLink) {
            val error = "Line ${i+1}: Missing or malformed issue link. Each entry must end with a reference to an Issue or a PR, for example: ([#X](https://github.com/apache/beam/issues/X)): $line"
            println("  Adding error: $error")
            errors.add(error)
          }
        }
      }
    }

    println("Found ${errors.size} errors")

    if (errors.isNotEmpty()) {
      throw GradleException("CHANGES.md validation failed with the following errors:\n${errors.joinToString("\n")}\n\nYou can run `./gradlew formatChanges` to correct some issues.")
    }

    println("CHANGES.md validation successful")
  }
}


tasks.register("python311PostCommit") {
  dependsOn(":sdks:python:test-suites:dataflow:py311:postCommitIT")
  dependsOn(":sdks:python:test-suites:direct:py311:postCommitIT")
  dependsOn(":sdks:python:test-suites:direct:py311:hdfsIntegrationTest")
  dependsOn(":sdks:python:test-suites:direct:py311:azureIntegrationTest")
  dependsOn(":sdks:python:test-suites:portable:py311:postCommitPy311")
}

tasks.register("python312PostCommit") {
  dependsOn(":sdks:python:test-suites:dataflow:py312:postCommitIT")
  dependsOn(":sdks:python:test-suites:direct:py312:postCommitIT")
  dependsOn(":sdks:python:test-suites:direct:py312:hdfsIntegrationTest")
  dependsOn(":sdks:python:test-suites:portable:py312:postCommitPy312")
  dependsOn(":sdks:python:test-suites:dataflow:py312:inferencePostCommitITPy312")
}

tasks.register("python313PostCommit") {
  dependsOn(":sdks:python:test-suites:dataflow:py313:postCommitIT")
  dependsOn(":sdks:python:test-suites:direct:py313:postCommitIT")
  dependsOn(":sdks:python:test-suites:direct:py313:hdfsIntegrationTest")
  dependsOn(":sdks:python:test-suites:portable:py313:postCommitPy313")
}

tasks.register("python314PostCommit") {
  dependsOn(":sdks:python:test-suites:dataflow:py314:postCommitIT")
  dependsOn(":sdks:python:test-suites:direct:py314:postCommitIT")
  dependsOn(":sdks:python:test-suites:direct:py314:hdfsIntegrationTest")
  dependsOn(":sdks:python:test-suites:portable:py314:postCommitPy314")
}

tasks.register("portablePythonPreCommit") {
  dependsOn(":sdks:python:test-suites:portable:py311:preCommitPy311")
  dependsOn(":sdks:python:test-suites:portable:py314:preCommitPy314")
}

tasks.register("pythonSparkPostCommit") {
  dependsOn(":sdks:python:test-suites:portable:py311:sparkValidatesRunner")
  dependsOn(":sdks:python:test-suites:portable:py314:sparkValidatesRunner")
}

tasks.register("websitePreCommit") {
  dependsOn(":website:preCommit")
}

tasks.register("communityMetricsPreCommit") {
  dependsOn(":beam-test-infra-metrics:preCommit")
}

tasks.register("communityMetricsProber") {
  dependsOn(":beam-test-infra-metrics:checkProber")
}

tasks.register("javaExamplesDataflowPrecommit") {
  dependsOn(":runners:google-cloud-dataflow-java:examples:preCommit")
  dependsOn(":runners:google-cloud-dataflow-java:examples-streaming:preCommit")
  dependsOn(":runners:google-cloud-dataflow-java:examplesJavaRunnerV2PreCommit")
}

tasks.register("whitespacePreCommit") {
  // TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify the tasks without hardcoding py version.
  dependsOn(":sdks:python:test-suites:tox:py311:archiveFilesToLint")
  dependsOn(":sdks:python:test-suites:tox:py311:unpackFilesToLint")
  dependsOn(":sdks:python:test-suites:tox:py311:whitespacelint")
}

tasks.register("typescriptPreCommit") {
  // TODO(https://github.com/apache/beam/issues/20209): Find a better way to specify the tasks without hardcoding py version.
  dependsOn(":sdks:python:test-suites:tox:py311:eslint")
  dependsOn(":sdks:python:test-suites:tox:py311:jest")
}

tasks.register("pushAllRunnersDockerImages") {
  dependsOn(":runners:spark:3:job-server:container:docker")
  for (version in project.ext.get("allFlinkVersions") as Array<*>) {
    dependsOn(":runners:flink:${version}:job-server-container:docker")
  }

  doLast {
    if (project.hasProperty("prune-images")) {
      exec {
        executable("docker")
        args("system", "prune", "-a", "--force")
      }
    }
  }
}

tasks.register("pushAllSdkDockerImages") {
  // Enforce ordering to allow the prune step to happen between runs.
  // This will ensure we don't use up too much space (especially in CI environments)
  if (!project.hasProperty("skip-runner-images")) {
    mustRunAfter(":pushAllRunnersDockerImages")
  }

  dependsOn(":sdks:java:container:pushAll")
  if (!project.hasProperty("skip-python-images")) {
    dependsOn(":sdks:python:container:pushAll")
  }
  dependsOn(":sdks:go:container:pushAll")
  dependsOn(":sdks:typescript:container:pushAll")

  doLast {
    if (project.hasProperty("prune-images")) {
      exec {
        executable("docker")
        args("system", "prune", "-a", "--force")
      }
    }
  }
}

tasks.register("pushAllXlangDockerImages") {
  // Enforce ordering to allow the prune step to happen between runs.
  // This will ensure we don't use up too much space (especially in CI environments)
  if (!project.hasProperty("skip-sdk-images")) {
    mustRunAfter(":pushAllSdkDockerImages")
  }

  dependsOn(":sdks:java:expansion-service:container:docker")
  dependsOn(":sdks:java:transform-service:controller-container:docker")
  dependsOn(":sdks:python:expansion-service-container:docker")

  doLast {
    if (project.hasProperty("prune-images")) {
      exec {
        executable("docker")
        args("system", "prune", "-a", "--force")
      }
    }
  }
}

tasks.register("pushAllDockerImages") {
  if (!project.hasProperty("skip-runner-images")) {
    dependsOn(":pushAllRunnersDockerImages")
  }
  if (!project.hasProperty("skip-sdk-images")) {
    dependsOn(":pushAllSdkDockerImages")
  }
  if (!project.hasProperty("skip-xlang-images")) {
    dependsOn(":pushAllXlangDockerImages")
  }
}

// Use this task to validate the environment set up for Go, Python and Java
tasks.register("checkSetup") {
  dependsOn(":sdks:go:examples:wordCount")
  dependsOn(":sdks:python:wordCount")
  dependsOn(":examples:java:wordCount")
}

// if not disabled make spotlessApply dependency of compileJava and compileTestJava
val disableSpotlessCheck: String by project
val isSpotlessDisabled = (project.hasProperty("disableSpotlessCheck") &&
        disableSpotlessCheck == "true") || project.hasProperty("disableSpotlessApply")
if (!isSpotlessDisabled) {
  subprojects {
    afterEvaluate {
      tasks.findByName("spotlessApply")?.let {
        listOf("compileJava", "compileTestJava").forEach {
          t -> tasks.findByName(t)?.let { f -> f.dependsOn("spotlessApply") }
        }
      }
    }
  }
}

// Generates external transform config
project.tasks.register("generateExternalTransformsConfig") {
  dependsOn(":sdks:python:generateExternalTransformsConfig")
}

// Generates the Managed IO Beam web page
project.tasks.register("generateManagedIOPage") {
  dependsOn(":sdks:python:generateManagedIOPage")
}

// Reports linkage errors across multiple Apache Beam artifact ids.
//
// To use (from the root of project):
//    ./gradlew -Ppublishing -PjavaLinkageArtifactIds=artifactId1,artifactId2,... :checkJavaLinkage
//
// For example:
//    ./gradlew -Ppublishing -PjavaLinkageArtifactIds=beam-sdks-java-core,beam-sdks-java-io-jdbc :checkJavaLinkage
//
// Note that this task publishes artifacts into your local Maven repository.
if (project.hasProperty("javaLinkageArtifactIds")) {
  if (!project.hasProperty("publishing")) {
    throw GradleException("You can only check linkage of Java artifacts if you specify -Ppublishing on the command line as well.")
  }

  val linkageCheckerJava by configurations.creating
  dependencies {
    linkageCheckerJava("com.google.cloud.tools:dependencies:1.5.16")
  }

  // We need to evaluate all the projects first so that we can find depend on all the
  // publishMavenJavaPublicationToMavenLocal tasks below.
  for (p in rootProject.subprojects) {
    if (p.path != project.path) {
      evaluationDependsOn(p.path)
    }
  }

  project.tasks.register<JavaExec>("checkJavaLinkage") {
    dependsOn(project.getTasksByName("publishMavenJavaPublicationToMavenLocal", true /* recursively */))
    classpath = linkageCheckerJava
    mainClass.value("com.google.cloud.tools.opensource.classpath.LinkageCheckerMain")
    val javaLinkageArtifactIds: String = project.property("javaLinkageArtifactIds") as String? ?: ""
    var arguments = arrayOf("-a", javaLinkageArtifactIds.split(",").joinToString(",") {
      if (it.contains(":")) {
        "${project.ext.get("mavenGroupId")}:${it}"
      } else {
        // specify the version if not provided
        "${project.ext.get("mavenGroupId")}:${it}:${project.version}"
      }
    })

    // Exclusion file filters out existing linkage errors before a change
    if (project.hasProperty("javaLinkageWriteBaseline")) {
      arguments += "--output-exclusion-file"
      arguments += project.property("javaLinkageWriteBaseline") as String
    } else if (project.hasProperty("javaLinkageReadBaseline")) {
      arguments += "--exclusion-file"
      arguments += project.property("javaLinkageReadBaseline") as String
    }
    args(*arguments)
    doLast {
      println("NOTE: This task published artifacts into your local Maven repository. You may want to remove them manually.")
    }
  }
}
if (project.hasProperty("testJavaVersion")) {
  var testVer = project.property("testJavaVersion")

  tasks.getByName("javaPreCommitPortabilityApi").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer")
  tasks.getByName("javaExamplesDataflowPrecommit").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer")
} else {
  allprojects {
    tasks.withType(Test::class).configureEach {
      exclude("**/JvmVerification.class")
    }
  }
}