| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # Replacement for dorny/paths-filter, which is not on the apache org allow | |
| # list. Reads a list of changed files (one per line) and emits per-job | |
| # "<name>=true|false" lines suitable for $GITHUB_OUTPUT. | |
| # | |
| # Each output folds together two independent questions: | |
| # | |
| # 1. Did the change touch files this job covers? FILTERS, below. Pattern | |
| # semantics match dorny/picomatch: "**" spans path segments, "*" stays | |
| # within a segment, and a leading "!" marks an exclude pattern. | |
| # 2. Does this event permit the job to run at all? POLICY, below. | |
| # | |
| # Question 2 used to live in ci.yml as a four-line `${{ }}` expression | |
| # repeated on every heavy job. Keeping it here instead means the whole | |
| # routing policy is in one place, is readable without evaluating GitHub | |
| # expression syntax in your head, and is covered by the cases in | |
| # dev/ci/check-ci-config.py, which YAML expressions never could be. | |
| import json | |
| import os | |
| import re | |
| import sys | |
| from pathlib import Path | |
| FILTERS = { | |
| "build_linux": [ | |
| "native/**", | |
| "common/**", | |
| "spark/**", | |
| "spark-integration/**", | |
| # The strict warnings job compiles the Delta contrib against Spark 3.5. | |
| "contrib/delta-spark/**", | |
| "pom.xml", | |
| "**/pom.xml", | |
| ".mvn/**", | |
| "mvnw", | |
| "Makefile", | |
| "rust-toolchain.toml", | |
| "dev/ci/**", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/pr_build_linux.yml", | |
| ".github/actions/setup-builder/**", | |
| ".github/actions/java-test/**", | |
| ".github/actions/maven-bootstrap/**", | |
| ".github/actions/rust-test/**", | |
| ".github/actions/upload-artifact-retry/**", | |
| ".github/actions/download-artifact-retry/**", | |
| "!**.md", | |
| "!native/core/benches/**", | |
| "!native/spark-expr/benches/**", | |
| "!native/operators/benches/**", | |
| "!spark/src/test/scala/org/apache/spark/sql/benchmark/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| ], | |
| # Same inputs as build_linux: not a separate job but a second POLICY | |
| # decision for the same call, selecting the full pipeline rather than the | |
| # cache-populating subset. ci.yml folds it into the reusable workflow's | |
| # `cache-refresh-only` input. Populated below, after the dict, so the two | |
| # lists cannot drift. | |
| "build_linux_full": [], | |
| # A third POLICY decision on the same inputs: whether the linux-test matrix | |
| # runs every Spark profile or only the PR-tier one. ci.yml folds it into | |
| # the reusable workflow's `profiles` input. Populated below as well. | |
| "build_linux_all_profiles": [], | |
| "build_macos": [ | |
| "native/**", | |
| "common/**", | |
| "spark/**", | |
| "spark-integration/**", | |
| "pom.xml", | |
| "**/pom.xml", | |
| ".mvn/**", | |
| "mvnw", | |
| "Makefile", | |
| "rust-toolchain.toml", | |
| "dev/ci/**", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/pr_build_macos.yml", | |
| ".github/actions/setup-macos-builder/**", | |
| ".github/actions/java-test/**", | |
| ".github/actions/upload-artifact-retry/**", | |
| ".github/actions/download-artifact-retry/**", | |
| ".github/actions/maven-bootstrap/**", | |
| "!**.md", | |
| "!native/core/benches/**", | |
| "!native/spark-expr/benches/**", | |
| "!native/operators/benches/**", | |
| "!spark/src/test/scala/org/apache/spark/sql/benchmark/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| ], | |
| "benchmark": [ | |
| "native/core/benches/**", | |
| "native/spark-expr/benches/**", | |
| "native/operators/benches/**", | |
| "spark/src/test/scala/org/apache/spark/sql/benchmark/**", | |
| ], | |
| # dev/verify-contrib-delta-gate.sh proves the default cargo, Maven and | |
| # libcomet builds carry no Delta surface and that the gated build does. | |
| # It reads the cargo tree, the effective pom, the compiled classes and the | |
| # dylib symbol table: main sources and build inputs, never tests. | |
| "delta_gate": [ | |
| "native/**", | |
| "common/src/main/**", | |
| "spark/src/main/**", | |
| "contrib/delta/**", | |
| "pom.xml", | |
| "**/pom.xml", | |
| ".mvn/**", | |
| "mvnw", | |
| "Makefile", | |
| "rust-toolchain.toml", | |
| "dev/verify-contrib-delta-gate.sh", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/delta_build_gate.yml", | |
| ".github/actions/setup-builder/**", | |
| ".github/actions/maven-bootstrap/**", | |
| "!**.md", | |
| "!native/core/benches/**", | |
| "!native/spark-expr/benches/**", | |
| "!native/operators/benches/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| ], | |
| # A real Python worker against each Spark 4.x Arrow runner. The list is | |
| # deliberately narrow: the suite builds Comet three times, once per Spark | |
| # version, and covers map-in-batch wiring and the native scalar Arrow UDF. | |
| "pyarrow_udf": [ | |
| "pom.xml", | |
| "common/pom.xml", | |
| "native/shuffle/src/spark_unsafe/row.rs", | |
| "native/core/src/execution/python_udf.rs", | |
| "native/core/src/execution/operators/arrow_python_udf.rs", | |
| "spark/src/main/spark-4.1+/org/apache/spark/sql/comet/CometArrowEvalPythonExec.scala", | |
| "spark/src/main/spark-4.1+/org/apache/spark/sql/comet/shims/ShimCometArrowEvalPythonExec.scala", | |
| "spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala", | |
| "spark/pom.xml", | |
| "spark/src/main/java/org/apache/comet/vector/**", | |
| "spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java", | |
| "spark/src/main/scala/org/apache/comet/CometConf.scala", | |
| "spark/src/main/scala/org/apache/comet/rules/EliminateRedundantTransitions.scala", | |
| "spark/src/main/scala/org/apache/comet/vector/**", | |
| "spark/src/main/scala/org/apache/spark/sql/comet/CometMapInBatchExec.scala", | |
| "spark/src/main/scala/org/apache/spark/sql/comet/shims/MapInBatchInfo.scala", | |
| "spark/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimCometMapInBatch.scala", | |
| "spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimCometMapInBatch.scala", | |
| "spark/src/main/spark-4.0/org/apache/spark/sql/comet/shims/ShimCometMapInBatch.scala", | |
| "spark/src/main/spark-4.0/org/apache/spark/sql/execution/python/CometArrowPythonRunner.scala", | |
| "spark/src/main/spark-4.1/org/apache/spark/sql/comet/shims/ShimCometMapInBatch.scala", | |
| "spark/src/main/spark-4.1/org/apache/spark/sql/execution/python/CometArrowPythonRunner.scala", | |
| "spark/src/main/spark-4.2/org/apache/spark/sql/comet/shims/ShimCometMapInBatch.scala", | |
| "spark/src/main/spark-4.2/org/apache/spark/sql/execution/python/CometArrowPythonRunner.scala", | |
| "spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/Spark4xMapInBatchSupport.scala", | |
| "spark/src/main/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerBase.scala", | |
| "spark/src/test/resources/pyspark/conftest.py", | |
| "spark/src/test/resources/pyspark/test_pyarrow_udf.py", | |
| "spark/src/test/resources/pyspark/test_native_arrow_udf_files.py", | |
| "spark/src/test/resources/pyspark/test_pyarrow_udf_fuzz.py", | |
| "spark/src/test/resources/pyspark/test_pyarrow_udf_dictionary_shuffle.py", | |
| "spark/src/test/spark-3.5/org/apache/spark/sql/comet/CometMapInBatchSuite.scala", | |
| "spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala", | |
| "spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala", | |
| ".mvn/**", | |
| "mvnw", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/pyarrow_udf_test.yml", | |
| ".github/actions/setup-builder/**", | |
| ".github/actions/maven-bootstrap/**", | |
| ], | |
| "docs": [ | |
| ".asf.yaml", | |
| ".github/workflows/docs.yaml", | |
| "docs/**", | |
| # The docs deploy renders and then verifies the site's mermaid diagrams with this | |
| # script, so a change to it has to be exercised by a real build, not just by the | |
| # preflight run that renders the fences. | |
| "dev/ci/check-mermaid.py", | |
| # Generated docs (configs.md, per-version expression compatibility pages) are | |
| # built from these Scala sources by GenerateDocs, so changes to them must | |
| # republish the site even when no docs/ file is touched. | |
| "spark/src/main/scala/org/apache/comet/CometConf.scala", | |
| "spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| "spark/src/main/scala/org/apache/comet/serde/**", | |
| "spark/src/main/scala/org/apache/comet/expressions/**", | |
| "spark/src/main/spark-*/**", | |
| ], | |
| "spark_3_4": [ | |
| "native/**/src/**", | |
| "native/**/Cargo.toml", | |
| "native/Cargo.lock", | |
| "common/src/main/**", | |
| "common/pom.xml", | |
| "spark/src/main/**", | |
| "!spark/src/main/spark-3.5/**", | |
| "!spark/src/main/spark-4.0/**", | |
| "!spark/src/main/spark-4.1/**", | |
| "!spark/src/main/spark-4.2/**", | |
| "!spark/src/main/spark-4.x/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| "spark/pom.xml", | |
| "dev/diffs/3.4.3.diff", | |
| "pom.xml", | |
| "rust-toolchain.toml", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/spark_sql_test_reusable.yml", | |
| "dev/ci/spark-sql-modules.py", | |
| ".github/actions/setup-builder/**", | |
| ".github/actions/setup-spark-builder/**", | |
| ".github/actions/upload-artifact-retry/**", | |
| ".github/actions/download-artifact-retry/**", | |
| ".github/actions/maven-bootstrap/**", | |
| ".mvn/**", | |
| "mvnw", | |
| ], | |
| "spark_3_5": [ | |
| "native/**/src/**", | |
| "native/**/Cargo.toml", | |
| "native/Cargo.lock", | |
| "common/src/main/**", | |
| "common/pom.xml", | |
| "spark/src/main/**", | |
| "!spark/src/main/spark-3.4/**", | |
| "!spark/src/main/spark-4.0/**", | |
| "!spark/src/main/spark-4.1/**", | |
| "!spark/src/main/spark-4.2/**", | |
| "!spark/src/main/spark-4.x/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| "spark/pom.xml", | |
| "dev/diffs/3.5.9.diff", | |
| "pom.xml", | |
| "rust-toolchain.toml", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/spark_sql_test_reusable.yml", | |
| "dev/ci/spark-sql-modules.py", | |
| ".github/actions/setup-builder/**", | |
| ".github/actions/setup-spark-builder/**", | |
| ".github/actions/upload-artifact-retry/**", | |
| ".github/actions/download-artifact-retry/**", | |
| ".github/actions/maven-bootstrap/**", | |
| ".mvn/**", | |
| "mvnw", | |
| ], | |
| "spark_4_0": [ | |
| "native/**/src/**", | |
| "native/**/Cargo.toml", | |
| "native/Cargo.lock", | |
| "common/src/main/**", | |
| "common/pom.xml", | |
| "spark/src/main/**", | |
| "!spark/src/main/spark-3.4/**", | |
| "!spark/src/main/spark-3.5/**", | |
| "!spark/src/main/spark-3.x/**", | |
| "!spark/src/main/spark-4.1/**", | |
| "!spark/src/main/spark-4.2/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| "spark/pom.xml", | |
| "dev/diffs/4.0.4.diff", | |
| "pom.xml", | |
| "rust-toolchain.toml", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/spark_sql_test_reusable.yml", | |
| "dev/ci/spark-sql-modules.py", | |
| ".github/actions/setup-builder/**", | |
| ".github/actions/setup-spark-builder/**", | |
| ".github/actions/upload-artifact-retry/**", | |
| ".github/actions/download-artifact-retry/**", | |
| ".github/actions/maven-bootstrap/**", | |
| ".mvn/**", | |
| "mvnw", | |
| ], | |
| "spark_4_1": [ | |
| "native/**/src/**", | |
| "native/**/Cargo.toml", | |
| "native/Cargo.lock", | |
| "common/src/main/**", | |
| "common/pom.xml", | |
| "spark/src/main/**", | |
| "!spark/src/main/spark-3.4/**", | |
| "!spark/src/main/spark-3.5/**", | |
| "!spark/src/main/spark-3.x/**", | |
| "!spark/src/main/spark-4.0/**", | |
| "!spark/src/main/spark-4.2/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| "spark/pom.xml", | |
| "dev/diffs/4.1.3.diff", | |
| "pom.xml", | |
| "rust-toolchain.toml", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/spark_sql_test_reusable.yml", | |
| "dev/ci/spark-sql-modules.py", | |
| ".github/actions/setup-builder/**", | |
| ".github/actions/setup-spark-builder/**", | |
| ".github/actions/upload-artifact-retry/**", | |
| ".github/actions/download-artifact-retry/**", | |
| ".github/actions/maven-bootstrap/**", | |
| ".mvn/**", | |
| "mvnw", | |
| ], | |
| "spark_4_2": [ | |
| "native/**/src/**", | |
| "native/**/Cargo.toml", | |
| "native/Cargo.lock", | |
| "common/src/main/**", | |
| "common/pom.xml", | |
| "spark/src/main/**", | |
| "!spark/src/main/spark-3.4/**", | |
| "!spark/src/main/spark-3.5/**", | |
| "!spark/src/main/spark-3.x/**", | |
| "!spark/src/main/spark-4.0/**", | |
| "!spark/src/main/spark-4.1/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| "spark/pom.xml", | |
| "dev/diffs/4.2.0.diff", | |
| "pom.xml", | |
| "rust-toolchain.toml", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/spark_sql_test_reusable.yml", | |
| "dev/ci/spark-sql-modules.py", | |
| ".github/actions/setup-builder/**", | |
| ".github/actions/setup-spark-builder/**", | |
| ".github/actions/upload-artifact-retry/**", | |
| ".github/actions/download-artifact-retry/**", | |
| ".github/actions/maven-bootstrap/**", | |
| ".mvn/**", | |
| "mvnw", | |
| ], | |
| # Same inputs as spark_4_1: this is not a separate job but a second | |
| # POLICY decision for the same call, selecting the sql_hive matrix rows. | |
| # ci.yml folds the two outputs into the reusable workflow's `modules` | |
| # input. Populated below, after the dict, so the two lists cannot drift. | |
| "spark_4_1_hive": [], | |
| "iceberg_1_8": [ | |
| "native/**/src/**", | |
| "native/**/Cargo.toml", | |
| "native/Cargo.lock", | |
| "common/src/main/**", | |
| "common/pom.xml", | |
| "spark/src/main/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| "spark/pom.xml", | |
| "dev/diffs/iceberg/**", | |
| "pom.xml", | |
| "rust-toolchain.toml", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/iceberg_spark_test_reusable.yml", | |
| ".github/actions/setup-builder/**", | |
| ".github/actions/setup-iceberg-builder/**", | |
| "dev/ci/iceberg-test-shards.gradle", | |
| "dev/ci/check-iceberg-shards.py", | |
| "dev/ci/test-iceberg-shards.py", | |
| ".github/actions/upload-artifact-retry/**", | |
| ".github/actions/download-artifact-retry/**", | |
| ".github/actions/maven-bootstrap/**", | |
| ".mvn/**", | |
| "mvnw", | |
| ], | |
| "iceberg_1_9": [ | |
| "native/**/src/**", | |
| "native/**/Cargo.toml", | |
| "native/Cargo.lock", | |
| "common/src/main/**", | |
| "common/pom.xml", | |
| "spark/src/main/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| "spark/pom.xml", | |
| "dev/diffs/iceberg/**", | |
| "pom.xml", | |
| "rust-toolchain.toml", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/iceberg_spark_test_reusable.yml", | |
| ".github/actions/setup-builder/**", | |
| ".github/actions/setup-iceberg-builder/**", | |
| "dev/ci/iceberg-test-shards.gradle", | |
| "dev/ci/check-iceberg-shards.py", | |
| "dev/ci/test-iceberg-shards.py", | |
| ".github/actions/upload-artifact-retry/**", | |
| ".github/actions/download-artifact-retry/**", | |
| ".github/actions/maven-bootstrap/**", | |
| ".mvn/**", | |
| "mvnw", | |
| ], | |
| "iceberg_1_10": [ | |
| "native/**/src/**", | |
| "native/**/Cargo.toml", | |
| "native/Cargo.lock", | |
| "common/src/main/**", | |
| "common/pom.xml", | |
| "spark/src/main/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| "spark/pom.xml", | |
| "dev/diffs/iceberg/**", | |
| "pom.xml", | |
| "rust-toolchain.toml", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/iceberg_spark_test_reusable.yml", | |
| ".github/actions/setup-builder/**", | |
| ".github/actions/setup-iceberg-builder/**", | |
| "dev/ci/iceberg-test-shards.gradle", | |
| "dev/ci/check-iceberg-shards.py", | |
| "dev/ci/test-iceberg-shards.py", | |
| ".github/actions/upload-artifact-retry/**", | |
| ".github/actions/download-artifact-retry/**", | |
| ".github/actions/maven-bootstrap/**", | |
| ".mvn/**", | |
| "mvnw", | |
| ], | |
| "iceberg_1_11": [ | |
| "native/**/src/**", | |
| "native/**/Cargo.toml", | |
| "native/Cargo.lock", | |
| "common/src/main/**", | |
| "common/pom.xml", | |
| "spark/src/main/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| "spark/pom.xml", | |
| "dev/diffs/iceberg/**", | |
| "pom.xml", | |
| "rust-toolchain.toml", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/iceberg_spark_test_reusable.yml", | |
| ".github/actions/setup-builder/**", | |
| ".github/actions/setup-iceberg-builder/**", | |
| "dev/ci/iceberg-test-shards.gradle", | |
| "dev/ci/check-iceberg-shards.py", | |
| "dev/ci/test-iceberg-shards.py", | |
| ".github/actions/upload-artifact-retry/**", | |
| ".github/actions/download-artifact-retry/**", | |
| ".github/actions/maven-bootstrap/**", | |
| ".mvn/**", | |
| "mvnw", | |
| ], | |
| "delta_3_5": [ | |
| "contrib/delta-spark/**", | |
| "!**.md", | |
| "native/**/src/**", | |
| "native/**/Cargo.toml", | |
| "native/Cargo.lock", | |
| "common/src/main/**", | |
| "common/pom.xml", | |
| "spark/src/main/**", | |
| "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", | |
| # The contrib suites extend these through the spark test-jar. | |
| "spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala", | |
| "spark/src/test/scala/org/apache/comet/CometS3TestBase.scala", | |
| "spark/pom.xml", | |
| "pom.xml", | |
| "rust-toolchain.toml", | |
| ".github/workflows/ci.yml", | |
| ".github/workflows/delta_contrib_test.yml", | |
| ".github/actions/setup-builder/**", | |
| ], | |
| # Same inputs as delta_3_5: the Delta contrib workflow is called once per | |
| # Spark profile, and the three calls share one path filter. Populated | |
| # below, after the dict, so the lists cannot drift. | |
| "delta_4_0": [], | |
| "delta_4_1": [], | |
| } | |
| FILTERS["spark_4_1_hive"] = FILTERS["spark_4_1"] | |
| FILTERS["build_linux_full"] = FILTERS["build_linux"] | |
| FILTERS["build_macos_full"] = FILTERS["build_macos"] | |
| FILTERS["build_linux_all_profiles"] = FILTERS["build_linux"] | |
| FILTERS["delta_4_0"] = FILTERS["delta_3_5"] | |
| FILTERS["delta_4_1"] = FILTERS["delta_3_5"] | |
| # Which events may run each job, independent of the path filters above. | |
| # | |
| # "pr" every pull request | |
| # "queue" the merge queue, i.e. a merge_group event | |
| # "nightly" the scheduled run against main, once a day | |
| # "push" push to main | |
| # "label:<name>" a pull request carrying that label | |
| # | |
| # workflow_dispatch always runs everything, so it is not listed. "pr" and | |
| # "label:" are mutually exclusive -- a job is either unconditional on pull | |
| # requests or opt-in, never both -- and check-ci-config.py rejects a job that | |
| # lists both rather than letting the label quietly win. | |
| # | |
| # The merge queue is the authoritative gate: it tests the merge result rather | |
| # than the PR head, and every "queue" job has to pass before a change lands. | |
| # "nightly" is for the suites that catch a regression on a Spark or Iceberg | |
| # version other than the default one: about 870 of the 1,900 runner-minutes a | |
| # queue run cost in September 2026, and the most common reason a queue run | |
| # went red on a good tree (issue #5870). A regression there is real but rare, | |
| # and a day's delay in seeing it costs less than running the suites on every | |
| # merge. The scheduled run diffs main against the commit the last successful | |
| # scheduled run tested and routes through FILTERS like any other event. A job | |
| # is "queue" or "nightly", never both; check-ci-config.py enforces that. | |
| # | |
| # "push" is reserved for work that can only happen once a commit is on main. | |
| # Adding "push" back to a test job would make every merge run it twice, once | |
| # in the queue and once after, which is the thing the queue was adopted to | |
| # avoid. | |
| # | |
| # A pull request that targets a release branch (`branch-N.M`) is the one | |
| # exception: it runs every "pr", "queue" and "nightly" job. On main the tiers | |
| # spread the suites over three events, and every change still meets all of | |
| # them. A release branch has only the pull request: the merge queue covers the | |
| # default branch alone, ci.yml runs on push only for main, and GitHub fires | |
| # `schedule` only on the default branch. A tier there would not defer a suite, | |
| # it would drop it. And the tiers exist because of main's volume, which a | |
| # release branch does not have: branch-1.0 took 28 pull requests in its first | |
| # two months. "push" jobs (the site deploy) and label-only jobs (Spark 3.4) | |
| # keep their rules on a release branch. | |
| POLICY = { | |
| # The one test job that also runs on push to main, and only because of | |
| # actions/cache scoping: a pull request can restore caches saved on its | |
| # own branch or on main, and nowhere else. The queue runs on a throwaway | |
| # gh-readonly-queue/* branch, so whatever it saves is deleted with that | |
| # branch. Without a push run, a Cargo.lock or pom.xml change would leave | |
| # main's cargo-ci, Maven and TPC-H/TPC-DS caches stale | |
| # forever, and every later pull request would pay the delta on top of the | |
| # restore-keys prefix match. | |
| # | |
| # On push that is the *only* thing it is for. The queue already tested the | |
| # exact tree that landed, so re-running the lints and the linux-test | |
| # matrix there tests nothing, and they are 514 of the 587 runner-minutes a | |
| # push run costs. The split below keeps the cache writers on push and moves | |
| # everything else behind `build_linux_full`. | |
| "build_linux": ["pr", "queue", "push"], | |
| # The lints and the test matrix inside pr_build_linux.yml. Deliberately no | |
| # "push": ci.yml turns this output into the workflow's `cache-refresh-only` | |
| # input, so dropping "push" here is what trims the push tier down to the | |
| # jobs that write an actions/cache entry. See issue #5929. | |
| "build_linux_full": ["pr", "queue"], | |
| # The linux-test matrix's Spark profiles other than the default one. The | |
| # five profiles cost about the same each, roughly 2,300 runner-minutes a | |
| # day apiece on pull requests in mid-September 2026, and together they | |
| # were three quarters of the Linux build. A pull request and the queue run | |
| # the Comet test suites against Spark 4.1 only; the nightly run covers the | |
| # other four. The lint-java matrix still compiles every Spark profile on | |
| # every pull request, so what waits for the nightly is runtime behaviour, | |
| # not a shim that fails to build. ci.yml turns this output into the | |
| # workflow's `profiles` input. | |
| "build_linux_all_profiles": ["nightly", "label:run-all-spark-profiles"], | |
| # macOS runners are the scarcest capacity we have, and the Linux build | |
| # already covers rustfmt and the Rust/JVM compile on every PR. The label | |
| # is for a change that touches platform-specific code. | |
| # | |
| # Split like the Linux build: `build_macos` runs pr_build_macos.yml at all, | |
| # `build_macos_full` runs its test matrix too. Push to main sets only the | |
| # first, so the workflow runs in cache-refresh-only mode there and keeps | |
| # main's macOS cargo cache warm. Nothing else runs on main, and an entry | |
| # written from the queue's throwaway branch is visible to nobody, so | |
| # without the push run every queue build compiled the native library from | |
| # scratch (issue #6390). | |
| "build_macos": ["queue", "push", "label:run-macos-tests"], | |
| "build_macos_full": ["queue", "label:run-macos-tests"], | |
| # Benchmark sources are compiled and linted, never run, so a break there | |
| # cannot affect a PR's correctness verdict; the queue catches it. | |
| "benchmark": ["queue", "label:run-benchmark-check"], | |
| # The Delta build gate only proves a build-system property, and the | |
| # PyArrow suite builds Comet once per Spark 4.x version to drive a real | |
| # Python worker. Neither changes often enough to earn a PR-tier slot; the | |
| # queue catches a regression before it lands, and the label is the escape | |
| # hatch for a change to the surface they cover. | |
| "delta_gate": ["queue", "label:run-delta-build-gate"], | |
| "pyarrow_udf": ["queue", "label:run-pyarrow-udf-tests"], | |
| # docs deploys to asf-site, so it must not run from a pull request or from | |
| # the queue's throwaway branch. | |
| "docs": ["push"], | |
| # Spark 3.4 is deprecated, so it is the one test job outside the queue | |
| # tier: a failure there no longer blocks a merge. It stays runnable on | |
| # demand -- the label on a pull request, or a workflow_dispatch -- so | |
| # anyone who wants to check a change against 3.4 still can. | |
| "spark_3_4": ["label:run-spark-3.4-tests"], | |
| # Spark 4.1 is the default build profile and the one Spark SQL suite the | |
| # queue runs; 3.5, 4.0 and 4.2 run nightly, or on a pull request with | |
| # their label. | |
| "spark_3_5": ["nightly", "label:run-spark-3.5-tests"], | |
| "spark_4_0": ["nightly", "label:run-spark-4.0-tests"], | |
| # No Spark SQL suite runs on a plain pull request. Spark 4.1 was the last | |
| # one in the PR tier, first whole (issue #5870 pulled the sql_hive shards | |
| # out) and then catalyst and sql_core alone. What changed is how often a | |
| # pull request is pushed: with agent-driven review and agent-driven | |
| # replies to review, a PR now goes through several more rounds before it | |
| # is queued, and each round paid for the whole 4.1 build. The queue still | |
| # runs every shard before anything lands; the two labels bring the run | |
| # forward. `run-spark-4.1-tests` selects the whole suite, so it appears on | |
| # both outputs; `run-spark-4.1-hive-tests` selects only the hive shards. | |
| "spark_4_1": ["queue", "label:run-spark-4.1-tests"], | |
| "spark_4_1_hive": [ | |
| "queue", | |
| "label:run-spark-4.1-tests", | |
| "label:run-spark-4.1-hive-tests", | |
| ], | |
| # Spark 4.2 support is experimental, but the suite passes, so it sits in | |
| # the nightly tier with the other non-default versions rather than being | |
| # reachable only on demand. Nightly is the right tier for it twice over: | |
| # a 4.2 regression blocks nobody's merge, and running it every night is | |
| # what keeps the 4.2 diff in `dev/diffs` from silently rotting as the | |
| # other diffs are updated -- the failure mode an on-demand suite hides | |
| # until someone thinks to ask for it. | |
| "spark_4_2": ["nightly", "label:run-spark-4.2-tests"], | |
| # Same shape for Iceberg: 1.11 is the only Spark 4.1 coverage, so it is | |
| # the one Iceberg version the queue runs; the three older versions run | |
| # nightly. One label opts a pull request into all four. | |
| "iceberg_1_8": ["nightly", "label:run-iceberg-tests"], | |
| "iceberg_1_9": ["nightly", "label:run-iceberg-tests"], | |
| "iceberg_1_10": ["nightly", "label:run-iceberg-tests"], | |
| "iceberg_1_11": ["queue", "label:run-iceberg-tests"], | |
| # The Delta contrib suites all run nightly. The PR tier's strict warnings | |
| # job compiles the contrib against Spark 3.5, so a change that breaks its | |
| # build on Spark 3.5 fails on its own pull request. One label opts a pull | |
| # request into all three. | |
| "delta_3_5": ["nightly", "label:run-delta-tests"], | |
| "delta_4_0": ["nightly", "label:run-delta-tests"], | |
| "delta_4_1": ["nightly", "label:run-delta-tests"], | |
| } | |
| # Release branches are named `branch-<major>.<minor>`, as in branch-0.17 and | |
| # branch-1.0. A pull request against one runs these tiers; see the end of the | |
| # comment above POLICY. | |
| RELEASE_BRANCH = re.compile(r"branch-\d+\.\d+") | |
| RELEASE_BRANCH_TIERS = ("pr", "queue", "nightly") | |
| def gating_labels(job): | |
| return [t[len("label:"):] for t in POLICY[job] if t.startswith("label:")] | |
| def event_allows(job, event): | |
| """Does `event` permit `job` to run, ignoring which files changed? | |
| `event` is {"name", "action", "label", "labels", "base"}: the workflow event | |
| name, the pull_request action, the label just added on a `labeled` event, | |
| the labels currently on the pull request, and the branch it targets. | |
| """ | |
| tiers = POLICY[job] | |
| name = event.get("name") | |
| if name == "workflow_dispatch": | |
| return True | |
| if name == "push": | |
| return "push" in tiers | |
| if name == "merge_group": | |
| return "queue" in tiers | |
| if name == "schedule": | |
| return "nightly" in tiers | |
| if name != "pull_request": | |
| return False | |
| if RELEASE_BRANCH.fullmatch(event.get("base", "")): | |
| return release_branch_allows(job, event) | |
| gates = gating_labels(job) | |
| if gates: | |
| if not any(label in event.get("labels", []) for label in gates): | |
| return False | |
| elif "pr" not in tiers: | |
| return False | |
| # A `labeled` event fires at the same commit as the opened/synchronize run | |
| # that already tested it, and GitHub cannot filter a pull_request trigger | |
| # by label name. So on `labeled`, run only the job the new label gates; | |
| # everything else would be duplicating a pipeline. See issue #5007 for what | |
| # happens when this is expressed as a job-level `if:` instead. | |
| if event.get("action") == "labeled": | |
| return event.get("label") in gates | |
| return True | |
| def release_branch_allows(job, event): | |
| """event_allows for a pull request that targets a release branch.""" | |
| tiers = POLICY[job] | |
| gates = gating_labels(job) | |
| automatic = any(tier in tiers for tier in RELEASE_BRANCH_TIERS) | |
| # The opened/synchronize run already ran every automatic job at this | |
| # commit, so a `labeled` run adds only a job that nothing else runs there, | |
| # which leaves the label-only Spark 3.4 suite. | |
| if event.get("action") == "labeled": | |
| return not automatic and event.get("label") in gates | |
| return automatic or any(label in event.get("labels", []) for label in gates) | |
| def compute(files, event): | |
| """Return {job: bool}, folding the path filter and the event policy.""" | |
| return { | |
| name: event_allows(name, event) and matches(patterns, files) | |
| for name, patterns in FILTERS.items() | |
| } | |
| def event_from_env(): | |
| labels = os.environ.get("PR_LABELS", "") | |
| return { | |
| "name": os.environ.get("EVENT_NAME", ""), | |
| "action": os.environ.get("EVENT_ACTION", ""), | |
| "label": os.environ.get("LABEL_NAME", ""), | |
| "labels": json.loads(labels) if labels.strip() else [], | |
| "base": os.environ.get("PR_BASE_REF", ""), | |
| } | |
| def glob_to_regex(pat): | |
| # Translate a picomatch-style glob to a regex. "**/" at the start or | |
| # interior is optional ("(?:.*/)?") so that "**/pom.xml" matches at the | |
| # repo root; bare "**" is greedy across path separators. | |
| out = [] | |
| i = 0 | |
| while i < len(pat): | |
| c = pat[i] | |
| if c == "*" and i + 1 < len(pat) and pat[i + 1] == "*": | |
| if i + 2 < len(pat) and pat[i + 2] == "/": | |
| out.append("(?:.*/)?") | |
| i += 3 | |
| else: | |
| out.append(".*") | |
| i += 2 | |
| elif c == "*": | |
| out.append("[^/]*") | |
| i += 1 | |
| elif c == "?": | |
| out.append("[^/]") | |
| i += 1 | |
| elif c in r".+(){}[]^$|\\": | |
| out.append("\\" + c) | |
| i += 1 | |
| else: | |
| out.append(c) | |
| i += 1 | |
| return "^" + "".join(out) + "$" | |
| def matches(patterns, files): | |
| includes = [re.compile(glob_to_regex(p)) for p in patterns if not p.startswith("!")] | |
| excludes = [re.compile(glob_to_regex(p[1:])) for p in patterns if p.startswith("!")] | |
| for f in files: | |
| if any(r.match(f) for r in includes) and not any(r.match(f) for r in excludes): | |
| return True | |
| return False | |
| if __name__ == "__main__": | |
| event = event_from_env() | |
| # workflow_dispatch has no meaningful base to diff against, so the caller | |
| # passes an empty list and every path filter is treated as matched. | |
| if event["name"] == "workflow_dispatch": | |
| for name in FILTERS: | |
| print(f"{name}=true") | |
| sys.exit(0) | |
| files_path = Path(sys.argv[1]) | |
| files = [line.strip() for line in files_path.read_text().splitlines() if line.strip()] | |
| for name, flag in compute(files, event).items(): | |
| print(f"{name}={'true' if flag else 'false'}") |