)]}' { "log": [ { "commit": "38c3f8bce35642479117ba49f44a02fee31e6158", "tree": "b9624aff838d4ebf2a4bec8a380f5876c9d3ba11", "parents": [ "61b61de68c0739416ebd5e94acd08a8d5adc9083" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 29 16:52:46 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 29 16:52:46 2026 -0600" }, "message": "fix: throw ARITHMETIC_OVERFLOW for Long.MinValue div -1 under ANSI mode (#5084)\n\n* fix: throw ARITHMETIC_OVERFLOW for Long.MinValue div -1 under ANSI mode\n\n* refactor: reuse Spark\u0027s checkDivideOverflow and dedupe overflow check\n\n* test: use ARITHMETIC_OVERFLOW condition name in integral divide expect_error\n\nMatches the convention used by the other expect_error queries in\narithmetic_ansi.sql. Verified against the spark-3.4, spark-3.5,\nspark-4.0, spark-4.1, and spark-4.2 profiles." }, { "commit": "61b61de68c0739416ebd5e94acd08a8d5adc9083", "tree": "b4b658d57293e579bd4baa24842027a841bf0602", "parents": [ "cde6052c534b8410ae2f372d4a0df543bffa4c74" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 29 15:49:38 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 29 15:49:38 2026 -0600" }, "message": "chore: stop enabling incompatible casts in plan stability suite (#5139)\n\n* stop enabling incompatible casts in plan stability suite\n\n* fix: remove unused Cast import in CometPlanStabilitySuite" }, { "commit": "cde6052c534b8410ae2f372d4a0df543bffa4c74", "tree": "30fe6f4fb955cc184eb339a38f76dc02491da8d9", "parents": [ "a1b022b52a5815672ec650c0b8f689e91ea87517" ], "author": { "name": "Matt Butrovich", "email": "mbutrovich@users.noreply.github.com", "time": "Wed Jul 29 17:16:02 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 29 15:16:02 2026 -0600" }, "message": "fix: work around DataFusion 54.1.0 Parquet page-index regression (#5132)" }, { "commit": "a1b022b52a5815672ec650c0b8f689e91ea87517", "tree": "b46996a9584bd1d966fa028bca1582442780b5a2", "parents": [ "76b9d2eee42929a885023e7602abf8410250d7dd" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 29 12:38:25 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 29 12:38:25 2026 -0600" }, "message": "fix: honor fail_on_error in native make_decimal (#5080)\n\n* fix: honor fail_on_error in native make_decimal\n\nMakeDecimal with nullOnOverflow\u003dfalse (ANSI mode) must throw\nNUMERIC_VALUE_OUT_OF_RANGE on overflow, matching Spark\u0027s Decimal.set.\nThe serde already serialized the flag but the native registration\ndiscarded it, so Comet returned NULL where Spark throws.\n\n* refactor: simplify long_to_decimal with map_or" }, { "commit": "76b9d2eee42929a885023e7602abf8410250d7dd", "tree": "7fae4e363248c493363f03b971e30aeb202e4757", "parents": [ "038fd6e25c365b0076db91c0c5927541bba4d37b" ], "author": { "name": "Oleks V", "email": "comphead@users.noreply.github.com", "time": "Wed Jul 29 11:26:15 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 29 11:26:15 2026 -0700" }, "message": "feat: add codegen dispatch fallback for CometCast incompatible/unsupported cases (including legacy config paths) (#5079)" }, { "commit": "038fd6e25c365b0076db91c0c5927541bba4d37b", "tree": "dabca89df08d01a2ae8b2729a9872a1dceb6ab55", "parents": [ "e344f7490a3db8a07e147c57fdecc7bbe74f3dd6" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 29 10:01:36 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 29 10:01:36 2026 -0600" }, "message": "fix: honour Spark\u0027s legacy `null IN ()` behavior (#5127)\n\n* fix: honour Spark\u0027s legacy `null IN ()` behavior\n\nComet\u0027s native `in` kernel always returns `false` for an operand against an\nempty `IN` list. Spark only does that under the non-legacy behavior; under the\nlegacy behavior a `NULL` operand evaluates to `NULL` (SPARK-44550). That legacy\nbehavior is in effect more often than the config name suggests: always on Spark\n3.4 (no config), by default on Spark 3.5, and whenever ANSI mode is disabled on\nSpark 4.0+, where `spark.sql.legacy.nullInEmptyListBehavior` defaults to\n`!spark.sql.ansi.enabled`.\n\nMark the empty-list case `Unsupported` in `CometIn` / `CometInSet` when the\nlegacy behavior applies. Both serdes mix in `CodegenDispatchFallback`, so these\ncases run Spark\u0027s own `doGenCode` inside the Comet pipeline rather than falling\nthe whole operator back to Spark. `CometNot`\u0027s `Not(In(...))` fast path is\nexcluded for the same case so it defers to `CometIn`.\n\n`OptimizeIn` folds empty `IN` lists away, so this only affects plans where that\nrule is excluded, and non-empty `IN` lists are unchanged.\n\nCloses #4786\n\n* refactor: dedup the In/InSet support gates\n\n- Share the empty-list + collation gate between `CometIn` and `CometInSet` via\n `ComparisonUtils.inSupportLevel` / `inUnsupportedReasons` instead of two\n copies that must be edited in lockstep.\n- Read the legacy config through the existing `CometConf.getBooleanConf` helper\n rather than hand-rolling the string-key read, and make the resolver private.\n- Have `CometNot`\u0027s fused fast paths consult the child serde\u0027s `getSupportLevel`\n instead of re-checking its conditions by hand, so gates added to `CometIn` or\n `CollationAwareBinaryPredicate` no longer need mirroring here.\n- Test: use `Rule.ruleName` for the excluded optimizer rules so a Spark rename\n cannot silently stop exercising the case." }, { "commit": "e344f7490a3db8a07e147c57fdecc7bbe74f3dd6", "tree": "42c85f5a57ff141b4aaea3012c745a0713a2ad60", "parents": [ "ce84a24c198688d980b5ff3493c74fc91764215f" ], "author": { "name": "Matt Butrovich", "email": "mbutrovich@users.noreply.github.com", "time": "Wed Jul 29 09:59:21 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 29 09:59:21 2026 -0400" }, "message": "docs: update post 1.0.0 roadmap (#5064)" }, { "commit": "ce84a24c198688d980b5ff3493c74fc91764215f", "tree": "d7460723e0c26225e8258789de832c2792b6332e", "parents": [ "1db954d08580c725c7e3b2906a0ae1c887f52f9a" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 28 18:48:58 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 28 18:48:58 2026 -0600" }, "message": "docs: document known correctness issues in the compatibility guide (#5085)\n\nAdds compat notes and expression-serde `getCompatibleNotes` /\n`getIncompatibleReasons` entries for each open `correctness`+`bug` issue,\nso the auto-generated compatibility guide surfaces them:\n\n- Expression-level (per-category pages):\n - `Round` (Long/scale\u003c\u003d-19 overflow) #5070\n - `IntegralDivide` (Long.MinValue div -1) #5065\n - `MakeDecimal` (ignores fail_on_error) #5066\n - `NextDay`, `MakeDate` (wrong ANSI exception class) #5073\n - `MapFromArrays`, `MapFromEntries` (null key not rejected) #4680\n - `GetJsonObject` (duplicate keys) #4947\n\n- Static sections in `compatibility/index.md`:\n - ANSI-mode error classes / messages (#5071, #5072, #5073, #4967)\n - Known result-value divergences (boolean-\u003edecimal cast #5068,\n string-\u003eboolean ISO controls #4959, RANGE window overflow #5022,\n null IN () legacy behavior #4786)\n - Object store cache key drops ABFS container #4993\n\n- `compatibility/scans.md`: expand the datetime-rebase note (wider than\n \"written before Spark 3.0\", `datetimeRebaseModeInRead` is not honoured,\n `spark.comet.exceptionOnDatetimeRebase` is currently dead code) and\n link the tracker #5010.\n\nNot covered here (already handled elsewhere):\n- #5067 (Remainder Float/Double NaN) - fixed by 9bd5a6d05\n- #4764 (EPIC UTF-8 ingress) - already documented in index.md" }, { "commit": "1db954d08580c725c7e3b2906a0ae1c887f52f9a", "tree": "d70fa46bad73154b2d7c6baf0597bc68dd908268", "parents": [ "cf81ab37a45d2a423a4fef9692e444968e40eb2a" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 28 17:34:44 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 28 17:34:44 2026 -0600" }, "message": "fix: resolve Comet jar without hardcoding the SNAPSHOT qualifier (#5108)\n\nLocate built jars by version-agnostic glob so they still resolve on a release\nbranch, and document the version-bump checks in the release process guide." }, { "commit": "cf81ab37a45d2a423a4fef9692e444968e40eb2a", "tree": "5477887761c128582595ec45050c877d292627c9", "parents": [ "880cecfcacdf292555727a0ef765943ffca1bf5a" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 28 17:34:21 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 28 17:34:21 2026 -0600" }, "message": "test: add isolated and small-batch benchmarks for columnar-to-row conversion (#5113)\n\nAdd CometC2RIsolatedBench, which measures columnar-to-row conversion in\nisolation on in-memory Arrow batches at batch sizes 8192, 512, and 32,\ncomparing the JVM rowIterator plus UnsafeProjection path against\nNativeColumnarToRowConverter. The existing end-to-end benchmark is\ndominated by the parquet scan and cannot observe conversion cost.\n\nAlso extend CometColumnarToRowBenchmark with two scenarios that the\nexisting groups do not cover: dictionary-encoded parquet input and a\nScala UDF consumer that defeats escape analysis, and disable ANSI mode\nin its session so the data generation queries no longer fail with a\ncast overflow on Spark 4.x.\n\nPart of #5112" }, { "commit": "880cecfcacdf292555727a0ef765943ffca1bf5a", "tree": "4f28e60549617f14562c5b00bf6dfe06dd585e1f", "parents": [ "aebe70ba93f5dda376e936fe0364d679b0658211" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 28 12:22:39 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 28 12:22:39 2026 -0600" }, "message": "docs: expand tuning guide with performance and memory configs (#4908)\n\n* refactor: unify shuffle configs under `spark.comet.shuffle.*` prefix\n\nShuffle-related configs were spread across four disjoint prefixes:\n`spark.comet.exec.shuffle.*`, `spark.comet.columnar.shuffle.*`,\n`spark.comet.native.shuffle.*`, and `spark.comet.shuffle.*`. This\nconsolidates all shuffle configs under `spark.comet.shuffle.*` with\n`.columnar.` and `.native.` sub-namespaces for mode-specific settings.\n\nEvery renamed key is registered via `.withAlternative(...)`, so\nexisting user configurations continue to work (a deprecation warning\nis logged when an old key is read). Per-mode leaves that were\npreviously misplaced move to their honest namespace:\n\n - `spark.comet.exec.shuffle.writeBufferSize` (native-only) becomes\n `spark.comet.shuffle.native.writeBufferSize`.\n - `spark.comet.shuffle.preferDictionary.ratio` (JVM-only) becomes\n `spark.comet.shuffle.columnar.preferDictionary.ratio`.\n\nThe `spark.comet.columnar.shuffle.*` keys also fix Category 2 in\napache/datafusion-comet#4978 (dots-in-segment): `spill.threshold`,\n`memory.factor`, and `batch.size` become `spillThreshold`,\n`memoryFactor`, and `batchSize`.\n\nPart of apache/datafusion-comet#4978.\n\n* refactor: rename `.columnar.` sub-namespace to `.jvm.`\n\nThe JVM-side shuffle path in Comet is not the only columnar shuffle —\nnative shuffle is also columnar (both use Arrow). The distinguishing\naxis is *where* the shuffle logic runs. Renaming the sub-namespace\nto `.jvm.` aligns it with the mode value users already set:\n`spark.comet.shuffle.mode \u003d jvm`.\n\nKeys renamed:\n\n - spark.comet.shuffle.columnar.spillThreshold → spark.comet.shuffle.jvm.spillThreshold\n - spark.comet.shuffle.columnar.memoryFactor → spark.comet.shuffle.jvm.memoryFactor\n - spark.comet.shuffle.columnar.batchSize → spark.comet.shuffle.jvm.batchSize\n - spark.comet.shuffle.columnar.preferDictionary.ratio → spark.comet.shuffle.jvm.preferDictionary.ratio\n\nThe pre-existing deprecated aliases (`spark.comet.columnar.shuffle.*`)\nare untouched — they were never `.columnar.`-scoped under the new\nprefix. Val names follow: `COMET_SHUFFLE_COLUMNAR_*` → `COMET_SHUFFLE_JVM_*`.\n\nThe Scala/Java class names (`CometColumnarShuffle`, `CometColumnarShuffleSuite`, etc.)\nstill use \"columnar\" and are left alone — that is a separate rename to\nconsider once the config vocabulary settles.\n\n* chore: re-run prettier on shuffle docs\n\nTable column alignment tweaks after the `.columnar.` → `.jvm.` key\nrename. Content unchanged.\n\n* style: apply spotless formatting to CometShuffleExternalSorter\n\n* refactor: move maxBufferBytes under spark.comet.shuffle.native prefix\n\n* test: update Spark diffs for renamed shuffle configs\n\nRegenerate the Spark SQL test diffs to reference shuffle configs by their\nstring keys rather than CometConf constants, so the config rename in this\nPR no longer breaks test compilation.\n\n- spark.comet.exec.shuffle.enabled -\u003e spark.comet.shuffle.enabled\n- CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -\u003e\n \"spark.comet.shuffle.native.partitioning.range.enabled\"\n\n* docs: expand tuning guide with performance and memory configs for 1.0\n\nAudited all CometConf entries and added documentation for user-facing\nperformance and memory knobs that were undocumented in the tuning guide:\nbatch size, spill disk cap, Parquet parallel I/O and filter pushdown,\nIceberg data-file concurrency, shuffle compression codec/level,\nrow/columnar transition revert, and metrics overhead. Also fixed an\nincorrect claim about the default shuffle compression codec (LZ4, not\nZSTD) and moved spark.comet.maxTempDirectorySize into the tuning\ncategory so it renders in the Memory \u0026 Tuning table.\n\n* style: apply spotless formatting to COMET_MAX_TEMP_DIRECTORY_SIZE doc string\n\n* docs: address review feedback on tuning guide\n\n- Remove ### Parallel I/O subsection: the four spark.comet.parquet.read.parallel.io.*\n and .mergeRanges* configs were removed in #4981\n- Fix maxTempDirectorySize scope: limit is per Spark task, not per executor\n- Move ### Parquet Native Scans (merged in from main) under new\n ## Parquet Reader Tuning h2 so all Parquet guidance is co-located\n- Promote ### Reducing Row/Columnar Conversion Overhead to its own h2 since\n spark.comet.exec.transitionRevert.* counts C2R transitions across the whole\n stage and is not shuffle-specific\n- Clarify shuffle compression: spark.comet.exec.shuffle.compression.codec\n governs both native and JVM columnar shuffle; spark.shuffle.compress\u003dfalse\n disables compression only for the native shuffle\n- Fix Iceberg wording: spark.comet.scan.icebergNative.enabled defaults to true\n\n* docs: use renamed shuffle config keys in tuning guide\n\n---------\n\nCo-authored-by: Matt Butrovich \u003cmbutrovich@users.noreply.github.com\u003e" }, { "commit": "aebe70ba93f5dda376e936fe0364d679b0658211", "tree": "46d15c5933f2f330a2281c3977a815c50f56f18e", "parents": [ "5fd68c219a1df2f7d5b1e12be5c23ea974cfea83" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 28 11:19:26 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 28 11:19:26 2026 -0600" }, "message": "docs: define post-1.0 versioning policy and legacy config process (#5056)\n\n* docs: define post-1.0 versioning policy and legacy config process\n\nComet is a plugin rather than a library, so its configuration is its primary\nAPI surface. Rewrite the versioning policy to be 1.0-forward: define the\ncompatibility surface as spark.comet.* keys, an enumerated public Java and\nScala API, and Compatible query results; allow behavior changes in a minor\nrelease only when a spark.comet.legacy.* key restores the previous behavior;\nexempt correctness fixes from being breaking changes; and place Spark version\nsupport outside semantic versioning.\n\nEnumerate the public API explicitly rather than leaving it implied, and mark it\nin source with a new org.apache.comet.annotation.Public annotation: the class\nnames named in Spark config properties (CometPlugin, ExtendedExplainInfo) and\nthe org.apache.comet.cloud.s3 credential provider SPI that vendors compile\nagainst. Everything else is internal whatever its access modifier says.\nCometPublicApiSuite pins the annotated set to the list in the policy, so\ngrowing or shrinking the public API cannot happen by accident.\n\nAdd a user-facing upgrade guide that records behavior changes per release, and\ndocument the legacy config recipe in the contributor guide alongside the\nexisting rename process.\n\n* docs: state versioning policy as effective from 1.0.0\n\n* fix: materialize class-name walk strictly for Scala 2.12\n\nOn Scala 2.12 Iterator.toSeq is toStream, so the file walk in\nCometPublicApiSuite was only forced after the finally block closed it,\nfailing with IllegalStateException. Scala 2.13 toSeq is strict, so only\nthe Spark 3.4 / Scala 2.12 job was affected." }, { "commit": "5fd68c219a1df2f7d5b1e12be5c23ea974cfea83", "tree": "65f3ec264237038dbc1d7962f132965831a8703e", "parents": [ "f4ae9e026f38e46dbfbe55e44c23a7177a9951c8" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 28 11:18:01 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 28 11:18:01 2026 -0600" }, "message": "docs: fix references to configuration keys that do not exist (#5063)\n\n* refactor: unify shuffle configs under `spark.comet.shuffle.*` prefix\n\nShuffle-related configs were spread across four disjoint prefixes:\n`spark.comet.exec.shuffle.*`, `spark.comet.columnar.shuffle.*`,\n`spark.comet.native.shuffle.*`, and `spark.comet.shuffle.*`. This\nconsolidates all shuffle configs under `spark.comet.shuffle.*` with\n`.columnar.` and `.native.` sub-namespaces for mode-specific settings.\n\nEvery renamed key is registered via `.withAlternative(...)`, so\nexisting user configurations continue to work (a deprecation warning\nis logged when an old key is read). Per-mode leaves that were\npreviously misplaced move to their honest namespace:\n\n - `spark.comet.exec.shuffle.writeBufferSize` (native-only) becomes\n `spark.comet.shuffle.native.writeBufferSize`.\n - `spark.comet.shuffle.preferDictionary.ratio` (JVM-only) becomes\n `spark.comet.shuffle.columnar.preferDictionary.ratio`.\n\nThe `spark.comet.columnar.shuffle.*` keys also fix Category 2 in\napache/datafusion-comet#4978 (dots-in-segment): `spill.threshold`,\n`memory.factor`, and `batch.size` become `spillThreshold`,\n`memoryFactor`, and `batchSize`.\n\nPart of apache/datafusion-comet#4978.\n\n* refactor: rename `.columnar.` sub-namespace to `.jvm.`\n\nThe JVM-side shuffle path in Comet is not the only columnar shuffle —\nnative shuffle is also columnar (both use Arrow). The distinguishing\naxis is *where* the shuffle logic runs. Renaming the sub-namespace\nto `.jvm.` aligns it with the mode value users already set:\n`spark.comet.shuffle.mode \u003d jvm`.\n\nKeys renamed:\n\n - spark.comet.shuffle.columnar.spillThreshold → spark.comet.shuffle.jvm.spillThreshold\n - spark.comet.shuffle.columnar.memoryFactor → spark.comet.shuffle.jvm.memoryFactor\n - spark.comet.shuffle.columnar.batchSize → spark.comet.shuffle.jvm.batchSize\n - spark.comet.shuffle.columnar.preferDictionary.ratio → spark.comet.shuffle.jvm.preferDictionary.ratio\n\nThe pre-existing deprecated aliases (`spark.comet.columnar.shuffle.*`)\nare untouched — they were never `.columnar.`-scoped under the new\nprefix. Val names follow: `COMET_SHUFFLE_COLUMNAR_*` → `COMET_SHUFFLE_JVM_*`.\n\nThe Scala/Java class names (`CometColumnarShuffle`, `CometColumnarShuffleSuite`, etc.)\nstill use \"columnar\" and are left alone — that is a separate rename to\nconsider once the config vocabulary settles.\n\n* chore: re-run prettier on shuffle docs\n\nTable column alignment tweaks after the `.columnar.` → `.jvm.` key\nrename. Content unchanged.\n\n* style: apply spotless formatting to CometShuffleExternalSorter\n\n* refactor: move maxBufferBytes under spark.comet.shuffle.native prefix\n\n* test: update Spark diffs for renamed shuffle configs\n\nRegenerate the Spark SQL test diffs to reference shuffle configs by their\nstring keys rather than CometConf constants, so the config rename in this\nPR no longer breaks test compilation.\n\n- spark.comet.exec.shuffle.enabled -\u003e spark.comet.shuffle.enabled\n- CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -\u003e\n \"spark.comet.shuffle.native.partitioning.range.enabled\"\n\n* docs: refresh stale issue references and normalize issue link format (#5062)\n\nAudit the user guide and contributor guide for references to issues that\nhave since been closed, and make the remaining issue and PR links use a\nsingle format.\n\nThirteen places described a limitation that no longer exists, verified\nagainst the current serde and native code rather than the issue state\nalone: size() over MapType, try_mod / EvalMode.TRY, CAST(map AS map),\nspark.sql.legacy.castComplexTypesToString, replace with an empty search\nstring, initcap, str_to_map with the legacy truncate flag, AVG(decimal)\nover a window, Percentile, the pyarrow-udf row round-trip, the\ndatafusion-spark migration epic, the awslabs TPC-DS epic, and the\ninterval type epic.\n\nAlso drop or repoint links that pointed at a closed or unrelated tracker:\nthe #4098 rows in expressions.md, translate, the TimestampNTZ scan note,\nand the from_utc_timestamp timezone-parser note.\n\nConvert bare URLs and bare #NNNN mentions to inline [#NNNN](url), keeping\nreference-style definitions where a file already used them, and point URLs\nat /pull/ where the number is a pull request.\n\n* docs: fix references to configuration keys that do not exist\n\nSeveral docs referenced configuration keys that were never registered or were\nrenamed. Corrected the expression/operator opt-in prefix (spark.comet.expr.* -\u003e\nspark.comet.expression.* / spark.comet.operator.*), removed references to a\nglobal allowIncompatible key that does not exist, and updated stale shuffle,\nmemory pool, and metrics keys." }, { "commit": "f4ae9e026f38e46dbfbe55e44c23a7177a9951c8", "tree": "d5e6176b420600431f6f50951468a9f335996c85", "parents": [ "9e9e3b895cc090f1b950392d8dced9fdeae55742" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 28 10:07:19 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 28 12:07:19 2026 -0400" }, "message": "refactor: unify shuffle configs under `spark.comet.shuffle.*` prefix (#4986)\n\n* refactor: unify shuffle configs under `spark.comet.shuffle.*` prefix\n\nShuffle-related configs were spread across four disjoint prefixes:\n`spark.comet.exec.shuffle.*`, `spark.comet.columnar.shuffle.*`,\n`spark.comet.native.shuffle.*`, and `spark.comet.shuffle.*`. This\nconsolidates all shuffle configs under `spark.comet.shuffle.*` with\n`.columnar.` and `.native.` sub-namespaces for mode-specific settings.\n\nEvery renamed key is registered via `.withAlternative(...)`, so\nexisting user configurations continue to work (a deprecation warning\nis logged when an old key is read). Per-mode leaves that were\npreviously misplaced move to their honest namespace:\n\n - `spark.comet.exec.shuffle.writeBufferSize` (native-only) becomes\n `spark.comet.shuffle.native.writeBufferSize`.\n - `spark.comet.shuffle.preferDictionary.ratio` (JVM-only) becomes\n `spark.comet.shuffle.columnar.preferDictionary.ratio`.\n\nThe `spark.comet.columnar.shuffle.*` keys also fix Category 2 in\napache/datafusion-comet#4978 (dots-in-segment): `spill.threshold`,\n`memory.factor`, and `batch.size` become `spillThreshold`,\n`memoryFactor`, and `batchSize`.\n\nPart of apache/datafusion-comet#4978.\n\n* refactor: rename `.columnar.` sub-namespace to `.jvm.`\n\nThe JVM-side shuffle path in Comet is not the only columnar shuffle —\nnative shuffle is also columnar (both use Arrow). The distinguishing\naxis is *where* the shuffle logic runs. Renaming the sub-namespace\nto `.jvm.` aligns it with the mode value users already set:\n`spark.comet.shuffle.mode \u003d jvm`.\n\nKeys renamed:\n\n - spark.comet.shuffle.columnar.spillThreshold → spark.comet.shuffle.jvm.spillThreshold\n - spark.comet.shuffle.columnar.memoryFactor → spark.comet.shuffle.jvm.memoryFactor\n - spark.comet.shuffle.columnar.batchSize → spark.comet.shuffle.jvm.batchSize\n - spark.comet.shuffle.columnar.preferDictionary.ratio → spark.comet.shuffle.jvm.preferDictionary.ratio\n\nThe pre-existing deprecated aliases (`spark.comet.columnar.shuffle.*`)\nare untouched — they were never `.columnar.`-scoped under the new\nprefix. Val names follow: `COMET_SHUFFLE_COLUMNAR_*` → `COMET_SHUFFLE_JVM_*`.\n\nThe Scala/Java class names (`CometColumnarShuffle`, `CometColumnarShuffleSuite`, etc.)\nstill use \"columnar\" and are left alone — that is a separate rename to\nconsider once the config vocabulary settles.\n\n* chore: re-run prettier on shuffle docs\n\nTable column alignment tweaks after the `.columnar.` → `.jvm.` key\nrename. Content unchanged.\n\n* style: apply spotless formatting to CometShuffleExternalSorter\n\n* refactor: move maxBufferBytes under spark.comet.shuffle.native prefix\n\n* test: update Spark diffs for renamed shuffle configs\n\nRegenerate the Spark SQL test diffs to reference shuffle configs by their\nstring keys rather than CometConf constants, so the config rename in this\nPR no longer breaks test compilation.\n\n- spark.comet.exec.shuffle.enabled -\u003e spark.comet.shuffle.enabled\n- CometConf.COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED.key -\u003e\n \"spark.comet.shuffle.native.partitioning.range.enabled\"" }, { "commit": "9e9e3b895cc090f1b950392d8dced9fdeae55742", "tree": "4326a492e374d1ba2e8de9c2345807473200c7eb", "parents": [ "6bfd5450afdc2c722552673c66fa3b26cc47c71f" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 28 09:28:29 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 28 09:28:29 2026 -0600" }, "message": "fix: make native pow implementation compatible with Spark (#5033)\n\n* feat: native pow implementation compatible with Spark\n\n* fix: match Java Math.pow for |base| \u003d\u003d 1 with non-finite exponent\n\nJava\u0027s Math.pow returns NaN when the base has magnitude 1 and the\nexponent is infinite or NaN, whereas C99 pow (Rust\u0027s f64::powf) returns\n1. Special-case this in spark_pow so the native result matches Spark.\n\nExpand the pow.sql edge cases to cover signed zero, negative infinity,\nsubnormals, and the |base| \u003d\u003d 1 non-finite exponent divergence, and\ncompare them exactly. The tolerance path skips NaN rows and ignores the\nsign of Infinity, so it would pass vacuously on these cases. Add matching\nRust unit tests covering every argument-shape code path.\n\n* test: canonicalize NaN when comparing Comet and Spark answers\n\nThe exact comparison over pow(base, exp) failed on the Linux Spark 3.4,\n3.5 and 4.0 jobs while passing on 4.1, 4.2 and macOS. Each side returned\n27 identical-looking rows, and no row was marked as differing, because\nthe only difference was in NaN payloads: QueryTest.compare compares\ndoubles by raw bits, and Double.toString is injective on non-NaN values.\n\nNaN payloads are not specified. Math.pow may return any NaN and the JVM\nproduces different payloads on different architectures, which is why the\nsame commit passed on macOS. Spark fixed this in compare itself for\n4.1.2 and 4.2.0, and the versions that still compare raw bits are\nexactly the ones that failed.\n\nCanonicalize NaN in normalizeForComparison, which already walks nested\nrows, arrays and maps, so every supported Spark version asserts NaN-ness\nwithout asserting the payload. Signed zero is left alone since\ndistinguishing it is why the raw-bit comparison exists. This also closes\na coverage gap: tolerance comparison makes no assertion at all for NaN,\nso tests no longer have to choose between skipping NaN and depending on\nits payload.\n\nWith that in place pow.sql keeps a single exact query over all rows." }, { "commit": "6bfd5450afdc2c722552673c66fa3b26cc47c71f", "tree": "d79062e314a8c0b7ebee56284ccc2e941c310a60", "parents": [ "c27eed3398cd48fe8c87c417959ce28eaca5138c" ], "author": { "name": "Scott Schenkein", "email": "schenksj@yahoo.com", "time": "Tue Jul 28 11:25:53 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 28 09:25:53 2026 -0600" }, "message": "fix: materialize ConstantColumnVector on Comet\u0027s serialize/export paths (#4532)" }, { "commit": "c27eed3398cd48fe8c87c417959ce28eaca5138c", "tree": "4d1d6fa90ff5bfc962d9991ecde7c3680af655b5", "parents": [ "2c05a1598f62064e2c93ebb55e0ce4c591b9909f" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 28 07:28:42 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 28 09:28:42 2026 -0400" }, "message": "docs: refresh stale issue references and normalize issue link format (#5062)\n\nAudit the user guide and contributor guide for references to issues that\nhave since been closed, and make the remaining issue and PR links use a\nsingle format.\n\nThirteen places described a limitation that no longer exists, verified\nagainst the current serde and native code rather than the issue state\nalone: size() over MapType, try_mod / EvalMode.TRY, CAST(map AS map),\nspark.sql.legacy.castComplexTypesToString, replace with an empty search\nstring, initcap, str_to_map with the legacy truncate flag, AVG(decimal)\nover a window, Percentile, the pyarrow-udf row round-trip, the\ndatafusion-spark migration epic, the awslabs TPC-DS epic, and the\ninterval type epic.\n\nAlso drop or repoint links that pointed at a closed or unrelated tracker:\nthe #4098 rows in expressions.md, translate, the TimestampNTZ scan note,\nand the from_utc_timestamp timezone-parser note.\n\nConvert bare URLs and bare #NNNN mentions to inline [#NNNN](url), keeping\nreference-style definitions where a file already used them, and point URLs\nat /pull/ where the number is a pull request." }, { "commit": "2c05a1598f62064e2c93ebb55e0ce4c591b9909f", "tree": "5cf7401ac97444bdd72d5ed98395beb2abb87f11", "parents": [ "1d3044f35bda457a20f1ccf4324a4699b35f5158" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 28 07:03:29 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 28 07:03:29 2026 -0600" }, "message": "refactor: group orphan `spark.comet.explain.*` configs under one prefix (#5026)" }, { "commit": "1d3044f35bda457a20f1ccf4324a4699b35f5158", "tree": "d05d9111c453e0f595a5d5e750766f38506dce8d", "parents": [ "82180b87a6780d85500e381c442b7792652ecbd2" ], "author": { "name": "Peter Lee", "email": "peterxcli@gmail.com", "time": "Tue Jul 28 10:06:32 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jul 27 20:06:32 2026 -0600" }, "message": "feat: support interval codegen dispatch for nested values and native shuffle (#4976)" }, { "commit": "82180b87a6780d85500e381c442b7792652ecbd2", "tree": "2e4e8245e35c2bdd001f056f15c7d752d5ae0471", "parents": [ "41cb698da868aee230f70caf6fee4471b4350d29" ], "author": { "name": "Matt Butrovich", "email": "mbutrovich@users.noreply.github.com", "time": "Mon Jul 27 15:18:02 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jul 27 15:18:02 2026 -0400" }, "message": "docs: avoid \"not a blocker\" comments in reviews for review skill (#5057)" }, { "commit": "41cb698da868aee230f70caf6fee4471b4350d29", "tree": "0be1e695ae78e1fa45cf326562fe4dc64c6cb1fd", "parents": [ "8a9473c5690be2716a5c8a6ae2d83605dbd10539" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Mon Jul 27 11:27:28 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jul 27 11:27:28 2026 -0600" }, "message": "docs: add blog posts and talks page (#5043)" }, { "commit": "8a9473c5690be2716a5c8a6ae2d83605dbd10539", "tree": "3250f340bc1886d6d0308e50d180f5260bbf2a96", "parents": [ "69df0f12b2925d6416b2abf8750688a2ccec177b" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Sun Jul 26 11:07:11 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sun Jul 26 11:07:11 2026 -0600" }, "message": "feat: native randstr implementation compatible with Spark (#5035)\n\n* feat: native randstr implementation compatible with Spark\n\n* refactor: gate randstr shape restrictions in getSupportLevel, avoid per-row UTF-8 scan\n\nMove the literal-length/literal-seed/non-negative-length restrictions from convert\ninto getSupportLevel + getUnsupportedReasons so they surface in EXPLAIN and the\ncompatibility docs, matching the rand/randn serde pattern. Build each string in a\nreused String buffer instead of validating a byte buffer per row.\n\n* test: add golden-value, partition-index, and filter coverage for randstr\n\nAddress review feedback on the randstr PR:\n\n- Rust golden-value test asserting bit-for-bit equality with Spark 4.1.1\n across positive, zero, and negative seeds, independent of the SQL tests.\n- Rust partition-index test exercising the seed + partition_index arithmetic.\n- Broaden the zero-length test across a wider seed set (seed is irrelevant\n when length is zero).\n- Rust large-length smoke test guarding the builder capacity math, and\n saturating_mul on the capacity hint to avoid overflow on huge lengths.\n- SQL test asserting an Int seed and its Long literal produce identical\n output (serde toLong sign extension).\n- SQL test running randstr through a native projection feeding a filter." }, { "commit": "69df0f12b2925d6416b2abf8750688a2ccec177b", "tree": "813cf5d9978ed44998e79aafb6ceb2c41347d36b", "parents": [ "c7657745f0f95f3f26f1634dcec94a47f1fe0a91" ], "author": { "name": "Peter Lee", "email": "peterxcli@gmail.com", "time": "Sat Jul 25 20:47:41 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Jul 25 06:47:41 2026 -0600" }, "message": "feat: add CalendarIntervalType support (#4898)\n\n* Add CalendarIntervalType Arrow support\n\n* Support interval vectors in UDF and shuffle codegen\n\n* remove stale test\n\n* spotless apply" }, { "commit": "c7657745f0f95f3f26f1634dcec94a47f1fe0a91", "tree": "164095a5cf5c3f1ae9de3b5ac27a72c47f34977b", "parents": [ "376676c7abdd6588c5aec640fa06c8f251fbabb6" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Fri Jul 24 21:23:11 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 21:23:11 2026 -0600" }, "message": "fix: raise CAST_INVALID_INPUT for invalid calendar dates in ANSI cast to date (#5014)" }, { "commit": "376676c7abdd6588c5aec640fa06c8f251fbabb6", "tree": "4b92014fe2590418a8c5510530594dee9d208dfa", "parents": [ "2e907e5602425b7d671ba185b5cd0a6b7b68a490" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Fri Jul 24 16:56:30 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 16:56:30 2026 -0600" }, "message": "docs: show implementation kind for each expression (#5028)\n\n* docs: show implementation kind for each expression in supported expressions page\n\nAdds an Implementation column (Native / Codegen dispatch / Hybrid / --)\nto every table in the supported expressions user-guide page. Values are\nauto-generated by GenerateDocs from the QueryPlanSerde maps: the serde\u0027s\ntrait mixins (NativeOptInAvailable, CodegenDispatchFallback, plain\nCometCodegenDispatch, plain CometExpressionSerde) decide the kind, and\nSpark\u0027s FunctionRegistry bridges SQL function names to Catalyst classes.\nRows for functions that are planned, aliased/rewritten before Comet sees\nthem, or handled at the operator level fall through to the \"--\" bucket\nand are explained in the notes column.\n\n* docs: document how to regenerate the Implementation column\n\nUpdate the contributor guide to explain what fills the new Implementation\ncolumn in expressions.md and how to regenerate it locally with the\ngenerate-docs Maven profile. Also mention the column in the release-docs\nstep so it is refreshed alongside the compatibility pages." }, { "commit": "2e907e5602425b7d671ba185b5cd0a6b7b68a490", "tree": "22e75d327b9a49fa078e79152986aa2b4f3de288", "parents": [ "756ddd7e1457bf97d784c3f04810fcee766140a4" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Fri Jul 24 12:33:05 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 12:33:05 2026 -0600" }, "message": "feat: native collect_list / array_agg aggregate (#4720)\n\n* feat: native collect_list / array_agg aggregate\n\nWires Spark\u0027s CollectList aggregate to datafusion-spark\u0027s\nSparkCollectList. array_agg, registered as a SQL alias of\nCollectList in FunctionRegistry, is also covered.\n\nCloses #2524.\n\n* fix: fall back collect_list/collect_set in multi-stage distinct aggregates\n\nA distinct aggregate combined with collect_list/collect_set produces a\nmulti-stage plan (Partial -\u003e PartialMerge -\u003e Final). CollectList/CollectSet\ndeclare a BinaryType buffer in Spark but produce a native ArrayType state, so\nComet cannot read a Spark-produced Binary buffer, nor round-trip its own\nArrayType buffer across the intermediate PartialMerge stages. Both led to\nnative crashes (\"could not cast Binary to List\" / \"cast List to Binary\").\n\nForce these multi-stage aggregates to fall back to Spark consistently:\n- tag the feeding Partial when a PartialMerge stage of CollectList/CollectSet\n is present (CometExecRule.tagUnsafePartialAggregates), and\n- fall back a PartialMerge stage whose buffer was produced by a Spark partial\n (CometBaseAggregate.doConvert).\n\nTwo-stage collect_list/collect_set continue to run natively.\n\nPatch the upstream SPARK-22223 plan-shape test to disable Comet, since native\ncollect_list removes the ObjectHashAggregateExec it asserts on.\n\nEnabling fully-native multi-stage execution is tracked in #4724.\n\n* feat: fall back collect_list/collect_set on Spark 4.2 RESPECT NULLS\n\nSpark 4.2 adds an ignoreNulls field to CollectList and CollectSet, and\ncollect_list(x) RESPECT NULLS sets it to false, keeping null elements. The\nnative path delegates to SparkCollectList/SparkCollectSet, which always drop\nnulls, so it would silently return a different result from Spark. Add a\nper-version CometCollectShim that reads ignoreNulls (always true on Spark 3.4\nthrough 4.1, where the field is absent) and fall back to Spark in\ngetSupportLevel when it is false.\n\nAlso rename QueryPlanSerde.hasIncompatibleBufferAgg to hasNativeArrayBufferAgg\nto describe what it detects: an aggregate whose native ArrayType state cannot\nround-trip Spark\u0027s declared BinaryType buffer.\n\n* test: accept Comet operators in SPARK-22223 instead of disabling Comet\n\nThe SPARK-22223 ObjectHashAggregate test asserts on the executed plan.\nWith Comet enabled, collect_list runs natively as CometHashAggregateExec,\nso ObjectHashAggregateExec is no longer present. Rather than disabling\nComet, update the operator assertion to also accept CometHashAggregateExec,\nmatching the pattern already used elsewhere in the diffs. The exchange\nassertion already matches ShuffleExchangeLike, which CometShuffleExchangeExec\nimplements, so the single-shuffle check still holds.\n\n* style: reflow adjustOutputForNativeState doc comment\n\nAdding CollectList to the comment pushed a line past the column limit\nduring the apache/main merge; reflow to satisfy spotless.\n\n* fix: coerce collect_list/collect_set nested field nullability\n\ncollect_list and collect_set build their result list with all element\nfields marked nullable, but SparkCollectList/SparkCollectSet derive the\nreturn type from the child, preserving non-nullable nested fields. When\nthe child is a nested type with a non-nullable inner field (e.g. a struct\nfield built from non-nullable columns), the declared aggregate output\ndisagrees with the array the accumulator produces, and the grouped native\nAggregateExec fails validating its output batch with \"column types must\nmatch schema types\".\n\nCast the collect child to the all-nullable variant of its type so the\ndeclared and produced types stay consistent.\n\n* review: drop unreachable buffer-source block; consolidate 3.4/3.5 shim; add FILTER + map tests\n\noperators.scala: drop the CollectList/CollectSet PartialMerge fallback block. It is unreachable now that the general missingCometProducer + aggsNotSupportingMixedExecution guard just above returns None for the same case. CometExecRule.scala: add a comment on the collect-specific tagging block explaining it is separate from the tagging block just above because canAggregateBeConverted skips the child-native check, so an all-native distinct collect chain would otherwise slip past. QueryPlanSerde.scala: narrow the hasNativeArrayBufferAgg doc comment to describe what the code actually matches and note that Percentile has the same shape but is not matched here. Consolidate CometCollectShim: move the identical spark-3.4 and spark-3.5 copies to spark-3.x. Add collect_list FILTER (WHERE ...) test and a map-input test hitting make_all_fields_nullable\u0027s Map arm (size-only via spark_answer_only, since sort_array cannot order a MapType).\n\n* test: cover collect_list/collect_set RESPECT NULLS fallback on Spark 4.2\n\nAdd a Spark 4.2-gated SQL file test asserting that collect_list and\ncollect_set with RESPECT NULLS fall back to Spark with Spark-identical\nresults, and expand the getSupportLevel comments to explain that the\nfallback branch is only reachable on 4.2+ (a no-op on 3.4 through 4.1)." }, { "commit": "756ddd7e1457bf97d784c3f04810fcee766140a4", "tree": "953bf9554fd83a2b94b210b6be24ea675b618a98", "parents": [ "8324fe97a9a3d40ffc52ec34a27545a97dc8afe0" ], "author": { "name": "Myx778", "email": "92378059+Myx778@users.noreply.github.com", "time": "Sat Jul 25 01:30:55 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 11:30:55 2026 -0600" }, "message": "feat: Support Spark levenshtein expression in native execution (#4105)\n\n* feat: Support Spark levenshtein expression in native execution\n\nImplements the Levenshtein edit distance function as a native Comet\nscalar UDF, enabling Spark\u0027s `levenshtein(str1, str2)` to run via\nDataFusion instead of falling back to JVM.\n\nImplementation:\n- Rust kernel: O(min(m,n)) space DP algorithm with Unicode char-level\n distance computation and proper NULL propagation\n- Scala serde: Register via CometScalarFunction(\"levenshtein\") in\n QueryPlanSerde stringExpressions map\n- Tests: Basic, NULL handling, and Unicode test cases\n\nCloses #3084\n\n* style: fix Spotless formatting violations in test code\n\n- Reformat Seq literal to match scalafmt maxColumn\u003d98 constraint\n- Break long sql() call into multi-line format\n- Reformat unicode test data as multi-line Seq\n\n* fix: address CI lint failures\n\n- Fix Clippy needless_range_loop: use iter_mut().enumerate() instead\n of indexing loop for initializing prev array\n- Fix Scalafix unused import: remove Levenshtein from strings.scala\n imports (it\u0027s used via wildcard import in QueryPlanSerde.scala)\n\n* test: add SLT tests for levenshtein expression\n\nAdd SQL logic test file covering:\n- Column arguments with NULL propagation\n- Column + literal combinations\n- Literal + literal edge cases\n- Identical string comparison\n- Unicode character-level distance\n\n* Add levenshtein to benchmark and update expressions.md\n\n- Add levenshtein benchmark entry in CometStringExpressionBenchmark\n- Update expressions.md to list Levenshtein as supported\n\n* Support three-argument levenshtein with threshold and add collation check\n\n- Implement threshold semantics: levenshtein(str1, str2, threshold) returns\n -1 when distance exceeds threshold, matching Spark behavior\n- Add CometLevenshtein serde with getSupportLevel that falls back for\n non-default collations (Spark 4 StringTypeWithCollation)\n- Add Rust tests for threshold and NULL threshold cases\n- Add SLT tests for threshold variants\n- Add Scala integration test for threshold\n- Add levenshtein_threshold to benchmark\n\n* Fix Spotless formatting in levenshtein threshold test\n\n* Fix Spotless formatting in strings.scala and test suite\n\n* Use scalarFunctionExprToProtoWithReturnType for levenshtein\n\nOverride convert() to use scalarFunctionExprToProtoWithReturnType\nwith IntegerType, so the native planner skips the DataFusion\nregistry lookup and does not conflict with DataFusion\u0027s built-in\n2-arg levenshtein function when 3 args (threshold) are passed.\n\nAlso fix Spotless: remove unnecessary string interpolation prefix.\n\n* Fix Spark 3.4 compatibility: skip levenshtein threshold tests on Spark \u003c 3.5\n\nThe three-argument levenshtein(str1, str2, threshold) is only available\nin Spark 3.5+. This moves threshold-related SQL tests to a separate file\nwith MinSparkVersion: 3.5 and adds assume(isSpark35Plus) to the unit test.\n\n* Remove duplicate levenshtein benchmark entry\n\n* Support threshold as column (not just literal) in levenshtein\n\n- Refactored spark_levenshtein to handle threshold as either\n ColumnarValue::Scalar or ColumnarValue::Array using into_array()\n- NULL threshold in a column produces NULL result for that row\n- Added Rust unit tests for array threshold, nulls, and negative values\n- Added Scala integration tests with threshold as column reference\n- Added SQL file tests with threshold column and NULL scenarios\n\n* Fix unused import warning: move ScalarValue to test module\n\n* Fix Spotless formatting in threshold column tests\n\n* Run cargo fmt\n\n---------\n\nCo-authored-by: yusinnmao \u003cyusinnmao@tencent.com\u003e\nCo-authored-by: Cursor \u003ccursoragent@cursor.com\u003e" }, { "commit": "8324fe97a9a3d40ffc52ec34a27545a97dc8afe0", "tree": "c5342536a81a07f20a6173611e6a1c6fde84133f", "parents": [ "8c504872a1d215a6f454eba841969e5a3a85cd94" ], "author": { "name": "Matt Butrovich", "email": "mbutrovich@users.noreply.github.com", "time": "Fri Jul 24 12:03:04 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 12:03:04 2026 -0400" }, "message": "fix: apply 1.11 Iceberg diff changes from #4991 to other versions (#5020)" }, { "commit": "8c504872a1d215a6f454eba841969e5a3a85cd94", "tree": "23a901089c0ece251d843cae6b643f6f3b3aa5c2", "parents": [ "02054c4ad0faf80d406bfea97d695a27b98ed1a9" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Fri Jul 24 11:05:03 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 11:05:03 2026 -0400" }, "message": "chore(deps): bump actions/checkout from 6 to 7 (#4881)\n\nBumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7.\n- [Release notes](https://github.com/actions/checkout/releases)\n- [Commits](https://github.com/actions/checkout/compare/v6...v7)\n\n---\nupdated-dependencies:\n- dependency-name: actions/checkout\n dependency-version: \u00277\u0027\n dependency-type: direct:production\n update-type: version-update:semver-major\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "02054c4ad0faf80d406bfea97d695a27b98ed1a9", "tree": "bb9232389039404e282a6b54d630ed370fbad72b", "parents": [ "a27d430008a4ebb5f5f89335170ee910c1eb9c2d" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Fri Jul 24 11:04:55 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 11:04:55 2026 -0400" }, "message": "chore(deps): bump actions/cache from 5 to 6 (#4882)\n\nBumps [actions/cache](https://github.com/actions/cache) from 5 to 6.\n- [Release notes](https://github.com/actions/cache/releases)\n- [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md)\n- [Commits](https://github.com/actions/cache/compare/v5...v6)\n\n---\nupdated-dependencies:\n- dependency-name: actions/cache\n dependency-version: \u00276\u0027\n dependency-type: direct:production\n update-type: version-update:semver-major\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "a27d430008a4ebb5f5f89335170ee910c1eb9c2d", "tree": "7ee77ba6d47304b31238b0146a2854c1bd3e0ba2", "parents": [ "3755def28bb7587a8c80547d2248885ba940dac4" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Fri Jul 24 11:04:46 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 11:04:46 2026 -0400" }, "message": "chore(deps): bump the codeql-actions group with 2 updates (#5016)\n\nBumps the codeql-actions group with 2 updates: [github/codeql-action/init](https://github.com/github/codeql-action) and [github/codeql-action/analyze](https://github.com/github/codeql-action).\n\n\nUpdates `github/codeql-action/init` from 4.37.1 to 4.37.3\n- [Release notes](https://github.com/github/codeql-action/releases)\n- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)\n- [Commits](https://github.com/github/codeql-action/compare/7188fc363630916deb702c7fdcf4e481b751f97a...e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81)\n\nUpdates `github/codeql-action/analyze` from 4.37.1 to 4.37.3\n- [Release notes](https://github.com/github/codeql-action/releases)\n- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)\n- [Commits](https://github.com/github/codeql-action/compare/7188fc363630916deb702c7fdcf4e481b751f97a...e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81)\n\n---\nupdated-dependencies:\n- dependency-name: github/codeql-action/init\n dependency-version: 4.37.3\n dependency-type: direct:production\n update-type: version-update:semver-patch\n dependency-group: codeql-actions\n- dependency-name: github/codeql-action/analyze\n dependency-version: 4.37.3\n dependency-type: direct:production\n update-type: version-update:semver-patch\n dependency-group: codeql-actions\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "3755def28bb7587a8c80547d2248885ba940dac4", "tree": "c731d20837a8aa3cd6cd1f1732467bf8ca96b7c6", "parents": [ "f86cc96f7a73ccd250b6046875fdb8cf55923389" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Fri Jul 24 11:04:35 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 11:04:35 2026 -0400" }, "message": "chore(deps): bump actions/setup-python from 6 to 7 (#5017)\n\nBumps [actions/setup-python](https://github.com/actions/setup-python) from 6 to 7.\n- [Release notes](https://github.com/actions/setup-python/releases)\n- [Commits](https://github.com/actions/setup-python/compare/v6...v7)\n\n---\nupdated-dependencies:\n- dependency-name: actions/setup-python\n dependency-version: \u00277\u0027\n dependency-type: direct:production\n update-type: version-update:semver-major\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "f86cc96f7a73ccd250b6046875fdb8cf55923389", "tree": "0e35d151ad7eea6fc13f7b5b109034092001c0fb", "parents": [ "0986e4a37343cc99cbfe3483fc0c7684f2a0e361" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Fri Jul 24 11:03:49 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 11:03:49 2026 -0400" }, "message": "chore(deps): bump base64 from 0.22.1 to 0.23.0 in /native (#5018)\n\nBumps [base64](https://github.com/marshallpierce/rust-base64) from 0.22.1 to 0.23.0.\n- [Changelog](https://github.com/marshallpierce/rust-base64/blob/master/RELEASE-NOTES.md)\n- [Commits](https://github.com/marshallpierce/rust-base64/compare/v0.22.1...v0.23.0)\n\n---\nupdated-dependencies:\n- dependency-name: base64\n dependency-version: 0.23.0\n dependency-type: direct:production\n update-type: version-update:semver-minor\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "0986e4a37343cc99cbfe3483fc0c7684f2a0e361", "tree": "64bf43c273abd6e1eaebf7abfb3e57aa04de666a", "parents": [ "68489a7bc6cf51fa3afcaefdce5494fc93db4a7b" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Fri Jul 24 11:03:40 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 11:03:40 2026 -0400" }, "message": "chore(deps): bump actions/stale from 10.3.0 to 10.4.0 (#4879)\n\nBumps [actions/stale](https://github.com/actions/stale) from 10.3.0 to 10.4.0.\n- [Release notes](https://github.com/actions/stale/releases)\n- [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md)\n- [Commits](https://github.com/actions/stale/compare/eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899...1e223db275d687790206a7acac4d1a11bd6fe629)\n\n---\nupdated-dependencies:\n- dependency-name: actions/stale\n dependency-version: 10.4.0\n dependency-type: direct:production\n update-type: version-update:semver-minor\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "68489a7bc6cf51fa3afcaefdce5494fc93db4a7b", "tree": "c6008372c83fd10604c72dda2f8a04d9d5dff13f", "parents": [ "7123be7b52251739c85259a8f92fff7bf961892c" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Fri Jul 24 08:48:17 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 08:48:17 2026 -0600" }, "message": "perf: encode shuffle IPC schema once per writer instead of per block (#5006)\n\n* perf: encode shuffle IPC schema once per writer instead of per block\n\nShuffleBlockWriter::write_batch used StreamWriter::try_new for every\nblock, which re-serializes the schema flatbuffer into each block. Since\nevery block in a shuffle shares one schema, this repeats identical work,\nwhich matters for wide or deeply nested schemas and for shuffles with\nmany small blocks (high partition counts).\n\nPre-encode the schema message once in try_new and write it verbatim at\nthe start of every block, writing the record batch via\nIpcDataGenerator::encode. Schemas containing dictionary types keep using\nStreamWriter, whose dictionary-id bookkeeping ties schema and batch\nencoding together. Output framing is unchanged.\n\nA new criterion benchmark (shuffle_block_schema_encoding) covers a wide\nflat schema and a deeply nested schema. Encoding an 8192-row batch is\n~7% (wide flat schema) / ~24% (deeply nested schema) faster with the\npre-encoded schema.\n\nPart of #5002.\n\n* chore: trigger CI\n\n* refactor: fold schema-encoding fields into single Option, document header size\n\n* refactor: address review on shuffle schema pre-encoding\n\n- Fold schema/schema_message fields into a SchemaEncoding enum stating the\n Precoded/Fallback invariant and dropping the fast-path deep schema clone\n- Detect dictionary types via Schema::flattened_fields instead of a\n hand-maintained recursive match, covering all containers arrow recurses into\n- Store one IpcWriteOptions on the writer, shared by the schema and batch\n encoders, and pin it to metadata version V5 (which IPC_EOS depends on)\n\n* fix: share shuffle block writer buffers via Arc to avoid per-partition deep copy\n\nShuffleBlockWriter is cloned once per output partition when building\nper-partition spill writers. After pre-encoding the IPC schema, each clone\ndeep-copied the schema message (and header) Vec, so a shuffle with a very\nlarge partition count (e.g. the SPARK-48037 test uses 16M+ partitions)\nallocated gigabytes and stalled the executor.\n\nHold the immutable header and pre-encoded schema behind Arc so cloning is an\nO(1) refcount bump. Add a regression test asserting the buffers are shared\nacross clones." }, { "commit": "7123be7b52251739c85259a8f92fff7bf961892c", "tree": "03b7a14cfeb6ddcfa7ced2091732d06d0ddf41b2", "parents": [ "b75c5707489f51485b15fa1d5079948e4d3b9ef2" ], "author": { "name": "Peter Lee", "email": "peterxcli@gmail.com", "time": "Fri Jul 24 22:33:13 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 24 08:33:13 2026 -0600" }, "message": "feat: support multiply_dt_interval with codegen dispatch (#4900)" }, { "commit": "b75c5707489f51485b15fa1d5079948e4d3b9ef2", "tree": "6b3739baf0668c4b1c77e05c8a5773cb218bdbfd", "parents": [ "406e18271c59c73ffd21c2f9ed5b3ff23a26ab2d" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Thu Jul 23 19:22:17 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 23 19:22:17 2026 -0600" }, "message": "perf: bypass shuffle BatchCoalescer for already-sized batches (#5003)" }, { "commit": "406e18271c59c73ffd21c2f9ed5b3ff23a26ab2d", "tree": "a46cb489eab6863d5f39c9a5e98178168ae2b313", "parents": [ "bc53ad5c56eff6e4e3c2d702a6869b7cf28352ae" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Thu Jul 23 18:21:37 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 23 18:21:37 2026 -0600" }, "message": "perf: optimize `spark_ceil` (3x faster) (#4926)\n\n* perf: optimize spark_floor in datafusion-comet-spark-expr\n\n* refactor: unify make_decimal_scalar/array signatures on impl Fn\n\nAdopts impl Fn on both helpers so they are symmetric and no callsite needs to pass \u0026f, addressing coordination feedback with #4926 which uses impl Fn too. Floor\u0027s scalar path drops its stray \u0026f.\n\n* chore: Codegen fallback explain for plan (#4891)\n\n* perf: optimize spark_cast_int_to_int in datafusion-comet-spark-expr (#4920)\n\n* perf: optimize spark_ceil in datafusion-comet-spark-expr\n\n* perf: share dispatch_pow10 with floor; add wide-path decimal test\n\nCeil now uses the same dispatch_pow10 + decimal_ceil_pow10\u003cconst EXP\u003e pattern that floor uses in the same batch, replacing the per-row runtime i64::try_from branch. The wide fallback is an out-of-line cold function mirroring floor. Adds test_ceil_decimal128_wide_array covering Decimal128(38, 6) unscaled values above i64::MAX for positive-remainder, negative-remainder, and exact-multiple cases so the fallback arm and the negative-remainder branch have direct coverage.\n\n---------\n\nCo-authored-by: Oleks V \u003ccomphead@users.noreply.github.com\u003e" }, { "commit": "bc53ad5c56eff6e4e3c2d702a6869b7cf28352ae", "tree": "4fcac35e93bb3aea1af8ff1336f40483dcf14ceb", "parents": [ "0761e549a43fee618af462a305645e70d663a380" ], "author": { "name": "Matt Butrovich", "email": "mbutrovich@users.noreply.github.com", "time": "Thu Jul 23 18:25:23 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 23 18:25:23 2026 -0400" }, "message": "feat: Iceberg table format V3: native table decryption, fall back for other V3 features (#4991)" }, { "commit": "0761e549a43fee618af462a305645e70d663a380", "tree": "9116af896a8dd8af1b77ef34efc6dfa379ddf6af", "parents": [ "f06aa31b3ee00256bf395d4a87566c2714bdce52" ], "author": { "name": "Matt Butrovich", "email": "mbutrovich@users.noreply.github.com", "time": "Thu Jul 23 13:42:05 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 23 13:42:05 2026 -0400" }, "message": "perf: dedupe Iceberg residuals and delete files in native scan serde (#4982)" }, { "commit": "f06aa31b3ee00256bf395d4a87566c2714bdce52", "tree": "0afb920beedd56f908da2603ca69b10ac8da8040", "parents": [ "d075ab326e7601ffb65dc3d156ee4b00362ec06b" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Thu Jul 23 06:04:19 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 23 08:04:19 2026 -0400" }, "message": "feat: support approx_count_distinct aggregate expression (#4819)\n\n* feat: support approx_count_distinct aggregate expression\n\nAdd native support for Spark\u0027s approx_count_distinct, a faithful port of\nSpark\u0027s HyperLogLogPlusPlus / HyperLogLogPlusPlusHelper.\n\nEach non-null input is hashed with Comet\u0027s Spark-compatible XxHash64\n(seed 42, floats normalized first), and the HyperLogLog++ registers are\nstored in Spark\u0027s exact packed-Long buffer layout (10 six-bit registers\nper word). The cardinality is estimated with the same linear-counting and\nbias-correction tables Spark uses, so results are bit-identical to Spark\nand the partial-aggregation state matches Spark\u0027s aggBufferSchema.\n\nIncludes a vectorized GroupsAccumulator, SQL file tests comparing against\nSpark across a range of cardinalities, native unit tests, benchmark\ncoverage, and documentation updates.\n\n* feat: address review feedback on approx_count_distinct\n\n- restrict decimal inputs to precision \u003c\u003d 18 (Spark hashes wider decimals\n through BigDecimal, which the native i128 path does not match)\n- set supportsMixedPartialFinal\u003dtrue; the register buffer matches Spark\u0027s\n aggBufferSchema, enabling mixed Comet/Spark partial and final aggregation\n- add getUnsupportedReasons so the Compatibility Guide reflects the type limits\n- derive Hash, add assert/debug_assert invariants and a float-order comment,\n reuse the shared normalize_float, drop a redundant field, make bias\n correction lazy, and reuse a per-accumulator hash scratch buffer\n- compact BIAS_DATA layout with rustfmt::skip (values unchanged)\n- add tests: decimal boundary and wide-decimal fallback, non-UTC timestamp,\n collated-string fallback, and mixed partial/final plan shape" }, { "commit": "d075ab326e7601ffb65dc3d156ee4b00362ec06b", "tree": "bb1de53d067b16a45167fabca45477e10fbee5e6", "parents": [ "6934ef85e47a8576c76d05f6a48ccb5027cb4d10" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Thu Jul 23 00:23:49 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 23 00:23:49 2026 -0600" }, "message": "docs: document Spark version adoption and support-lifetime policy (#4977)" }, { "commit": "6934ef85e47a8576c76d05f6a48ccb5027cb4d10", "tree": "168af724bc3d0b2b3721994de814c0e72dd2ef40", "parents": [ "56405e031f5b644d187770308831544ba2b5b6eb" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Thu Jul 23 00:23:26 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 23 00:23:26 2026 -0600" }, "message": "perf: optimize `spark_cast_float64_to_utf8` (~40% faster) (#4918)" }, { "commit": "56405e031f5b644d187770308831544ba2b5b6eb", "tree": "1a1fb4a287e790ed8de46f657abb608b86a4f6cc", "parents": [ "52140f3930e7e11504d5df893a82c2adb08c406b" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 22 21:22:29 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 22 21:22:29 2026 -0600" }, "message": "perf: optimize `date_trunc` (\u003e2x faster) (#4915)\n\n* perf: optimize date_trunc in datafusion-comet-spark-expr\n\n* perf: table-ize timestamp_trunc format lookup for NTZ and TZ paths\n\nApplies the DATE_TRUNC_FORMATS table pattern to both timestamp_trunc paths. ntz_trunc_fn_for_format and tz_trunc_fn_for_format resolve the format via eq_ignore_ascii_case against ASCII tables, eliminating the per-invocation and per-row to_uppercase() String allocations from the scalar-format tz path and the array-format helper. Also drops the unused Cow / is_ascii split from date_trunc_fn_for_format since eq_ignore_ascii_case only folds ASCII and every table entry is ASCII, so a non-ASCII input cannot match either way." }, { "commit": "52140f3930e7e11504d5df893a82c2adb08c406b", "tree": "e442bc57cdc9a1ab792fde09cc71a4ddb9c629ad", "parents": [ "c7f9a1f97762a58957d8e32fc809e072761138e5" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 22 15:01:16 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 22 15:01:16 2026 -0600" }, "message": "perf: optimize cast_decimal128_to_utf8 in datafusion-comet-spark-expr (#4924)\n\n* perf: optimize cast_decimal128_to_utf8 in datafusion-comet-spark-expr\n\n* test: cover multi-group leading zeros and signed scientific coefficients\n\nAdds three assertions to test_decimal128_to_java_string: 10^19 + 5 to exercise the render_digits leading-zeros branch on a lower group; i128::MAX at scale 45 to hit the scientific-notation coefficient split on a two-group value; and its negation to exercise the sign push on the same shape." }, { "commit": "c7f9a1f97762a58957d8e32fc809e072761138e5", "tree": "3a0d4f557905284d9fb00f395fb157dbbf45f385", "parents": [ "e2f5df6d04608e6968d837346c90738a6d066a10" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 22 12:48:22 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 22 12:48:22 2026 -0600" }, "message": "perf: optimize `date_parser` / cast string to date (up to 2x faster) (#4917)\n\n* perf: optimize date_parser in datafusion-comet-spark-expr\n\n* test: cover fast-path canonical-invalid dates; describe null-in-every-mode; assert i32 fit\n\nAdds a group of canonical yyyy-mm-dd invalid calendar dates and a leap-day positive case to date_parser_test so a regression that skipped calendar validation in the fast path would fail. Rewrites the fast-path comment to describe the actual Comet behavior (Ok(None) treated as null in every eval mode by the caller) rather than an incorrect claim about Spark. Reinstates loud failure on the general-path cast via debug_assert! so a future weakening of the year range check does not silently wrap." }, { "commit": "e2f5df6d04608e6968d837346c90738a6d066a10", "tree": "9f336d95363ca18e931ea3936be272ec4a060ec9", "parents": [ "7979527bee394ee2bda24fb20b12b57408387a93" ], "author": { "name": "KAZUYUKI TANIMURA", "email": "ktanimura@apple.com", "time": "Wed Jul 22 11:10:30 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 22 11:10:30 2026 -0700" }, "message": "chor: remove legacy use.lazyMaterialization (#4998)\n\n## Which issue does this PR close?\n\n## Rationale for this change\n\n`spark.comet.use.lazyMaterialization` is not a real option anymore, only used in tests without actual lazy materialization code.\n\n## What changes are included in this PR?\n\nRemoved the option\n\n## How are these changes tested?\n\nExisting tests" }, { "commit": "7979527bee394ee2bda24fb20b12b57408387a93", "tree": "c1cc11fc1109929e8e706c9f44bc922969ef454a", "parents": [ "81c2a84fe34040a2fe7cbe1d05e7a91b6f2b450f" ], "author": { "name": "Oleks V", "email": "comphead@users.noreply.github.com", "time": "Wed Jul 22 10:43:44 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 22 10:43:44 2026 -0700" }, "message": "chore: use DF `array_repeat` and `array_compact` (#4741)" }, { "commit": "81c2a84fe34040a2fe7cbe1d05e7a91b6f2b450f", "tree": "1b8f7c5375706b4311c6d934295ada078bd8dff9", "parents": [ "7b00c40cd1c4f875bee4032d8c34f6c9788650b0" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 22 10:49:22 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 22 12:49:22 2026 -0400" }, "message": "perf: optimize `parse_string_to_decimal` (30-40% faster) (#4916)\n\n* perf: optimize parse_string_to_decimal in datafusion-comet-spark-expr\n\n* test/refactor: fold duplicate bench; test 38-digit boundary; bound pow10_i128\n\nDeletes benches/cast_string_to_decimal.rs and folds its improvements (seeded StdRng, 8192-row batch, decimal_18_2) into the existing create_decimal_cast_string_batch in cast_from_string.rs so the two benches no longer collide on the same criterion group. Also drops the [[bench]] entry from Cargo.toml. Locks the 38-digit boundary that digits_to_i128\u0027s head/tail split relies on with three Rust unit tests: 38 nines parses, 39 nines overflows, 38 leading zeros plus \u002742\u0027 reduces to 42. Makes pow10_i128 return Option\u003ci128\u003e so an over-long fractional part maps to the invalid_decimal_cast error instead of the debug panic from 10_i128.pow(40)." }, { "commit": "7b00c40cd1c4f875bee4032d8c34f6c9788650b0", "tree": "f6b183f69e90fdf56a96962ff76c589cb223e4d3", "parents": [ "5ef4bd7a723dda4c12882bd1af50c694dc2c6474" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 22 10:27:28 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 22 12:27:28 2026 -0400" }, "message": "perf: optimize `cast_binary_to_string` binary-format styles (up to 27x faster) (#4912)\n\n* perf: optimize cast_binary_to_string in datafusion-comet-spark-expr\n\n* chore: drop now-unused hex dependency from spark-expr\n\n* cast_binary_to_string: simpler write_i8/write_upper_hex, tighter capacity, invariant note\n\nwrite_i8 collapses to write!(out, \"{}\", byte as i8) so the manual widening and three-digit expansion drop out. write_upper_hex builds a two-byte ASCII slice and writes it once via write_str + from_utf8_unchecked instead of two write_char calls through char::encode_utf8. Sizes the UTF-8 capacity at value_bytes (valid UTF-8 fits 1x; the builder grows on rare U+FFFD expansion) and drops the redundant + num_rows on the Base64 estimate since div_ceil(3) already rounds each row to a full 4-char group. Adds a one-line note at the null branch stating that the previous iteration always finalized its row with append_value, so no bytes are pending." }, { "commit": "5ef4bd7a723dda4c12882bd1af50c694dc2c6474", "tree": "44dd177c859f4b1baf1345884a51aa9cb042d880", "parents": [ "198109160221e41cc97ecd7e415971006fa3b637" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 22 10:19:12 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 22 12:19:12 2026 -0400" }, "message": "fix: widen local table scan child nullability to match native kernels (#4843)\n\nAn in-memory Seq/Map column read through CometLocalTableScanExec encodes its\narray element and map value as non-null (containsNull\u003dfalse /\nvalueContainsNull\u003dfalse), but Comet expression serdes promise nullable\nchildren. Feeding a non-null child into a native array/map kernel fails the\nplanned-vs-actual type assertion (e.g. spark_array_slice returning\nList(non-null Int32) while List(Int32) was promised) or panics building a\nListArray for map_entries.\n\nWiden nested child-field nullability on both sides so they agree: the declared\nscan schema (serialized from op.output in CometSink) and the Arrow batch\nproduced by the local scan. Without widening the declared schema too, the\nnative ScanExec casts the batch back to the non-null type in build_record_batch.\n\nThe widening is scoped to the local table scan via an overridable scanFieldType\nhook, leaving other sinks untouched. Map keys stay non-null, matching Arrow.\nThis mirrors the Parquet scan path, which already reads children as nullable.\n\nCloses #4789" }, { "commit": "198109160221e41cc97ecd7e415971006fa3b637", "tree": "e6c7839b6821e9080e557f650c87126a41565785", "parents": [ "e3363e2cb64174e7caf8d70f6b1534047e26c760" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 22 06:42:40 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 22 06:42:40 2026 -0600" }, "message": "feat: add spark.comet.shuffle.maxBufferBytes to cap native shuffle writer memory (#4989)" }, { "commit": "e3363e2cb64174e7caf8d70f6b1534047e26c760", "tree": "b1fa538ef4f2dded80de8580484a5f7c1aabea9c", "parents": [ "32e2d7c4a82fafd70ba023dd848c6076609f3c9a" ], "author": { "name": "Matt Butrovich", "email": "mbutrovich@users.noreply.github.com", "time": "Tue Jul 21 20:48:15 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 21 17:48:15 2026 -0700" }, "message": "deps: bump to datafusion 54.1.0 and latest iceberg-rust (#4996)" }, { "commit": "32e2d7c4a82fafd70ba023dd848c6076609f3c9a", "tree": "e14e9fca3cec9100e583337087a808732421a88b", "parents": [ "957886db83c631027fffd11224a50de233f7bcf6" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 21 07:00:15 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 21 07:00:15 2026 -0600" }, "message": "refactor: remove untested async columnar shuffle (#4985)\n\n* feat: add withAlternative alias mechanism for CometConf, rename replaceSortMergeJoin\n\nIntroduces `.withAlternative(oldKey, ...)` on `ConfigBuilder`. Reads check the primary\nkey first, then any registered alternatives in order; reading a value from an alternative\nlogs a one-time deprecation warning per JVM naming the current key. Threaded through\n`TypedConfigBuilder` into `ConfigEntryWithDefault` and `OptionalConfigEntry` so the same\nmechanism covers configs with defaults, env-var overrides, and optional configs.\n\nPilots the mechanism by renaming `spark.comet.exec.replaceSortMergeJoin` to\n`spark.comet.exec.forceShuffleHashJoin.enabled` (Scala symbol `COMET_FORCE_SHJ`), keeping\nthe old key working as an alias.\n\nAdds `CometConfSuite` with tests for primary-wins, alternative fallback, default,\nmultiple alternatives, OptionalConfigEntry, and type conversion through the alias path.\n\nAdds a Configuration Conventions contributor-guide page that documents the key/symbol\nnaming rules and the rename workflow.\n\nBroader rename backlog tracked in #4978, per the 1.0.0 tracker #4082.\n\n* fixup: forceShuffleHashJoin → forceShuffledHashJoin, drop .enabled suffix\n\nRename correction: use the past-participle form matching Spark\u0027s `ShuffledHashJoin`\nclass. Also drops the `.enabled` suffix — the `force` verb already encodes the\nboolean action, so `.enabled` is redundant for this key.\n\nSoftens the Configuration Conventions doc\u0027s boolean-suffix rule accordingly:\ndescriptor-form flags (debug.enabled, metrics.enabled) still end in `.enabled`;\naction-verb flags (force..., allow...) can omit it.\n\n* ci: register CometConfSuite in linux and macos PR workflows\n\n* refactor: remove untested async columnar shuffle\n\nThe async shuffle path (spark.comet.columnar.shuffle.async.enabled) has been\ndisabled by default and lacked meaningful test coverage. Remove it along with\nits supporting classes and configs:\n\n- Delete CometShuffleExternalSorterAsync and ShuffleThreadPool\n- Merge CometShuffleExternalSorterSync back into CometShuffleExternalSorter\n as a single concrete class (was an interface + factory)\n- Strip async branches from CometBypassMergeSortShuffleWriter and\n CometDiskBlockWriter\n- Delete spark.comet.columnar.shuffle.async.{enabled,thread.num} configs\n- Rename spark.comet.columnar.shuffle.async.max.thread.num to\n spark.comet.columnar.shuffle.max.writers.per.executor (the cap is actually\n on hash-shuffle writers, not threads); keep the old key as an alias via\n withAlternative\n- Remove CometAsyncShuffleSuite, DisableAQECometAsyncShuffleSuite, and the\n asyncShuffleEnable machinery from CometColumnarShuffleSuite; drop the\n async benchmark case\n- Drop the async config table entry from the JVM shuffle contributor doc\n\nNet: -773 lines." }, { "commit": "957886db83c631027fffd11224a50de233f7bcf6", "tree": "806283ce0c07ca45420a6f947342cc11b68da359", "parents": [ "b01655526d677431c369717fac6bf4a2d2010334" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Mon Jul 20 15:39:25 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jul 20 15:39:25 2026 -0600" }, "message": "feat: add withAlternative alias mechanism for CometConf (#4979)\n\n* feat: add withAlternative alias mechanism for CometConf, rename replaceSortMergeJoin\n\nIntroduces `.withAlternative(oldKey, ...)` on `ConfigBuilder`. Reads check the primary\nkey first, then any registered alternatives in order; reading a value from an alternative\nlogs a one-time deprecation warning per JVM naming the current key. Threaded through\n`TypedConfigBuilder` into `ConfigEntryWithDefault` and `OptionalConfigEntry` so the same\nmechanism covers configs with defaults, env-var overrides, and optional configs.\n\nPilots the mechanism by renaming `spark.comet.exec.replaceSortMergeJoin` to\n`spark.comet.exec.forceShuffleHashJoin.enabled` (Scala symbol `COMET_FORCE_SHJ`), keeping\nthe old key working as an alias.\n\nAdds `CometConfSuite` with tests for primary-wins, alternative fallback, default,\nmultiple alternatives, OptionalConfigEntry, and type conversion through the alias path.\n\nAdds a Configuration Conventions contributor-guide page that documents the key/symbol\nnaming rules and the rename workflow.\n\nBroader rename backlog tracked in #4978, per the 1.0.0 tracker #4082.\n\n* fixup: forceShuffleHashJoin → forceShuffledHashJoin, drop .enabled suffix\n\nRename correction: use the past-participle form matching Spark\u0027s `ShuffledHashJoin`\nclass. Also drops the `.enabled` suffix — the `force` verb already encodes the\nboolean action, so `.enabled` is redundant for this key.\n\nSoftens the Configuration Conventions doc\u0027s boolean-suffix rule accordingly:\ndescriptor-form flags (debug.enabled, metrics.enabled) still end in `.enabled`;\naction-verb flags (force..., allow...) can omit it.\n\n* ci: register CometConfSuite in linux and macos PR workflows" }, { "commit": "b01655526d677431c369717fac6bf4a2d2010334", "tree": "0010b3366bec2f8d302b707e979e1472f952a8bc", "parents": [ "f46719b46b2442b4e82a24084eda9d4acad70ed5" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Mon Jul 20 12:47:51 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jul 20 14:47:51 2026 -0400" }, "message": "chore: remove dead Parquet parallel-IO configs (#4981)\n\nThese configs were consumed by the JVM-side Parquet reader that has since\nbeen removed. They are referenced nowhere in current native or JVM code\nand mislead operators tuning benchmarks.\n\nRemoved configs:\n- spark.comet.parquet.read.parallel.io.enabled\n- spark.comet.parquet.read.parallel.io.thread-pool.size\n- spark.comet.parquet.read.io.mergeRanges\n- spark.comet.parquet.read.io.mergeRanges.delta\n- spark.comet.parquet.read.io.adjust.readRange.skew\n\nAlso removes the stale ParquetReadSuite \"test merge scan range\" test,\nwhich only asserted that the (dead) config propagated to the Hadoop conf.\n\nRefs #4842." }, { "commit": "f46719b46b2442b4e82a24084eda9d4acad70ed5", "tree": "2db0c1d8dbe12f8784bc31ee1903dc58fa17cac1", "parents": [ "9483db47f04293b3639a9df14c044194e19ed4c5" ], "author": { "name": "kid", "email": "19265318+u70b3@users.noreply.github.com", "time": "Mon Jul 20 22:35:04 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jul 20 08:35:04 2026 -0600" }, "message": "test: promote `try_to_date`/`try_to_timestamp` SQL tests to native coverage (#4973)\n\n* test: promote `try_to_date`/`try_to_timestamp` SQL tests to native coverage\n\nInvestigation of #4556 showed the reported fallback does not exist on\ncurrent main: in non-ANSI mode Spark\u0027s ReplaceExpressions expands\ntry_to_date/try_to_timestamp to the exact same tree as to_date/\nto_timestamp (Cast(..., LEGACY) or GetTimestamp(failOnError\u003dfalse)),\nwhich Comet already accelerates. The 15 spark_answer_only markers were\nconservative annotations from #4555 (that mode only compares answers\nwithout asserting native execution), not measured fallbacks.\n\nFlip the 15 try_to_date/try_to_timestamp queries to default query mode\n(native coverage + answer check), keeping the 3 try_make_timestamp\nmarkers which belong to #4554. Verified on Spark 4.1:\nCometSqlFileTestSuite try_datetime passes with all queries running\nfully native.\n\nAlso update the expression support matrix (try_to_date/try_to_timestamp\n🔜 -\u003e ✅) and add audit entries.\n\nCloses #4556\n\n* test: cover try timestamp parser policies natively" }, { "commit": "9483db47f04293b3639a9df14c044194e19ed4c5", "tree": "08586973fdc70602b0464360f5c6691fa6851335", "parents": [ "415421bdb33f636b4ee86641f04cc60bb061e179" ], "author": { "name": "kid", "email": "19265318+u70b3@users.noreply.github.com", "time": "Mon Jul 20 22:34:39 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jul 20 08:34:39 2026 -0600" }, "message": "perf: vectorize `spark_unscaled_value` (9x faster) (#4972)\n\nReplace the per-element Int64Builder loop in the array branch with the\nvectorized Arrow unary kernel: the Decimal128 values buffer is mapped to\nInt64 in a single pass and the null buffer is carried over untouched\n(zero-copy). The scalar branch is unchanged.\n\nBenchmark (8192 rows, Decimal128(20,2), benches/unscaled_value.rs):\n- no nulls: 29.5 us -\u003e 3.2 us (~9.3x)\n- sparse nulls (1 in 10): 35.5 us -\u003e 3.0 us (~11.7x)\n- dense nulls (1 in 2): 30.5 us -\u003e 2.9 us (~10.4x)\n\nPart of #4942" }, { "commit": "415421bdb33f636b4ee86641f04cc60bb061e179", "tree": "89a95871273d7aee96f2be0db60f81087aa26477", "parents": [ "92f0f33e752243c1fe5ef72dd05858d224b7984c" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Sat Jul 18 09:51:01 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Jul 18 09:51:01 2026 -0700" }, "message": "chore(deps): bump the codeql-actions group with 2 updates (#4970)\n\nBumps the codeql-actions group with 2 updates: [github/codeql-action/init](https://github.com/github/codeql-action) and [github/codeql-action/analyze](https://github.com/github/codeql-action).\n\n\nUpdates `github/codeql-action/init` from 4.36.2 to 4.37.1\n- [Release notes](https://github.com/github/codeql-action/releases)\n- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)\n- [Commits](https://github.com/github/codeql-action/compare/8aad20d150bbac5944a9f9d289da16a4b0d87c1e...7188fc363630916deb702c7fdcf4e481b751f97a)\n\nUpdates `github/codeql-action/analyze` from 4.36.2 to 4.37.1\n- [Release notes](https://github.com/github/codeql-action/releases)\n- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)\n- [Commits](https://github.com/github/codeql-action/compare/8aad20d150bbac5944a9f9d289da16a4b0d87c1e...7188fc363630916deb702c7fdcf4e481b751f97a)\n\n---\nupdated-dependencies:\n- dependency-name: github/codeql-action/init\n dependency-version: 4.37.1\n dependency-type: direct:production\n update-type: version-update:semver-minor\n dependency-group: codeql-actions\n- dependency-name: github/codeql-action/analyze\n dependency-version: 4.37.1\n dependency-type: direct:production\n update-type: version-update:semver-minor\n dependency-group: codeql-actions\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "92f0f33e752243c1fe5ef72dd05858d224b7984c", "tree": "55b66518e721ca8b7ae949755850f6d245bcdd6d", "parents": [ "cb71d700fbd6d28e8a0eab62802e8cb133c06a69" ], "author": { "name": "Oleks V", "email": "comphead@users.noreply.github.com", "time": "Fri Jul 17 14:39:47 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 17 15:39:47 2026 -0600" }, "message": "feat: fall back to Spark for collated predicate operands (#4948)\n\n* feat: fall back to Spark for collated predicate operands\n\nComet\u0027s native equality, ordering, and hashing compare raw UTF-8 bytes,\nso a predicate operand carrying a non-UTF8_BINARY collation (Spark 4+)\nwould produce wrong answers on the native path -- for example,\n`\u0027a\u0027 \u003d \u0027A\u0027` under UNICODE_CI returns true in Spark but false byte-wise.\n\nAdd a `ComparisonUtils.collationSupportLevel` shared helper and wire it\ninto `getSupportLevel` for the six binary comparison serdes plus\n`In`/`InSet`. `CometNot`\u0027s special-case rewrites for\n`Not(EqualTo|EqualNullSafe|In)` now guard on the same helper so they\nfall through to the generic path (and thus to Spark) when any operand\nis collated. The helper walks nested types via\n`hasNonDefaultStringCollation`, so collated strings inside\narray/map/struct operands are caught too.\n\n* feat: dispatch collated str_to_map through JVM codegen\n\nMix CodegenDispatchFallback into CometStrToMap so that when the\ninput string or a delimiter carries a non-UTF8_BINARY collation\nthe expression is executed by Spark\u0027s own doGenCode inside the\nComet kernel rather than falling the whole projection back to\nSpark. Update the str_to_map_collation.sql fixture to drop the\nexpect_fallback(...) assertion accordingly.\n\n* test: dispatch legacy-truncate str_to_map through codegen\n\nUpdate str_to_map_legacy_truncate.sql to drop the expect_fallback\nassertions -- with CodegenDispatchFallback on CometStrToMap the\nexpression now runs via Spark\u0027s own doGenCode inside the Comet\nkernel instead of falling the projection back to Spark.\n\n* feat: fall back to Spark for collated Like/Contains/StartsWith/EndsWith\n\nExtends 903a772b6 to the string-predicate serdes flagged in review.\nCometLike, CometContains, CometStartsWith, and CometEndsWith now route\ntheir `getSupportLevel` through `ComparisonUtils.collationSupportLevel`\nso any non-UTF8_BINARY operand triggers a clean fallback. Previously\nContains / StartsWith / EndsWith reached the native scalar-function\nbridge via `CometScalarFunction` with no collation guard, and Like\u0027s\n`getSupportLevel` only rejected custom escape characters. Native\nkernels compare raw UTF-8 bytes, so a case- or accent-insensitive\ncollation would yield wrong answers (e.g. `\u0027Hello\u0027 LIKE \u0027%hello%\u0027`\nreturns true in Spark under UNICODE_CI but false byte-wise).\n\n`collationSupportLevel` grows an optional `exprName` so each predicate\nsurfaces a fallback reason that names itself in EXPLAIN.\n\nExpands `sql-tests/.../collation.sql` with predicate coverage: the six\nbinary comparisons under UTF8_LCASE and UNICODE_CI, `IN`/`NOT IN`\n(scalar and `InSet`-threshold), CometNot\u0027s `NOT (\u003d)` / `NOT (\u003c\u003d\u003e)`\nrewrites, `LIKE`, `Contains`, `StartsWith`, `EndsWith`, collated\nstrings buried in `array`/`map`/`struct` operands (exercising\n`hasNonDefaultStringCollation`\u0027s recursion), and NULL propagation\nthrough each of the above.\n\n* test: expect fallback for collated-predicate SQL tests\n\nEvery predicate query in `collation.sql` now uses\n`query expect_fallback(non-UTF8_BINARY collated operands)` instead of\nplain `query`. The predicate serdes correctly return `Unsupported` for\ncollated operands, so the enclosing Project falls back to Spark --\nwhich is the whole point of the fix -- but that violates the default\n`CheckCoverageAndAnswer` assertion mode\u0027s \"only Comet native operators\"\ncheck. `expect_fallback` still verifies the answer against Spark and\nadditionally pins the shared fallback-reason substring, so a regressed\nguard shows up as either a wrong answer or a missing fallback.\n\nAlso drops the map test: `CometCreateMap` routes through the JVM\ncodegen dispatcher, so its interaction with the outer EqualTo\u0027s\ncollation guard is not the clean path this file is meant to pin down.\n\n* test: drop collation cases Spark folds or rejects at analysis\n\nTwo classes of tests never reached the predicate serdes and so could\nnot exercise the fix:\n\n- NULL-literal RHS: Spark\u0027s `NullPropagation` rewrites `x \u003c\u003d\u003e NULL`\n to `IsNull(x)`, folds `x \u003d NULL` and `NULL IN (...)` to `Literal(null)`,\n etc., so the predicate serde is never asked about a collated NULL\n operand. The row `(7, NULL)` in `test_collated_predicates` still\n exercises NULL propagation through every remaining predicate query.\n\n- UNICODE_CI + LIKE / Contains / StartsWith / EndsWith: Spark 4.0\n restricts these to `StringTypeNonCSAICollation`, which rejects\n UNICODE_CI at analysis time with a DATATYPE_MISMATCH. UTF8_LCASE\n is accepted and already covers the same fallback path.\n\n* test: tighten comments in collation.sql\n\nDrop the \"what\" narration and shrink each remaining note to a single\nsentence about *why* the case exists (NullPropagation folding,\nNonCSAICollation restriction, InSet threshold, etc.). No SQL changes.\n\n* collation fallback\n\n* feat: route collated predicates through JVM codegen dispatcher\n\nAddresses two review comments on #4948.\n\nCollationAwareBinaryPredicate, CometIn, CometInSet, and CometLike now\nmix in CodegenDispatchFallback, so an `Unsupported` result from a\ncollated operand stays in the Comet pipeline via Spark\u0027s own\n`doGenCode` rather than falling the whole projection or filter back to\nSpark. CometContains / CometStartsWith / CometEndsWith inherit the\nmixin through the trait. Collation semantics still come from Spark\u0027s\ncodegen path, so answers stay Spark-correct.\n\nEach of these serdes now overrides `getUnsupportedReasons` so the\ncollation case (and, for Like, the custom-escape case) is picked up by\nGenerateDocs and surfaces in the compatibility guide instead of being\nundocumented.\n\nThe SQL tests in `collation.sql` switch from\n`expect_fallback(non-UTF8_BINARY collated operands)` back to plain\n`query`: with the dispatcher taking over, the operator stays native\nand the fallback-coverage assertion catches any regression that would\nforce it back to Spark." }, { "commit": "cb71d700fbd6d28e8a0eab62802e8cb133c06a69", "tree": "4b065da2d001a7cad0c746e6e0b2fec29889e560", "parents": [ "f480db11917ca423b7c64509057e14e094851a2d" ], "author": { "name": "Oleks V", "email": "comphead@users.noreply.github.com", "time": "Fri Jul 17 13:42:59 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 17 13:42:59 2026 -0700" }, "message": "chore: codeql dependabot fix (#4962)" }, { "commit": "f480db11917ca423b7c64509057e14e094851a2d", "tree": "218c045134297ec1bb7b0be26fcdc1fba8e1e70f", "parents": [ "c151a5771fb7c17006c80ffff6343e51ebf096da" ], "author": { "name": "Matt Butrovich", "email": "mbutrovich@users.noreply.github.com", "time": "Fri Jul 17 15:27:33 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 17 15:27:33 2026 -0400" }, "message": "docs: add 0.17.1 changelog (#4961)" }, { "commit": "c151a5771fb7c17006c80ffff6343e51ebf096da", "tree": "ab1dfe73141400d2173e5e61cf4dd30233acc041", "parents": [ "c998e6f8544277763d81e7a0aca758f955036406" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Fri Jul 17 12:26:49 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 17 12:26:49 2026 -0700" }, "message": "chore(deps): bump lz4_flex from 0.13.1 to 0.14.0 in /native (#4958)\n\nBumps [lz4_flex](https://github.com/pseitz/lz4_flex) from 0.13.1 to 0.14.0.\n- [Release notes](https://github.com/pseitz/lz4_flex/releases)\n- [Changelog](https://github.com/PSeitz/lz4_flex/blob/main/CHANGELOG.md)\n- [Commits](https://github.com/pseitz/lz4_flex/compare/0.13.1...0.14.0)\n\n---\nupdated-dependencies:\n- dependency-name: lz4_flex\n dependency-version: 0.14.0\n dependency-type: direct:production\n update-type: version-update:semver-minor\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "c998e6f8544277763d81e7a0aca758f955036406", "tree": "959c785c592e4fdd5558bb6a4f5fcbd4a99c10d3", "parents": [ "884e0a675ac8f3bed38f140485b686b1f5f18c71" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Fri Jul 17 12:24:36 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 17 12:24:36 2026 -0700" }, "message": "chore(deps): bump the all-other-cargo-deps group across 1 directory with 6 updates (#4956)\n\nBumps the all-other-cargo-deps group with 6 updates in the /native directory:\n\n| Package | From | To |\n| --- | --- | --- |\n| [regex](https://github.com/rust-lang/regex) | `1.13.0` | `1.13.1` |\n| [tokio](https://github.com/tokio-rs/tokio) | `1.52.3` | `1.53.0` |\n| [uuid](https://github.com/uuid-rs/uuid) | `1.23.5` | `1.24.0` |\n| [clap](https://github.com/clap-rs/clap) | `4.6.1` | `4.6.2` |\n| [simd-adler32](https://github.com/mcountryman/simd-adler32) | `0.3.9` | `0.3.10` |\n| [snap](https://github.com/BurntSushi/rust-snappy) | `1.1.1` | `1.1.2` |\n\n\n\nUpdates `regex` from 1.13.0 to 1.13.1\n- [Release notes](https://github.com/rust-lang/regex/releases)\n- [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md)\n- [Commits](https://github.com/rust-lang/regex/compare/1.13.0...1.13.1)\n\nUpdates `tokio` from 1.52.3 to 1.53.0\n- [Release notes](https://github.com/tokio-rs/tokio/releases)\n- [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.52.3...tokio-1.53.0)\n\nUpdates `uuid` from 1.23.5 to 1.24.0\n- [Release notes](https://github.com/uuid-rs/uuid/releases)\n- [Commits](https://github.com/uuid-rs/uuid/compare/v1.23.5...v1.24.0)\n\nUpdates `clap` from 4.6.1 to 4.6.2\n- [Release notes](https://github.com/clap-rs/clap/releases)\n- [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md)\n- [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.6.1...clap_complete-v4.6.2)\n\nUpdates `simd-adler32` from 0.3.9 to 0.3.10\n- [Changelog](https://github.com/mcountryman/simd-adler32/blob/main/CHANGELOG.md)\n- [Commits](https://github.com/mcountryman/simd-adler32/compare/v0.3.9...v0.3.10)\n\nUpdates `snap` from 1.1.1 to 1.1.2\n- [Release notes](https://github.com/BurntSushi/rust-snappy/releases)\n- [Commits](https://github.com/BurntSushi/rust-snappy/compare/1.1.1...1.1.2)\n\n---\nupdated-dependencies:\n- dependency-name: clap\n dependency-version: 4.6.2\n dependency-type: direct:production\n update-type: version-update:semver-patch\n dependency-group: all-other-cargo-deps\n- dependency-name: regex\n dependency-version: 1.13.1\n dependency-type: direct:production\n update-type: version-update:semver-patch\n dependency-group: all-other-cargo-deps\n- dependency-name: simd-adler32\n dependency-version: 0.3.10\n dependency-type: direct:production\n update-type: version-update:semver-patch\n dependency-group: all-other-cargo-deps\n- dependency-name: snap\n dependency-version: 1.1.2\n dependency-type: direct:production\n update-type: version-update:semver-patch\n dependency-group: all-other-cargo-deps\n- dependency-name: tokio\n dependency-version: 1.53.0\n dependency-type: direct:production\n update-type: version-update:semver-minor\n dependency-group: all-other-cargo-deps\n- dependency-name: uuid\n dependency-version: 1.24.0\n dependency-type: direct:production\n update-type: version-update:semver-minor\n dependency-group: all-other-cargo-deps\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "884e0a675ac8f3bed38f140485b686b1f5f18c71", "tree": "f39a85aa5d868dfecdbe6911b628acaba21c107c", "parents": [ "bc84171769421ac36cdc61630352910a8a530574" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Fri Jul 17 13:24:32 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 17 13:24:32 2026 -0600" }, "message": "chore: bump spark-4.2 profile to the released 4.2.0 (#4960)\n\n* chore: bump spark-4.2 profile to the released 4.2.0\n\nSpark 4.2.0 is now in Maven Central, so the spark-4.2 profile no longer\nneeds to target a preview. Point spark.version at 4.2.0 and make the\nmain sources compile against it.\n\n- Drop the WIP compile-only comment on the spark-4.2 profile.\n- Spark 4.2.0 bundles its own copy of\n org.apache.datasketches.memory.internal.ResourceImpl inside\n spark-catalyst, colliding with the transitive datasketches-memory jar\n and failing the BanDuplicateClasses enforcer rule; ignore that class.\n- Spark 4.2 removes GeographyVal / GeometryVal and replaces\n getGeography / getGeometry on SpecializedGetters with a single\n getBinaryView returning BinaryView. CometInternalRowShim can no longer\n be shared from the spark-4.1+ source root, so split it into per-version\n spark-4.1 and spark-4.2 copies.\n- Correct the stale Jetty comment that referenced preview4.\n\nThis does not wire up the Spark SQL tests for 4.2; it only moves the\nprofile onto the released version and keeps it compiling.\n\n* chore: regenerate Spark 4.2 plan-stability golden files\n\nRegenerated with dev/regenerate-golden-files.sh --spark-version 4.2\nagainst the released 4.2.0. The q2, q5, q54 (v1_4) and q5a (v2_7)\ngoldens are pruned because their plans now match the shared fallback on\n4.2.0, and q77a (v2_7) is added to record a degraded plan.\n\n* fix: resolve Spark 4.2 test failures from the version bump\n\nTwo Spark 4.2.0 changes surfaced by moving the profile off preview4:\n\nIceberg scans job: Spark 4.2.0 turned `connector.catalog.View` from an\ninterface into a class. No Iceberg spark-runtime is published for 4.2, so\nthe build reuses the 4.0 runtime, whose `SparkView implements View` now\nthrows IncompatibleClassChangeError at class-load and aborts the Iceberg\nsuites. Report Iceberg as unavailable on Spark 4.2 in the shared probes so\nthe suites skip. Guard the fuzz suite\u0027s beforeAll with an early return\n(cancelling from beforeAll aborts the suite) and add the missing\n`assume(icebergAvailable)` to one native-scan test.\n\nExpressions job: Spark 4.2 normalizes NaN / -0.0 for array_distinct and\nthe array set operations by wrapping their inputs as\n`KnownFloatingPointNormalized(ArrayTransform(arr, x -\u003e NormalizeNaNAndZero(x)))`.\nThe serde only handled a scalar `NormalizeNaNAndZero` child and fell back.\nSince `KnownFloatingPointNormalized` is a runtime no-op tag, serialize any\nother child directly and let its serde (the ArrayTransform codegen\ndispatcher) carry the normalization, keeping these operations native." }, { "commit": "bc84171769421ac36cdc61630352910a8a530574", "tree": "b864f2422941b43bbd0e82480e016580b201c757", "parents": [ "a469fc6d49f295f7c25478f21acc5cb8f882cc51" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Fri Jul 17 12:24:12 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 17 12:24:12 2026 -0700" }, "message": "chore(deps): bump actions/setup-node from 6 to 7 (#4955)\n\nBumps [actions/setup-node](https://github.com/actions/setup-node) from 6 to 7.\n- [Release notes](https://github.com/actions/setup-node/releases)\n- [Commits](https://github.com/actions/setup-node/compare/v6...v7)\n\n---\nupdated-dependencies:\n- dependency-name: actions/setup-node\n dependency-version: \u00277\u0027\n dependency-type: direct:production\n update-type: version-update:semver-major\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "a469fc6d49f295f7c25478f21acc5cb8f882cc51", "tree": "e7f5e301247dad450931221c0266856a5e996fdd", "parents": [ "bf18cec52fbbbb0528754a5900f39c9ffbb1d3e5" ], "author": { "name": "Peter Lee", "email": "peterxcli@gmail.com", "time": "Sat Jul 18 01:20:00 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 17 11:20:00 2026 -0600" }, "message": "Report unsupported scalar serde cases in support level (#4745)" }, { "commit": "bf18cec52fbbbb0528754a5900f39c9ffbb1d3e5", "tree": "d5394fd5f4fd4fa44d49e8b23ca00d63b970a5ac", "parents": [ "83339d88914ed1e588f541882f3b6ddc27819325" ], "author": { "name": "Bolin Lin", "email": "linbolin1230@gmail.com", "time": "Fri Jul 17 12:05:15 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 17 09:05:15 2026 -0700" }, "message": "feat: Implement TimeType support - Infrastructure - shuffle (#4398)\n\n* feat: support native shuffle for timetype\n\n* feat: support columnar shuffle\n\n* feat: support TimeType in list shuffle serialization\n\n* feat: support TimeType hash partitioning\n\n* test: cover JVM TimeType range partitioning\n\n* test: refine make_time shuffle coverage" }, { "commit": "83339d88914ed1e588f541882f3b6ddc27819325", "tree": "6620e645ae80bcb5e560324547751c9c22fc4782", "parents": [ "e36a410a2fc3eb7d7c64270c6f401be7d7a23cad" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Fri Jul 17 09:18:11 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 17 11:18:11 2026 -0400" }, "message": "feat: hint at native pyarrow UDF path when the feature is disabled (#4892)\n\n* feat: hint at native pyarrow UDF path when the feature is disabled\n\nWhen spark.comet.exec.pyarrowUdf.enabled is off (the default), mapInArrow\nand mapInPandas operators silently fall back to Spark. Annotate the operator\nwith a non-fallback [COMET-INFO] hint naming the config to opt into the\nnative CometMapInBatchExec path, reusing the existing NativeOptIn wording.\n\nThe hint fires only on operators that would actually run natively if the\nflag were flipped: the eligibility check is unchanged, the conf gate simply\nmoves from the extractor into the rewrite arm.\n\n* docs: remove emphasis from EliminateRedundantTransitions comment" }, { "commit": "e36a410a2fc3eb7d7c64270c6f401be7d7a23cad", "tree": "499e46b90d7e9bebb90e6d1c933d9bad9567aeee", "parents": [ "c65a5ee8032a17ea1597eae0986945e81b787ad3" ], "author": { "name": "Zaki", "email": "140870617+marvelshan@users.noreply.github.com", "time": "Fri Jul 17 22:19:57 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 17 08:19:57 2026 -0600" }, "message": "feat: support size() for MapType input (#4580)" }, { "commit": "c65a5ee8032a17ea1597eae0986945e81b787ad3", "tree": "8e104504c6da9bdca418f62403b216e0ade5e329", "parents": [ "df01cdd3e2561dc857aece39b02d7e666083d285" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Fri Jul 17 07:42:28 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 17 07:42:28 2026 -0600" }, "message": "feat: support gzip compression in native Parquet writes (#4930)\n\n* feat: support gzip compression in the native Parquet writer\n\nAdd Gzip to the CompressionCodec proto enum and introduce a\nParquet-specific ParquetCompression enum in parquet_writer.rs so the\nnative writer can honor gzip without affecting the shuffle codec. The\nplanner maps the new proto variant to ParquetCompression::Gzip, which\nwrites with parquet\u0027s default GzipLevel (6), matching parquet-mr\u0027s zlib\ndefault. The shuffle writer path is unchanged and still rejects Gzip\nvia its catch-all error arm.\n\n* feat: write Parquet natively when the compression codec is gzip\n\n* fix: request parquet-rs codec features explicitly for native Parquet writer\n\nThe native writer\u0027s gzip, snappy, lz4, and zstd support only worked\nbecause Cargo feature unification happened to pull in parquet-rs\u0027s\nflate2/snap/lz4/zstd features via other workspace dependencies. Declare\nthese features directly on the parquet dependency so the native writer\ndoes not silently lose codec support if that unification changes.\n\n* fix: honor parquet.compression precedence and uncompressed codec alias\n\nparseCompressionCodec only checked the compression option and the\nSQLConf default, skipping the parquet.compression option that Spark\u0027s\nown ParquetOptions treats as the middle rung of precedence. Fix the\nlookup to match Spark\u0027s compression, parquet.compression, SQLConf\norder, and accept \"uncompressed\" as an alias for \"none\" since Spark\ndoes the same.\n\nAdd coverage for the parquet.compression option taking precedence over\nthe SQLConf default, for uncompressed as a none alias, and for an\nunsupported codec (brotli) causing the write to fall back to Spark\u0027s\nown writer instead of CometNativeWriteExec.\n\n* refactor: inline single-call-site compression_to_parquet delegate\n\ncompression_to_parquet was a one-line wrapper around\nParquetCompression::to_parquet with a single caller; call it directly\ninstead.\n\n* test: replace brittle brotli fallback test with lz4_raw round trip\n\nThe unsupported-codec fallback test relied on Spark\u0027s write failing\nwith ClassNotFoundException for BrotliCodec, so it only passed because\nthe environment lacks a Brotli codec class rather than because Comet\nrouted the write through the fallback path. Use lz4_raw instead, which\nSpark can write directly via parquet-mr and Comet does not support, so\nthe test can assert a real successful round trip and the correct\nfooter codec instead of intercepting an unrelated failure.\n\nRevert the allowFailure plumbing added to captureWritePlan for that\ntest since it is no longer needed.\n\n* test: skip lz4_raw fallback test on Spark 3.4\n\nSpark 3.4\u0027s PARQUET_COMPRESSION config validates against a fixed set of\ncodecs (brotli, uncompressed, lz4, gzip, lzo, snappy, none, zstd) that\ndoes not include lz4_raw, so withSQLConf threw before the fallback path\nran. Guard the test with assume(isSpark35Plus, ...) so it runs on\nversions where lz4_raw is a valid codec value.\n\n* test: cover full three-tier codec precedence for Parquet writes\n\nAdds a test that pins the `compression` \u003e `parquet.compression` \u003e\n`spark.sql.parquet.compression.codec` precedence: SQLConf is `zstd`,\n`parquet.compression` is `snappy`, and `compression` is `gzip`; the\nwritten file must report GZIP. A leak from either lower layer would\nsurface as a codec mismatch.\n\nThe existing test still covers `parquet.compression` beating the\nSQLConf default when `compression` is absent." }, { "commit": "df01cdd3e2561dc857aece39b02d7e666083d285", "tree": "b21e8a18b5db95cb78aca7290db71601c0260337", "parents": [ "eb24be133b6c381a9d38bbad467e50e97d54094f" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Thu Jul 16 15:34:55 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 16 15:34:55 2026 -0600" }, "message": "perf: optimize spark_regexp_extract in datafusion-comet-spark-expr (#4894)" }, { "commit": "eb24be133b6c381a9d38bbad467e50e97d54094f", "tree": "238c852c9c0179d908390589293e04ca70fe60df", "parents": [ "59c453bfd3f51209d1eb8300e64254b648a2a7d1" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Thu Jul 16 15:34:02 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 16 15:34:02 2026 -0600" }, "message": "perf: optimize `spark_get_json_object` (4x faster) (#4907)\n\n* perf: optimize spark_get_json_object in datafusion-comet-spark-expr\n\n* test: cover streaming stop-early parity guards for get_json_object\n\nAdd regression tests pinning the behaviors that keep the streaming\nDeserializeSeed path walk equivalent to a full parse:\n- trailing garbage after a match is rejected (de.end())\n- a malformed sibling after a match is rejected (visit_map drain)\n- a malformed array element after a match is rejected (visit_seq drain)\n- a duplicated key resolves to its last occurrence (preserve_order parity)\n- a null encountered mid-path yields no match (visit_unit)" }, { "commit": "59c453bfd3f51209d1eb8300e64254b648a2a7d1", "tree": "d7b167fece51b184440f64e3a438ceb784c759ca", "parents": [ "e743a471f8aa64809b815843c8776436b23e56cf" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Thu Jul 16 15:15:18 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 16 15:15:18 2026 -0600" }, "message": "perf: vectorize floating-point-to-decimal cast (#4940)\n\n* perf: vectorize floating-point-to-decimal cast\n\ncast_floating_point_to_decimal128 used a per-element Decimal128Builder loop with\na null check and branch per row. Replace it with a single vectorized unary_opt\npass: values with no in-range integer form (NaN / infinity) or that do not fit\nthe output precision map to null, and the input null buffer is carried over.\n\nANSI mode must raise on out-of-range values rather than nulling them. unary_opt\nonly nulls non-null inputs that overflow, so a null count beyond the input\u0027s\nsignals an overflow; that O(1) check gates a rare element-wise rescan that\nreports the first offending value with Spark\u0027s exact error.\n\nThis is a single pass for every eval mode and shape, so all shapes are faster\n(15-36%) with no regression, including the overflow case. Add unit tests for the\nfast path, ANSI no-overflow, and ANSI overflow-error paths, plus a benchmark.\n\nPart of #4936.\n\n* docs: record float-to-decimal cast performance audit entry\n\n* test: strengthen ANSI overflow assertions for float-to-decimal cast\n\nAddress review feedback on #4940:\n- Match on SparkError::NumericValueOutOfRange and assert exact fields\n instead of a bare is_err(), so a change to error variant/value/precision/\n scale would fail the test.\n- Add a second overflow row (9999.99) with a distinct string form to pin\n that the rescan reports the *first* offender in row order.\n- Add coverage for NaN and Infinity under ANSI mode, a distinct code path\n (closure returns None for non-finite input, rescan must recompute\n fits \u003d\u003d false), and lock the reported value strings (\"NaN\", \"inf\")." }, { "commit": "e743a471f8aa64809b815843c8776436b23e56cf", "tree": "a77f3d9c198f823c532d9e58e844dd2448b455a6", "parents": [ "2e49b6118c1820facc903b804a608ad835bfe73c" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Thu Jul 16 12:44:46 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 16 12:44:46 2026 -0600" }, "message": "perf: skip the null-masking pass in `DecimalRescaleCheckOverflow` when nothing overflows (#4938)\n\n* perf: skip the null-masking pass in DecimalRescaleCheckOverflow when nothing overflows\n\nDecimalRescaleCheckOverflow rescales decimal values and checks precision in a\nsingle try_unary pass, writing i128::MAX as an overflow sentinel. In legacy\n(non-ANSI) mode it then always ran null_if_overflow_precision, a second full pass\nthat allocates a new array to turn the sentinels into nulls, even when no value\noverflowed (the common decimal-cast case).\n\nOnly run that pass when a sentinel is actually present. The check short-circuits\nat the first sentinel, so the overflow path is unaffected while the common\nno-overflow path skips the extra allocation. Non-overflow shapes are 8-26%\nfaster; overflow and ANSI shapes are unchanged within noise across samples.\n\nAdd a legacy test that mixes overflow and null inputs, and a criterion benchmark.\n\nPart of #4936.\n\n* docs: record DecimalRescaleCheckOverflow performance audit entry\n\n* test: cover all-overflow and precision boundary; fix comment wording\n\nAddress review feedback on DecimalRescaleCheckOverflow:\n- Correct the comment to name contains, not any, as the short-circuiting call\n- Add a legacy test where every value overflows (sentinel at index 0)\n- Add a legacy test pinning the 10^p - 1 fits vs 10^p overflows boundary" }, { "commit": "2e49b6118c1820facc903b804a608ad835bfe73c", "tree": "99ea45d50420e93065cf786136665c8da0b8bb53", "parents": [ "2fd3960dfee464b2da5a98c0e754c049d720002b" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Thu Jul 16 07:26:27 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 16 09:26:27 2026 -0400" }, "message": "perf: optimize `CheckOverflow` with a shared no-overflow fast path (ANSI and non-ANSI) (#4937)\n\n* perf: optimize CheckOverflow with a no-overflow fast path\n\nCheckOverflow wraps the result of every decimal add/subtract/multiply/divide,\nsum, and avg, so it runs on the hot path of most TPC-DS queries. The non-ANSI\npath always allocated a new Decimal128 array via null_if_overflow_precision,\neven when no value overflowed (the common case).\n\nAdd a fast path that reuses the input buffers (to_data() clones only cheap Arc\nmetadata) when no value overflows the target precision. The overflow check uses\n`all`, which short-circuits at the first overflow, so the fallback null-masking\npath pays at most a tiny extra scan and does not regress.\n\nAdd array-path unit tests (previously only scalar inputs were covered) and a\ncriterion benchmark over no-overflow, null, sparse-overflow, dense-overflow, and\nANSI shapes.\n\nPart of #4936.\n\n* docs: record CheckOverflow performance audit entry\n\n* perf: extend CheckOverflow fast path to the ANSI branch\n\nThe ANSI (fail_on_error) branch detected overflow with the array-level\nvalidate_decimal_precision, a non-inlined per-value function that also\ncarries the error-formatting machinery. On the common no-overflow shape\nthat made it about 3x slower than the non-ANSI fast path even though it\nalready reused the input buffers.\n\nShare a single cheap fast-path scan built on the inlined\nis_valid_decimal_precision for both modes. When nothing overflows, reuse\nthe input buffers. The ANSI branch now falls back to\nvalidate_decimal_precision only when an overflow is present, purely to\nbuild the precise Spark error. The two checks compare identical\nprecision bounds, so detection is equivalent.\n\nBenchmark: ANSI no-overflow drops from 16.65us to 5.13us (about 69%),\nreaching parity with the non-ANSI fast path. Other shapes unchanged.\n\n* perf: simplify CheckOverflow ANSI overflow error path and fix negative overflow\n\nDrop the redundant validate_decimal_precision scan on the ANSI error\npath. The fast-path scan already proves an overflow exists, so build the\nSpark error from a single find over is_valid_decimal_precision, which\nchecks both precision bounds. This also fixes negative overflow: the old\ncode string-matched \"too large\" and missed the \"too small\" underflow\nbranch, reporting value 0 for an underflowing value.\n\nAdd array-path coverage for negative overflow (legacy null and ANSI\nerror), all-null reuse, all-overflow, and boundary precision." }, { "commit": "2fd3960dfee464b2da5a98c0e754c049d720002b", "tree": "d7ecd9892d512810d4e4b8d8f69c2e67cbe89544", "parents": [ "4c6f5d47362ad194c38663c689d3b4f8d475b0cc" ], "author": { "name": "Oleks V", "email": "comphead@users.noreply.github.com", "time": "Wed Jul 15 17:17:41 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 15 17:17:41 2026 -0700" }, "message": "chore: Support native writer `Overwrite` and `ErrorIfExists` modes (#4946)" }, { "commit": "4c6f5d47362ad194c38663c689d3b4f8d475b0cc", "tree": "f3854c329d6b63a1aa00387f18f0fef15579ea92", "parents": [ "88039f1039ea5d29b9d518e9cbc7e39cb2f8f178" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 15 14:40:19 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 15 14:40:19 2026 -0600" }, "message": "perf: optimize spark_lpad in datafusion-comet-spark-expr (#4919)" }, { "commit": "88039f1039ea5d29b9d518e9cbc7e39cb2f8f178", "tree": "6bc7c9766a7df6eea9e277e8c10099b3770ce1c6", "parents": [ "852303987a600e17e78cbd5079f8c2d8e0e5f47d" ], "author": { "name": "Oleks V", "email": "comphead@users.noreply.github.com", "time": "Wed Jul 15 12:18:14 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 15 12:18:14 2026 -0700" }, "message": "chore: use Datafusion `substring` (#4161)" }, { "commit": "852303987a600e17e78cbd5079f8c2d8e0e5f47d", "tree": "f6dc38a6d869ba154e6da74a55a71c76489fceec", "parents": [ "7e6cc083f88b9c8fb070c30addfd2343a6395a8c" ], "author": { "name": "Oleks V", "email": "comphead@users.noreply.github.com", "time": "Wed Jul 15 12:08:45 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 15 12:08:45 2026 -0700" }, "message": "doc: Document scan tuning for `spark.sql.files.maxPartitionBytes` (#4931)" }, { "commit": "7e6cc083f88b9c8fb070c30addfd2343a6395a8c", "tree": "05a7ace6676c7dad79cfa115829f9b64d2e35ba3", "parents": [ "faf9445279187a39db6770ab59296599757d1631" ], "author": { "name": "Matt Butrovich", "email": "mbutrovich@users.noreply.github.com", "time": "Wed Jul 15 13:59:09 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 15 13:59:09 2026 -0400" }, "message": "test: add version check on an Iceberg test for Variant type (#4935)" }, { "commit": "faf9445279187a39db6770ab59296599757d1631", "tree": "34be66e1b1de50dae616a9a1adb0ffd53eb46874", "parents": [ "5008edd844599bb15211905bde735395403ec758" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 15 11:34:46 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 15 11:34:46 2026 -0600" }, "message": "fix: decode CAST(binary AS string) JVM-compatibly instead of unsafe reinterpret (#4763)\n\n* fix: decode CAST(binary AS string) JVM-compatibly instead of unsafe reinterpret\n\nNative CAST(binary AS string) built a Rust String from non-UTF-8 bytes via\nfrom_utf8_unchecked, which is undefined behaviour (#4488), and an Arrow string\narray holding invalid UTF-8 is unsound for the downstream native string kernels\nthat read it via StringArray::value.\n\nDecode the bytes with decode_utf8_spark_lossy, which replaces ill-formed\nsequences with U+FFFD exactly as the JVM\u0027s new String(bytes, UTF_8) does,\nincluding the surrogate-range cases where Rust\u0027s from_utf8_lossy diverges. The\noutput is always valid UTF-8 and matches Spark\u0027s rendered result byte-for-byte,\nso the cast stays Compatible (native, no opt-in). It diverges from Spark only\nunder byte-level round-trips such as CAST(CAST(x AS string) AS binary).\n\nThis decoder already existed in the native shuffle (#4521); move it into the\nshared spark-expr utils module so shuffle and cast use one implementation.\nDocument the policy in the compatibility guide and the contributor guide.\n\n* fix: route Spark UTF8 binary formatter through JVM-compatible decoder\n\nAddresses review feedback on the CAST(binary AS string) UB fix:\n\n- BinaryOutputStyle::Utf8 previously rendered via\n String::from_utf8(..).unwrap(), panicking the executor on non-UTF-8\n input. With spark.sql.binaryOutputStyle\u003dUTF8 (Spark 4.0+), a\n ToPrettyString/show() over a binary column with invalid bytes hit\n this. Decode lossily instead so invalid bytes become U+FFFD, matching\n Spark\u0027s new String(bytes, UTF_8).\n- Build cast_binary_to_string with a GenericStringBuilder and append the\n decoder\u0027s borrowed Cow, so the common valid-UTF-8 path copies once\n instead of allocating a throwaway String and copying twice.\n- Move decode_utf8_spark_lossy into datafusion-comet-common next to\n bytes_to_i128 so the shuffle crate no longer reaches into spark-expr.\n- Document why the char::from_u32 unwraps in the decoder are infallible.\n\n* docs: document value-identity divergence and pin the decoder behavior in a Spark test\n\nAddresses further review feedback on the CAST(binary AS string) UB fix:\n\n- Add an end-to-end CometCastSuite test over known invalid byte sequences\n (two invalid bytes, the surrogate-range collapse, a truncated lead) that\n checks Comet against Spark and pins the U+FFFD output, so a change in\n decode_utf8_spark_lossy is caught at the Spark level.\n- Document that decoding is not byte-preserving in a second way: distinct\n ill-formed byte sequences all decode to U+FFFD, so strings that differ in\n Spark can compare equal in Comet, which can affect equality, joins,\n grouping, ordering, and byte-based functions such as contains. Previously\n only the round-trip divergence was called out.\n- Fix the contributor-guide reference to the decoder, which now lives in\n datafusion-comet-common rather than the spark-expr utils module." }, { "commit": "5008edd844599bb15211905bde735395403ec758", "tree": "818b97295ec24948f80261cf85571bef68a717bb", "parents": [ "02170ecffa557be98d05f3bf4b09e27ac24e6667" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 15 10:52:19 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 15 10:52:19 2026 -0600" }, "message": "perf: optimize spark_arrays_overlap in datafusion-comet-spark-expr (#4906)" }, { "commit": "02170ecffa557be98d05f3bf4b09e27ac24e6667", "tree": "cbf6e907202d7ff5a853a6523507ef87f0dbd30b", "parents": [ "078fed374179e39818e16188acc345bfc7263474" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 15 10:43:27 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 15 10:43:27 2026 -0600" }, "message": "docs: add scalar expression optimization guide, skill, and performance audits (#4933)\n\n* docs: add scalar expression optimization guide, skill, and performance audits\n\nAdd a contributor guide page documenting how to optimize native scalar\nexpressions in datafusion-comet-spark-expr: benchmark-first workflow,\ncriterion benchmark shapes, a catalog of proven techniques, correctness\nrules, and the no-regression gate. Add a matching optimize-comet-expression\nskill that points at the guide as the shared source of truth.\n\nExtend the per-expression audit pages to record performance audits alongside\ncorrectness audits, and backfill entries for the recently tuned cast, to_json,\nparse_url, size, and unhex expressions.\n\n* docs: harden optimize-comet-expression skill with baseline-hygiene and noise-vs-regression lessons\n\n* docs: add unary_opt technique for null-producing conversions to optimization guide\n\n* docs: apply prettier formatting to optimize-comet-expression skill" }, { "commit": "078fed374179e39818e16188acc345bfc7263474", "tree": "34e59cefb788aa12dca8568402d5e36d4d5ebd46", "parents": [ "275d2ae1a7db342f2951682ddf96fdad71709d0d" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Wed Jul 15 10:42:56 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 15 10:42:56 2026 -0600" }, "message": "perf: optimize try_arithmetic_kernel in datafusion-comet-spark-expr (#4910)" }, { "commit": "275d2ae1a7db342f2951682ddf96fdad71709d0d", "tree": "38c502884304ff1b32222b2cf99461b9ae1fe7e1", "parents": [ "0af2977a1e4ff3a433c2d97c6d928d87347a60f5" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 14 22:16:48 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 14 22:16:48 2026 -0600" }, "message": "fix: suppress spurious \"WriteFilesExec is not supported\" fallback (#4928)\n\nSpark\u0027s V1Writes rule wraps native writes as\nDataWritingCommandExec(cmd, WriteFilesExec(child)). CometExecRule\nconverts the DataWritingCommandExec to CometNativeWriteExec and\nunwraps the WriteFilesExec inside convertToComet, so the write\nruns natively end-to-end.\n\nHowever, transformUp visits WriteFilesExec before its parent.\nNo case in the outer match handles it, so it falls through to\nthe generic \"op.nodeName is not supported\" tag at the bottom.\nThe tag is set on the WriteFilesExec instance itself (which is\nthen discarded when the parent is converted), so the executed\nplan looks correct - but with COMET_LOG_FALLBACK_REASONS\u003dtrue,\nthe log emits a misleading warning:\n\n Comet cannot accelerate WriteFilesExec because:\n WriteFilesExec is not supported\n\nAdd WriteFilesExec to the list of operators that should never be\ntagged with a generic fallback reason." }, { "commit": "0af2977a1e4ff3a433c2d97c6d928d87347a60f5", "tree": "700117fa68fab77096ec0ac4542c30dee627d06f", "parents": [ "32caa13d000dbfc2458b68a7052ae4114f59c4d1" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Tue Jul 14 12:06:35 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 14 12:06:35 2026 -0600" }, "message": "perf: optimize spark_cast_int_to_int in datafusion-comet-spark-expr (#4920)" }, { "commit": "32caa13d000dbfc2458b68a7052ae4114f59c4d1", "tree": "be0e1c4d72018d16a0fcc0da43d21e3bbf151012", "parents": [ "dd30f411d0fc704c1c5f576299fd0a424aff4624" ], "author": { "name": "Oleks V", "email": "comphead@users.noreply.github.com", "time": "Tue Jul 14 09:02:18 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Jul 14 09:02:18 2026 -0700" }, "message": "chore: Codegen fallback explain for plan (#4891)" }, { "commit": "dd30f411d0fc704c1c5f576299fd0a424aff4624", "tree": "89da5a18f054790e2a3ec35025f5a95868fc3c41", "parents": [ "a0f3acf65aec3ee83554390bd7a610f36fcd43ca" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Mon Jul 13 15:24:14 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jul 13 15:24:14 2026 -0600" }, "message": "perf: optimize escape_string in datafusion-comet-spark-expr (#4902)" }, { "commit": "a0f3acf65aec3ee83554390bd7a610f36fcd43ca", "tree": "5fedad9f375491c7ffb626f9ceb3d0136ac53b65", "parents": [ "8e7cee8189b49c724ebf03eac8e17f4815a842c4" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Mon Jul 13 15:15:31 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jul 13 17:15:31 2026 -0400" }, "message": "test: cover invalid unhex inputs in SQL file tests (#4890)" }, { "commit": "8e7cee8189b49c724ebf03eac8e17f4815a842c4", "tree": "7fa4f643a61f633b15c91e906496377c2fdbf6de", "parents": [ "691e2ae948432dadbb8be752000d2fe1b2feade8" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Mon Jul 13 14:11:15 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jul 13 14:11:15 2026 -0600" }, "message": "fix: restrict array_filter array_compact fast path to the lambda variable (#4848)" }, { "commit": "691e2ae948432dadbb8be752000d2fe1b2feade8", "tree": "10688f8185c498c726bd5c48bfd41823c1b480df", "parents": [ "b9cbb6882b6aa2f5c7e6a77dd63ddf471d44ece9" ], "author": { "name": "Oleks V", "email": "comphead@users.noreply.github.com", "time": "Mon Jul 13 11:07:36 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jul 13 11:07:36 2026 -0700" }, "message": "chore: remove `hdfs` Comet crate (#4904)" }, { "commit": "b9cbb6882b6aa2f5c7e6a77dd63ddf471d44ece9", "tree": "89b325da15fa26323ef2835fd48d7ed7b3d582b9", "parents": [ "824fc4274074a07c4ea2de6c9db943e3a2210995" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Sat Jul 11 11:20:21 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Jul 11 11:20:21 2026 -0600" }, "message": "perf: optimize parse_url in datafusion-comet-spark-expr (#4893)" }, { "commit": "824fc4274074a07c4ea2de6c9db943e3a2210995", "tree": "704b0ae7528a6bd4aca54f5812e59a4e30d5bf29", "parents": [ "b7ad0ba277409e88f76a993c83da95e7b3b5775d" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Fri Jul 10 23:32:36 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 10 23:32:36 2026 -0600" }, "message": "perf: optimize spark_unhex in spark-expr (#4876)" }, { "commit": "b7ad0ba277409e88f76a993c83da95e7b3b5775d", "tree": "349dc8b299900006286f4dd2148992ba334a72dd", "parents": [ "edb7b65c983d73063bb5060e097df3711ccaf122" ], "author": { "name": "Scott Schenkein", "email": "schenksj@yahoo.com", "time": "Fri Jul 10 17:51:06 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 10 14:51:06 2026 -0700" }, "message": "feat: core SPI for contrib leaf scans (CometScanWithPlanData) [Delta contrib split, part 1] (#4700)\n\n* feat: core SPI for contrib leaf scans (CometScanWithPlanData)\n\nIntroduce a small extension contract so out-of-tree Comet contrib leaf scans\n(Delta, and future Hudi/etc.) can participate in native planning without core\nholding a compile-time reference to them -- mirroring the Iceberg-precedent of\nkeeping the data-source-specific code at the edge.\n\nWhat this adds:\n- `trait CometScanWithPlanData` (`sourceKey` / `commonData` / `perPartitionData`,\n plus optional `dynamicPruningFilters` / `withDynamicPruningFilters` for scans\n whose DPP filters live in a @transient field). `CometNativeScanExec` now mixes\n it in.\n- `CometNativeExec.foreachUntilCometInput` matches `case _: CometLeafExec` (a\n strict superset of the previous fixed scan enumeration -- all built-in leaf\n scans already extend `CometLeafExec`), so any leaf Comet exec is recognised as\n an input boundary.\n- `PlanDataInjector.findAllPlanData` collects per-partition planning data via the\n trait instead of a hardcoded `CometNativeScanExec` match.\n- `PlanDataInjector`\u0027s registry gains one reflective `DeltaPlanDataInjector$`\n slot, appended only when a contrib bundled it (`-Pcontrib-delta`). Default\n builds get a `ClassNotFoundException` -\u003e `None` and an unchanged injectors list,\n so there is zero contrib surface at runtime.\n- `CometPlanAdaptiveDynamicPruningFilters` rewrites AQE DPP filters in place for\n trait scans whose filters can\u0027t survive `makeCopy` (#3510).\n\nInert by construction: with no contrib on the classpath this is behavior-\npreserving (the leaf match is a superset; the trait match catches the same\n`CometNativeScanExec`; the reflective slot resolves to nothing).\n\nTests: `CometScanWithPlanDataSuite` (trait-contract defaults + reflective-slot\ngraceful absence). Verified `CometJoinSuite` (native scan fusion / DPP) stays\ngreen.\n\nFirst unit of the Delta-contrib PR split (tracking: #4366).\n\n* refactor: address review feedback on contrib leaf-scan SPI\n\nparthchandra:\n- Discover contrib PlanDataInjectors via java.util.ServiceLoader\n (META-INF/services) instead of a hardcoded reflective DeltaPlanDataInjector$\n slot, so contribs are discoverable without editing core.\n- Catch via scala.util.control.NonFatal (covers ServiceConfigurationError).\n- findAllPlanData\u0027s generic trait arm now mirrors the Iceberg arm: empty\n common/per-partition data contributes nothing instead of an empty entry\n that would fail downstream injection.\n- Add a stub CometLeafExec with CometScanWithPlanData wired through\n findAllPlanData, confirming the new path works independently of the\n existing scan classes; plus a ServiceLoader discovery test.\n\nandygrove:\n- Make CometScanWithPlanData a self-type `{ self: CometLeafExec \u003d\u003e }`, so\n \"is a leaf scan\" is a compile-time requirement and findAllPlanData drives\n the subquery lifecycle with no silent \"not a leaf\" fallback.\n- Mirror the TODO(#3510) marker on withDynamicPruningFilters.\n- Pre-emptively exclude CometScanWithPlanData from the non-Comet DPP arm in\n CometPlanAdaptiveDynamicPruningFilters so a future trait scan with a wrapped\n SAB but empty dynamicPruningFilters isn\u0027t misrouted.\n\nAlso relocate findAllPlanData from CometNativeExec into the PlanDataInjector\nobject (it uses no instance state), which fixes the now-correct\nPlanDataInjector.findAllPlanData scaladoc reference and makes it unit-testable.\n\nCometScanWithPlanDataSuite 5/5 on Spark 4.1/2.13; spark module also compiles\nclean under Scala 2.12 (Spark 3.4).\n\nCo-Authored-By: Claude Opus 4.8 (1M context) \u003cnoreply@anthropic.com\u003e\nClaude-Session: https://claude.ai/code/session_01BtErWgRQKCDRAg8Mk6qR4G\n\n* docs: fix findAllPlanData scaladoc references after relocation\n\nfindAllPlanData moved from CometNativeExec into the PlanDataInjector object;\nupdate the now-stale `CometNativeExec.findAllPlanData` references in the\nCometNativeScanExec and CometIcebergNativeScanExec doc comments to\n`PlanDataInjector.findAllPlanData`.\n\nCo-Authored-By: Claude Opus 4.8 (1M context) \u003cnoreply@anthropic.com\u003e" }, { "commit": "edb7b65c983d73063bb5060e097df3711ccaf122", "tree": "e9c5158f69e2357e1f7c9b51fd24a59a1e43213f", "parents": [ "1142c3a709805f210bedd3a2fff0f2b434c2e387" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Fri Jul 10 10:07:57 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Jul 10 12:07:57 2026 -0400" }, "message": "perf: optimize spark_size in spark-expr (#4877)" }, { "commit": "1142c3a709805f210bedd3a2fff0f2b434c2e387", "tree": "e2b82e4ce73a70117f1201ad00e24fd7d98fbef7", "parents": [ "20279ba51449246d48ed82af02f752531d8637ed" ], "author": { "name": "Oleks V", "email": "comphead@users.noreply.github.com", "time": "Thu Jul 09 14:09:14 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 09 14:09:14 2026 -0700" }, "message": "chore: fallback for `spark.sql.mapKeyDedupPolicy` \u003d\u003d `LAST_WIN` (#4863)" }, { "commit": "20279ba51449246d48ed82af02f752531d8637ed", "tree": "cea02ccd012c367da39ce5ce7bade53d7c63804b", "parents": [ "ff41f9648efc252e015164ac1dd7561133aee29e" ], "author": { "name": "hsiang-c", "email": "137842490+hsiang-c@users.noreply.github.com", "time": "Thu Jul 09 13:19:39 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 09 14:19:39 2026 -0600" }, "message": "feat: remove constraint on array of nested elements (#4714)" }, { "commit": "ff41f9648efc252e015164ac1dd7561133aee29e", "tree": "cf53b1b680be58cf9eb8beacb9b05651b312968c", "parents": [ "eb5b7615ae0d9b662bf03e15f5f5de666d527d2f" ], "author": { "name": "Zhen Wang", "email": "643348094@qq.com", "time": "Fri Jul 10 04:18:46 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 09 14:18:46 2026 -0600" }, "message": "refactor(shuffle): Introduce PartitionWriter interface to decouple shuffle partitioning from storage (#4779)" }, { "commit": "eb5b7615ae0d9b662bf03e15f5f5de666d527d2f", "tree": "40dd7af86b64e2149c314bc6ac9111e052bba3ee", "parents": [ "d4fbc1663ad39a415c66090c82081abb40062ef6" ], "author": { "name": "Andy Grove", "email": "agrove@apache.org", "time": "Thu Jul 09 12:47:17 2026 -0600" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jul 09 12:47:17 2026 -0600" }, "message": "feat: support approx_percentile / percentile_approx aggregate (#4801)" } ], "next": "d4fbc1663ad39a415c66090c82081abb40062ef6" }