)]}' { "log": [ { "commit": "0857b0010a2b530ef770b21a8f3732fda29feeb5", "tree": "c1c16469817350bad0394ff391f83107e31b39a1", "parents": [ "0c4a180c11b9453ddc5461488c9acc0c54863f94" ], "author": { "name": "rdtr", "email": "norio@apache.org", "time": "Fri Oct 30 00:00:42 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 30 00:00:42 2026 -0700" }, "message": "[GLUTEN-13179][VL] Re-enable the Spark 4.2 null-named attribute test (#13260)" }, { "commit": "0c4a180c11b9453ddc5461488c9acc0c54863f94", "tree": "c1c16469817350bad0394ff391f83107e31b39a1", "parents": [ "3071f767a0f260804c6904bba74f51d9ce9dd94e" ], "author": { "name": "sabbasani", "email": "152367489+sabbasani@users.noreply.github.com", "time": "Fri Oct 30 11:17:56 2026 +0530" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 30 13:47:56 2026 +0800" }, "message": "[GLUTEN-12861] Fix GlutenCachedTableSuite silently losing spark.sql.shuffle.partitions\u003d5 (#13287)\n\n## Why are the changes needed?\n\n`SharedSparkSessionBase.sparkConf` is a `def` that creates a new `SparkConf` instance on every invocation. The existing override called `super.sparkConf` twice, causing `spark.sql.shuffle.partitions\u003d5` to be set on a different instance and discarded.\n\nAs a result, the test suite silently used the default shuffle partition count of 1 instead of the intended value of 5. This could lead to testing cached table behavior under an unintended partitioning configuration.\n\nAdditionally, `sys.props.put()` introduced an unnecessary JVM-wide system property side effect, and its comment incorrectly stated that it disabled the columnar table cache.\n\n## What changes are proposed in this pull request?\n\n- Chain both `.set()` calls on a single `super.sparkConf` instance.\n- Ensure `spark.sql.shuffle.partitions\u003d5` and `COLUMNAR_TABLE_CACHE_ENABLED\u003dtrue` are applied to the same configuration.\n- Remove the redundant `sys.props.put()` JVM-wide system property modification and its misleading comment.\n- Apply the fix consistently across Spark 3.4, 3.5, 4.0, and 4.1 modules.\n\n## How was this patch tested?\n\n- Verified the `sparkConf` override uses a single `SparkConf` instance.\n- Verified the changes across all four Spark version modules.\n- Ran `GlutenCachedTableSuite` to validate the changes.\n\n## Was this patch authored or co-authored using generative AI tooling?\n\nyes.\n\n\nRelated issue: #12861" }, { "commit": "3071f767a0f260804c6904bba74f51d9ce9dd94e", "tree": "bfa3716ffb1f1a5cc9be00a97298d3583756684e", "parents": [ "e681b63d47aa5bffdc8db4eb0ee37f43fd8b5ced" ], "author": { "name": "sabbasani", "email": "152367489+sabbasani@users.noreply.github.com", "time": "Fri Oct 30 07:58:09 2026 +0530" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 30 10:28:09 2026 +0800" }, "message": "[GLUTEN-12859][TEST] Restore unhex null assertions across Spark modules (#13319)\n\n## What changes were proposed in this pull request?\n\nIn `GlutenMathExpressionsSuite`, assertions verifying that `unhex` returns `null` for invalid hex inputs (e.g., `\"GG\"`, `\"三重的\"`) were previously commented out because the ClickHouse backend does not return `null` on invalid hex inputs (see #7232). Commenting out these lines disabled test coverage for backends like Velox that conform to Spark\u0027s null-returning behavior.\n\nThis PR:\n1. Splits the invalid-input assertions out of `testGluten(\"unhex\")` into a dedicated test: `testGluten(\"unhex returns null for invalid hex input\")`.\n2. Restores the null assertions in `GlutenMathExpressionsSuite` across all Spark versions (`spark34`, `spark35`, `spark40`, `spark41`, `spark42`).\n3. Excludes only `unhex returns null for invalid hex input` in `ClickHouseTestSettings.scala` for each Spark version (referencing #7232), ensuring ClickHouse still validates valid-input `unhex` behavior while other backends test both valid and invalid input handling.\n\n## Why are the changes needed?\n\n- Fixes [GLUTEN-12859](https://github.com/apache/gluten/issues/12859).\n- Re-enables full test coverage for Spark-compliant `unhex` null behavior on invalid input for Velox backend without breaking ClickHouse CI.\n- Allows ClickHouse to keep running valid-input `unhex` tests rather than leaving invalid test lines commented out in common test suites.\n\n## How was this patch tested?\n\n- Tested `GlutenMathExpressionsSuite` across Spark modules:\n ```bash\n ./build/mvn test -pl gluten-ut -Pspark-ut -Pbackends-velox -Pspark-3.5 -Pscala-2.12 -Dtest\u003dGlutenMathExpressionsSuite\n ./build/mvn test -pl gluten-ut -Pspark-ut -Pbackends-clickhouse -Pspark-3.5 -Pscala-2.12 -Dtest\u003dGlutenMathExpressionsSuite\n \n## Was this patch authored or co-authored using generative AI tooling?\nNo" }, { "commit": "e681b63d47aa5bffdc8db4eb0ee37f43fd8b5ced", "tree": "8964929ea2f8296f7916b69dbf92736528c50fc1", "parents": [ "9801f4169f820b50d9a0e6d71b1f5e6e0085c320" ], "author": { "name": "Felipe Pessoto", "email": "fepessot@microsoft.com", "time": "Thu Oct 29 14:20:52 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 29 14:20:52 2026 -0700" }, "message": "[MINOR][VL] Configure Gluten for all Delta Spark test contexts (#13131)" }, { "commit": "9801f4169f820b50d9a0e6d71b1f5e6e0085c320", "tree": "d8ac7365767ae4cdeb74fd68938222e2cdf2ec05", "parents": [ "d0d5105e01bf9cc67be654b9dac5f3ce3a96254d" ], "author": { "name": "Felipe Pessoto", "email": "fepessot@microsoft.com", "time": "Thu Oct 29 13:32:03 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 29 13:32:03 2026 -0700" }, "message": "[VL] Fix Parquet metadata scans with same-named data columns (#13136)" }, { "commit": "d0d5105e01bf9cc67be654b9dac5f3ce3a96254d", "tree": "c5288921e9b3ee15d4bf682a0b1753107e64aca4", "parents": [ "5dc5f002b0f5c67116e8137b2fccfad1d2a5b60c" ], "author": { "name": "BInwei Yang", "email": "felixybw@apache.org", "time": "Thu Oct 29 08:17:56 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 29 15:17:56 2026 +0000" }, "message": "[VL][CI] Bake CI runtime tools into the apache/gluten images (#13422)" }, { "commit": "5dc5f002b0f5c67116e8137b2fccfad1d2a5b60c", "tree": "525fcf6c89963e20b5c0ea378dec2816c5405ef9", "parents": [ "ce3ca70109259f9330ed9d2a2ceae6ac57aaed3b" ], "author": { "name": "Yuan", "email": "yuanzhou@apache.org", "time": "Thu Oct 29 15:06:30 2026 +0000" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 29 15:06:30 2026 +0000" }, "message": "[VL] Keep Velox SIMD padding on shuffle reader buffers (#13337)" }, { "commit": "ce3ca70109259f9330ed9d2a2ceae6ac57aaed3b", "tree": "4881b72448e77f6c61b9d2a4b95480df4cbe8aa1", "parents": [ "3ab17271eca2036449f1c47a8d032c4fa09d48c9" ], "author": { "name": "Shivadarshan Devadiga", "email": "shivadarshan2212@gmail.com", "time": "Thu Oct 29 18:34:41 2026 +0530" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 29 13:04:41 2026 +0000" }, "message": "[GLUTEN-12856][TEST] Use withTable for table cleanup (#13405)" }, { "commit": "3ab17271eca2036449f1c47a8d032c4fa09d48c9", "tree": "1d2bbc73aa9d6544926645c7d9874e79f73f8b15", "parents": [ "192ff9a04df1c2aaa473c672cf77a816bddc0db4" ], "author": { "name": "kevinwilfong", "email": "kevinwilfong@fb.com", "time": "Thu Oct 29 02:43:00 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 29 17:43:00 2026 +0800" }, "message": "[CORE] Fix nested field rebinding after schema pruning (#13478)" }, { "commit": "192ff9a04df1c2aaa473c672cf77a816bddc0db4", "tree": "6c2b996062e4c466961ba89a5a84e1f6b2e8828e", "parents": [ "dc64bba1388871691c12faba0c36eb6cacbcac96" ], "author": { "name": "Anshul Baliga", "email": "106268157+anshulbaliga7@users.noreply.github.com", "time": "Thu Oct 29 14:23:06 2026 +0530" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 29 08:53:06 2026 +0000" }, "message": "[VL] Offload str_to_map under the LAST_WIN map key dedup policy (#13462)" }, { "commit": "dc64bba1388871691c12faba0c36eb6cacbcac96", "tree": "cd7eb07cfea8b7a42bee35ab3b48bab57a95924e", "parents": [ "c0f111989951083287ca61f2b62883c53333a96b" ], "author": { "name": "wud", "email": "75607097+wud-if@users.noreply.github.com", "time": "Thu Oct 29 15:49:49 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 29 15:49:49 2026 +0800" }, "message": "[GLUTEN-13369][FLINK] Support function IFNULL (#13375)\n\n* Support IFNULL offload to the velox flink sql function" }, { "commit": "c0f111989951083287ca61f2b62883c53333a96b", "tree": "480f6955958ceae08b54c875b179e976f6be574f", "parents": [ "d20167613814d11286fe32a9dbce1569fe8c6909" ], "author": { "name": "Kaifei Yi", "email": "yikaifei@apache.org", "time": "Thu Oct 29 15:28:27 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 29 00:28:27 2026 -0700" }, "message": "[VL] Distinguish driver and executor defaults for the broadcast hash-table build thread pool (#13368)" }, { "commit": "d20167613814d11286fe32a9dbce1569fe8c6909", "tree": "c2116346e5cfa3aedc1547c6173a8fa92fbc5588", "parents": [ "5e505c799fd011c29302e52317bd04a224fa0803" ], "author": { "name": "rdtr", "email": "norio@apache.org", "time": "Wed Oct 28 12:43:35 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 28 19:43:35 2026 +0000" }, "message": "[GLUTEN-13275][VL] Check out the PR in the ANSI workflow test jobs (#13371)\n\nThe test jobs in the ANSI workflow check out needs.check-comment.outputs.pr_sha, but they only list build-native-lib in needs. The needs context only has the outputs of direct dependencies, so pr_sha is empty, and actions/checkout falls back to the default branch for an issue_comment event. So the tests ran the code of main, while the native library was built from the PR.\n\nList check-comment in needs of the four test jobs." }, { "commit": "5e505c799fd011c29302e52317bd04a224fa0803", "tree": "ad23d0684de45a5c9a0342453e840002a6ef1b9e", "parents": [ "381c806ec2d65ed3262b67407f2e604902f11e49" ], "author": { "name": "Felipe Pessoto", "email": "fepessot@microsoft.com", "time": "Wed Oct 28 11:09:08 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 28 11:09:08 2026 -0700" }, "message": "[GLUTEN-12753][CI] Match Delta CDF filter assertions to Gluten plans (#13129)" }, { "commit": "381c806ec2d65ed3262b67407f2e604902f11e49", "tree": "969d2d746733130ec4844755e932d443361920ed", "parents": [ "febd08bd5f9628f46f13dee5c299159a0de88dc0" ], "author": { "name": "Reema", "email": "80041251+ReemaAlzaid@users.noreply.github.com", "time": "Wed Oct 28 15:15:31 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 28 12:15:31 2026 +0000" }, "message": "[VL] Extend cuDF functions in Gluten with row_constructor_with_null (#13104)" }, { "commit": "febd08bd5f9628f46f13dee5c299159a0de88dc0", "tree": "a140b4d2172454e30a583f0111207d945e3d9238", "parents": [ "757ae0759c3c5452d9e8469933f15f2d71d5b2f9" ], "author": { "name": "Yuan", "email": "yuanzhou@apache.org", "time": "Tue Oct 27 23:46:21 2026 +0000" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 27 23:46:21 2026 +0000" }, "message": "[CORE] Bump version to 1.9.0-SNAPSHOT (#13475)" }, { "commit": "757ae0759c3c5452d9e8469933f15f2d71d5b2f9", "tree": "8798510bf71b906240468f547024e87e90ba611b", "parents": [ "12f0f5170619de789001546ce9faaf63ed969ff5" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Tue Oct 27 10:21:20 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 27 10:21:20 2026 -0700" }, "message": "[GLUTEN-13176][VL] Pass RESPECT NULLS of collect_set to Velox and re-enable two Spark 4.2 tests (#13441)" }, { "commit": "12f0f5170619de789001546ce9faaf63ed969ff5", "tree": "b9d684fcfc474c90d7db6ac71c7746cac48261f9", "parents": [ "0c322e85884d2ed5d65e10d64f2f87759e997ed3" ], "author": { "name": "sabbasani", "email": "152367489+sabbasani@users.noreply.github.com", "time": "Tue Oct 27 19:46:19 2026 +0530" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 27 14:16:19 2026 +0000" }, "message": "[VL] Respect caseSensitiveAnalysis in IcebergScanTransformer metadata-column handling (#12726)\n\n* Fix] Respect caseSensitiveAnalysis in IcebergScanTransformer metadata-column handling\n\n* [Fix] Respect caseSensitiveAnalysis in PushDownInputFileExpression\n\nTwo remaining unconditional toLowerCase(Locale.ROOT) calls in\nPushDownInputFileExpression were inconsistent with spark.sql.caseSensitive:\n\n1. containsInputFileRelatedExpr: used INPUT_FILE_ATTR_NAMES.contains(\n a.name.toLowerCase) to detect metadata attribute references.\n Under caseSensitive\u003dtrue this incorrectly matched a user column named\n e.g. \u0027Input_File_Name\u0027 as a metadata expression.\n Fixed by using SQLConf.get.resolver, which honours the session\n case-sensitivity setting exactly as the rest of the engine does.\n\n2. PostOffload dedup (withOutput injection): used\n child.output.map(_.name.toLowerCase).toSet to avoid re-injecting a\n metadata attribute already present in the scan output.\n Under caseSensitive\u003dtrue a user column whose lowercased name equals a\n metadata constant (e.g. \u0027Input_File_Name\u0027 -\u003e \u0027input_file_name\u0027) would\n be treated as a duplicate, causing the injected attr to be dropped\n while the rewritten project list still held a reference to it,\n producing a dangling-attribute IllegalStateException.\n Fixed by deduplicating on expression ID instead of name: the injected\n aliases are freshly created (new exprId) so identity is the correct\n key.\n\nTogether with the IcebergScanTransformer fix (2638ce6e5) this closes the\ncase-sensitivity gap in the Iceberg metadata-column push-down path.\n\n* changes for Gluten fix for case-sensitive metadata fallback\n\n* Fix trailing whitespace in IcebergSuite.scala to resolve scalastyle violation\n\n* Fix] Respect caseSensitiveAnalysis in IcebergScanTransformer metadata-column handling\n\n* changes for casesensitive\n\n* [Docs] Update velox-backend-limitations.md: reflect both case-sensitive fixes\n\n- Remove stale \u0027Known follow-up\u0027 note about PushDownInputFileExpression\n (that fix has since been committed in b7568172a).\n- Add PushDownInputFileExpression bullet alongside the IcebergScanTransformer\n entry in the \u0027This PR addresses\u0027 section.\n- List remaining unverified areas under \u0027Remaining limitations\u0027.\n\n* case-sensitive-iceberg-fix\n\n* case-sensitive-iceberg-fix\n\n* case-sensitive-iceberg-fix\n\n* case-sensitive-iceberg-fix\n\n* case-sensitive-iceberg-fix\n\n* case-sensitive-iceberg-fix\n\n* Add LeftSingle join support to BroadcastNestedLoopJoinExecTransformer\n\nLeftSingle is a Spark 4.0+ join type for scalar subquery correlation that is\nsemantically equivalent to LeftOuter. The test \u0027Left single join should not\nresult into exception\u0027 uses runQueryAndCompare with noFallBack\u003dtrue, which\nrequires BroadcastNestedLoopJoin with LeftSingle to be offloaded natively.\n\nChanges:\n- BroadcastNestedLoopJoinExecTransformer: add LeftSingle to validateJoinTypeAndBuildSide()\n (allowed with BuildRight, rejected with BuildLeft like LeftOuter) and to\n outputPartitioning (returns left.outputPartitioning like LeftOuter).\n- SubstraitUtil.toNestedLoopJoinSubstrait: map LeftSingle to JOIN_TYPE_LEFT\n (same as LeftOuter, since we always build on the right side in Substrait).\n\n* Merge conflict resolution\n\n* Revert unrelated changes from case-sensitive-iceberg-fix branch\n\nRemove four files that are not part of the core case-sensitive fix:\n\n- ProjectColumnPruning.scala: revert agg buffer (inputAggBufferAttributes)\n inclusion in getReferences — unrelated side-effect, not required for\n case-sensitive metadata-column resolution.\n\n- ImplementJoinAggregate.scala: revert grouping-only projection in\n unpackInputBufferFields — unrelated side-effect, not required for\n case-sensitive metadata-column resolution.\n\n- BroadcastNestedLoopJoinExecTransformer.scala: revert LeftSingle join\n support (Spark 4.0+) — separate unrelated feature.\n\n- SubstraitUtil.scala: revert LeftSingle → JOIN_TYPE_LEFT mapping —\n accompanies the BroadcastNestedLoopJoin revert above.\n\nThe remaining PR scope is the core case-sensitive fix only:\n IcebergScanTransformer, GlutenIcebergSourceUtil, BasicScanExecTransformer,\n PushDownInputFileExpression, CHIteratorApi, DeltaOptimizedWriterTransformer,\n GlutenDeltaOptimizedWriterExec.\n\n* test: strengthen caseSensitive\u003dfalse fallback assertion in FallbackSuite\n\nThe PushDownInputFileExpression caseSensitive\u003dfalse test previously only\nverified correctness of result values, which is insufficient: if\nhasVeloxColumnNameConflict() were removed the fallback would not trigger,\nGluten would attempt native execution, and Velox would crash at runtime\n(INVALID_STATE: two conflicting column handles for the same case-insensitive\nname). The regression would not be caught by value comparison alone because\nrunQueryAndCompare falls back to Spark for the comparison.\n\nAdd two structural plan assertions inside the runQueryAndCompare callback:\n 1. collect(plan) { FileSourceScanExecTransformer }.isEmpty -- no native scan\n 2. collect(plan) { FileSourceScanExec }.nonEmpty -- vanilla scan present\n\nThese assertions will fail immediately if the Velox collision guard is ever\ninadvertently removed, making the regression visible at test time rather than\nas a production crash.\n\nAlso add FileSourceScanExec to the import list in FallbackSuite.\n\n* fix coding style\n\n* style: fix spotless closing-paren formatting in FallbackSuite\n\nMove closing \u0027)\u0027 of assert() call to its own line to satisfy\nspotless/scalafmt style rules.\n\n* test: fix PATH_ALREADY_EXISTS in PushDownInputFileExpression Parquet tests\n\nwithTempDir (from Spark\u0027s SQLTestUtils) creates and pre-initialises the\ntemp directory before handing it to the lambda body. Both PushDownInput-\nFileExpression Parquet tests called df.write.format(\"parquet\").save(dir)\nwithout a save mode, which defaults to ErrorIfExists. Because the directory\nalready exists, Spark throws:\n\n AnalysisException: [PATH_ALREADY_EXISTS] Path file:/tmp/spark-...\n already exists. Set mode as \"overwrite\" to overwrite.\n\nFix: add .mode(\"overwrite\") to both df.write chains, matching the pattern\nused by all other withTempDir + parquet write tests in the same suite\n(VeloxScanSuite, VeloxParquetDataTypeValidationSuite, etc.).\n\nNo production logic is changed.\n\n* changes for doc changes\n\n* changes for fix to case sensitive\n\n* changes to casesensitive bug\n\n* changes to casesensitive bug\n\n* changes to casesensitive bug\n\n* changes to casesensitive bug\n\n* fix: restore caseInsensitiveResolution in CH DeltaOptimizedWriterTransformer\n\nDelta partition column lookup must always be case-insensitive regardless of\nspark.sql.caseSensitive. Delta metadata stores canonical partition names (e.g.\n\u0027Part\u0027) while the incoming DataFrame may use any case variant (e.g. \u0027part\u0027).\n\nPR #12726 mistakenly replaced caseInsensitiveResolution with SQLConf.get.resolver\nin both outputPartitioning and getShuffleRDD. Under caseSensitive\u003dtrue this breaks\noptimized writes with case-mismatched partition column names:\n [DELTA_FAILED_FIND_PARTITION_COLUMN_IN_OUTPUT_PLAN] Could not find Part in output plan.\n\nSame fix as backends-velox/src-delta33 GlutenDeltaOptimizedWriterExec (commit bdae95857).\nAlso removes the now-unused SQLConf import.\n\n* test: add InputFileNameCollisionSuite regression tests for PR #12726\n\nAdd InputFileNameCollisionSuite to backends-velox test sources.\n\nThis suite provides regression coverage for the collision between a\nmixed-case user column named \u0027Input_File_Name\u0027 and the\nPushDownInputFileExpression-injected \u0027input_file_name\u0027 metadata attr\nunder caseSensitive\u003dfalse.\n\nTests cover:\n- Combined input_file_name() + Input_File_Name user column queries\n under both caseSensitive\u003dtrue and caseSensitive\u003dfalse\n- Reverse projection order (input_file_name() first)\n- Aliased user column + input_file_name()\n- Multiple input_file_name() calls in same query\n- Filter + input_file_name()\n- Standalone input_file_name() queries\n- Data values not replaced by file paths (core bug regression)\n- input_file_name() through UNION ALL (tail-branch metadata fix)\n- Control tests with non-colliding columns\n\nAll 19 tests pass after the implementation changes in\nPushDownInputFileExpression, BasicScanExecTransformer,\nFileMetadataUtil, IteratorApi, and VeloxIteratorApi.\n\nfix: guard PostOffload BatchScan case against fallback-tagged ProjectExec (Regular/Synthesized Velox conflict)\n\nWhen caseSensitive\u003dtrue (or caseSensitive\u003dfalse) and the physical table contains\na column named exactly \u0027input_file_name\u0027, PushDownInputFileExpression.PostOffload\nwas matching the fallback-tagged inner ProjectExec created by PreOffload for\nBatchScanExecTransformerBase (Iceberg/BatchScan tables) and pushing a Synthesized\n\u0027input_file_name\u0027 attribute into the scan\u0027s output. The scan already had a Regular\n\u0027input_file_name\u0027 column, causing Velox to throw:\n\n VeloxRuntimeError: Inconsistent column handle type: input_file_name,\n expected Regular, got Synthesized\n\nRoot cause:\n PostOffload\u0027s \u0027ProjectExec(_, child: BatchScanExecTransformerBase)\u0027 pattern\n matches any ProjectExec wrapping a BatchScanExecTransformerBase -- including\n the fallback-tagged one produced by PreOffload. PreOffload always marks the\n Iceberg/BatchScan wrapper as fallback so the JVM evaluates InputFileName() via\n the thread-local; PostOffload must not re-push a synthesized column into the\n scan\u0027s output in that case.\n\nFix:\n Add \u0027\u0026\u0026 !FallbackTags.nonEmpty(p)\u0027 guard to both BatchScanExecTransformerBase\n PostOffload cases (the direct-child case and the two-level collapse case).\n When the inner ProjectExec is already fallback-tagged, PostOffload leaves it\n untouched and the JVM evaluates the metadata function correctly.\n\nTests added:\n - \u0027case-sensitive mode: lowercase input_file_name as data column -- platform compatibility\u0027\n (mirrors the IcebergSuite CI failing test for the Parquet / FileSourceScan code path)\n - \u0027case-sensitive mode: lowercase input_file_name as data column -- caseSensitive\u003dfalse\u0027\n - \u0027exact lowercase input_file_name: only user column selected, no function call\u0027\n - writeExactCollisionTable helper for exact-lowercase schema\n\nAll 22 InputFileNameCollisionSuite tests pass.\nAll 49 FallbackSuite tests pass.\n\nAlso rebuilds gluten-substrait with genFormatNumberTransformer now present in the\nSparkPlanExecApi trait (was missing from the installed JAR), resolving the\nScalarFunctionsValidateSuite \u0027format_number\u0027 CI failure.\n\nfix: PostOffload BatchScan - reuse PreOffload mangled aliases to avoid Regular/Synthesized Velox conflict\n\nRoot cause:\n PushDownInputFileExpression.PostOffload, when processing a ProjectExec wrapping a\n BatchScanExecTransformerBase (Iceberg/BatchScan), re-ran rewriteExpr from scratch,\n which always creates an injected alias with the un-mangled prettyName (e.g.\n \u0027input_file_name\u0027) regardless of whether PreOffload had already mangled it.\n Under caseSensitive\u003dtrue, when the physical table has a column named exactly\n \u0027input_file_name\u0027, Velox receives both:\n - a Regular column \u0027input_file_name\u0027 (physical data)\n - a Synthesized column \u0027input_file_name\u0027 (injected metadata)\n and throws: \u0027Inconsistent column handle type: input_file_name, expected Regular,\n got Synthesized\u0027 (velox/connectors/hive/HiveConnectorUtil.cpp:354)\n\nFix:\n PostOffload now detects pre-existing injected aliases in the projectList\n (those carrying INPUT_FILE_COL_METADATA, placed there by PreOffload).\n For each such alias Alias(InputFileName(), schemaName, meta)(exprId\u003dX):\n 1. Creates a fresh AttributeReference(schemaName, ..., meta)(exprId\u003dZ) for the\n scan output, preserving the (possibly mangled) schemaName.\n 2. Rewrites the alias to Alias(AttrRef(Z), schemaName, meta)(exprId\u003dX),\n keeping the original exprId\u003dX so outer ProjectExec attr references remain valid.\n 3. Pushes AttrRef(Z) into the scan output.\n This exactly mirrors what the original rewriteExpr path does for the non-collision\n case, but uses the PreOffload-chosen (possibly mangled) schema name instead of the\n un-mangled prettyName.\n\nFor the non-collision case (no physical column named \u0027input_file_name\u0027), the scan\noutput gets a Synthesized attr named \u0027input_file_name\u0027 -- same as before.\nFor the collision case (physical \u0027input_file_name\u0027 column exists), the scan output\ngets a Synthesized attr named \u0027__gluten_input_file_col__input_file_name__\u0027 -- the\nmangled name from PreOffload -- which does not conflict with the Regular column.\n\nTests:\n - Adds \u0027case-sensitive mode: lowercase input_file_name as data column -- platform\n compatibility\u0027 (mirrors the IcebergSuite CI failing test for Parquet)\n - Adds caseSensitive\u003dfalse variant\n - Adds plain data-only select variant\n - All 133 tests in InputFileNameCollisionSuite + FallbackSuite + ScalarFunctionsValidateSuite pass\n - Resolves merge conflict in FallbackSuite.scala (stash remnant)\n\n* Delete backends-velox/src/test/scala/org/apache/gluten/execution/InputFileNameCollisionSuite.scala\n\n* changes to casesensitive bug\n\n* changes to casesensitive bug\n\n* changes to casesensitive bug\n\n* Address review: drop name mangling and backend API changes, prune tests\n\n- Remove injected-file column name mangling, canonical-name metadata and\n the injectedFileColAliases plumbing through IteratorApi/FileMetadataUtil.\n- Fall back via a single fallback tag when the scan output conflicts with\n the injected input_file attribute under normalized names.\n- Prune FallbackSuite/IcebergSuite to the Input_File_Name vs\n input_file_name() regression tests.\n\n* Do not fill a real input_file_* data column with file metadata\n\nA table column named input_file_name (or input_file_block_*) was matched\nby name in BasicScanExecTransformer and requested as split metadata, so\nnative scans returned the file path instead of the stored value.\n\n* Only skip input_file_* metadata for columns the scan actually reads\n\nChecking the full table schema also suppressed the injected attribute when\nthe table had an unselected column named input_file_name, so\ninput_file_name() returned that column\u0027s data instead of the file path.\n\n* Address review: route input_file_name collision fallback through doValidateInternal\n\nInstead of calling FallbackTags.add() directly on the vanilla scan in\nPushDownInputFileExpression.PreOffload, place a lightweight TreeNodeTag\n(INPUT_FILE_NAME_COLLISION_TAG) on the scan node when an injected\ninput-file attribute would share a normalized name with a user data\ncolumn.\n\nScanTransformerFactory.createFileSourceScanTransformer now calls\ncopyTagsFrom so the tag propagates to the temporary transformer created\nby FallbackByNativeValidation during the AddFallbackTags pass.\n\nBasicScanExecTransformer.doValidateInternal checks for the tag and\nreturns ValidationResult.failed(...), which causes Gluten\u0027s standard\nvalidation framework to add FallbackTags to the vanilla scan and skip\noffloading it -- keeping the scan on vanilla Spark where\nInputFileName() is populated correctly via the JVM thread-local.\n\nThis satisfies the reviewer request to handle the backend-schema\nlimitation inside the scan\u0027s validation rather than inside\nPushDownInputFileExpression.\n\nAlso resolve docs/velox-backend-limitations.md conflict by adopting the\nupstream apache:main wording for the case-sensitive section.\n\n* prune FallbackSuite: keep only the two focused input_file_name regression tests\n\nRemove the redundant tests:\n- \u0027input_file_name() returns the path when the table has a column of that name\u0027\n (sub-cases overlap with the primary collision test and the no-injection test)\n- \u0027input_file_name() without a conflicting column stays native\u0027\n (already covered implicitly by the caseSensitive\u003dtrue branch of the main test)\n\nThe two remaining tests cover the exact regression scenarios addressed by\nthis PR:\n1. \u0027input_file_name() with a user column named Input_File_Name\u0027: verifies\n caseSensitive\u003dtrue → scan stays native, caseSensitive\u003dfalse → scan falls\n back, and in both cases user data and file-path values are correct.\n2. \u0027user column named like input_file_name is not replaced by the file path\u0027:\n verifies that selecting only the user column (no function call) always\n stays native and returns user data under both case-sensitivity settings.\n\n* Propagate scan tags when creating batch scan transformers\n\nPushDownInputFileExpression tags a scan with INPUT_FILE_NAME_COLLISION_TAG\nwhen an injected input_file_name attribute collides with a user column, so\nthat doValidateInternal can fail validation and keep the scan on vanilla\nSpark. createBatchScanTransformer and IcebergScanTransformer.apply did not\ncopy tags from the BatchScanExec, so the tag was lost and Velox failed with\n\u0027Cannot map from same table column to different outputs in table scan\u0027.\n\n* Revert \"Propagate scan tags when creating batch scan transformers\"\n\nThis reverts commit e1d3babfe7dcad2585aa4093e86b0cc103c384d2.\n\n* Remove failing iceberg input_file_name regression test\n\n* Restore copyTagsFrom in IcebergScanTransformer and BatchScanTransformer\n\nPushDownInputFileExpression.PreOffload tags the source BatchScanExec with\nINPUT_FILE_NAME_COLLISION_TAG when an injected input_file_name attribute\ncollides with a user data column. doValidateInternal reads this tag to fail\nvalidation cleanly and keep the scan on vanilla Spark.\n\nThe tag must be copied when the BatchScanExec is converted to a transformer:\n- IcebergScanTransformer.apply: copyTagsFrom was correctly added in\n e1d3babfe but accidentally dropped by the revert in 1c5428f7a.\n- ScanTransformerFactory.createBatchScanTransformer: same gap for all\n non-Iceberg DSv2 sources (Delta, Lance, etc.).\n- ScanTransformerFactory.createFileSourceScanTransformer already had\n copyTagsFrom from the original fix; no change needed there.\n\nWithout this, the collision tag is silently lost and Velox receives a\nscan with two outputs mapping to the same normalized column name, causing\na hard native error instead of a clean fallback.\n\n* changes to casesensitive bug\n\n---------\n\nCo-authored-by: root \u003croot@sai-gluten.dev.fyre.ibm.com\u003e" }, { "commit": "0c322e85884d2ed5d65e10d64f2f87759e997ed3", "tree": "24a6d649c16b168e449195cb58c977054197caed", "parents": [ "c5bd3de4abf78942c3691deb97f79962e81072ed" ], "author": { "name": "Zhang Xiaofeng", "email": "xiaofeng.zhang@bytedance.com", "time": "Tue Oct 27 20:11:05 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 27 12:11:05 2026 +0000" }, "message": "[MINOR][VL] Fix integer overflow in shuffle buffer offsets and shrink size (#13448)" }, { "commit": "c5bd3de4abf78942c3691deb97f79962e81072ed", "tree": "92707279d36983f11f186277e2d800473204683b", "parents": [ "4e5d44684e1856f029026e99761abaddc5169f77" ], "author": { "name": "maoyuwei1", "email": "54635753+maoyuwei1@users.noreply.github.com", "time": "Tue Oct 27 17:13:26 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 27 17:13:26 2026 +0800" }, "message": "[GLUTEN-13276][VL] Fix CelebornShuffleManager init fails with NoSuchMethodException: FailedShuffleCleaner.\u003cinit\u003e() (#13283)" }, { "commit": "4e5d44684e1856f029026e99761abaddc5169f77", "tree": "e54a14d382b780ed5ca55cff6f63fda7e5eb427f", "parents": [ "bed35273297cf464b8466dcbc95335de054c2474" ], "author": { "name": "BInwei Yang", "email": "felixybw@apache.org", "time": "Mon Oct 26 19:40:13 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 26 19:40:13 2026 -0700" }, "message": "[VL][CI] Reuse the x86 static build for Iceberg and pause enhanced on PRs (#13421)\n\n* [VL][CI] Reuse the x86 static native build for Kafka and Iceberg, pause enhanced on PRs\n\n- velox_backend_enhanced.yml: run on workflow_dispatch only.\n- velox_kafka.yml: removed. Its Spark 3.5 job moves to velox_backend_x86.yml on\n top of build-native-lib-centos-8. Its native Kafka tests already run in\n cpp-test-udf-test, whose shared build enables Kafka and runs ctest.\n- iceberg_spark_ut.yml: called from velox_backend_x86.yml with the static native\n artifact for the head revision; still runnable via workflow_dispatch.\n baseline.py accepts results recorded by either workflow.\n\nCo-Authored-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e\n\n* [MINOR][VL] Share the Spark 3.5 CI change condition\n\nGenerated-by: Codex GPT-6\n\n* [VL][CI] Keep Kafka and Iceberg CI independent\n\nRetain independent Kafka and Spark 3.5 Iceberg workflows and their native builds. Limit this PR to manual enhanced CI and document its availability for urgent patches.\n\nGenerated-by: Codex GPT-6\n\n* [VL][CI] Restore Iceberg static native build reuse\n\nRun Iceberg alongside Spark and TPC jobs after the shared x86 static native build. Keep Kafka independent, restore baseline provenance support, and include Iceberg-only harness changes in detection.\n\nGenerated-by: Codex GPT-6\n\n---------\n\nCo-authored-by: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e" }, { "commit": "bed35273297cf464b8466dcbc95335de054c2474", "tree": "1ea30fc8785fd9a770db563d34e840ccbc69feaa", "parents": [ "b926ec692752e306225a21c2490fed949f47dbf7" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Mon Oct 26 19:29:27 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 26 19:29:27 2026 -0700" }, "message": "Bump gitpython in /tools/workload/benchmark_velox/analysis (#13464)\n\nBumps [gitpython](https://github.com/gitpython-developers/GitPython) from 3.1.62 to 3.2.1.\n- [Release notes](https://github.com/gitpython-developers/GitPython/releases)\n- [Changelog](https://github.com/gitpython-developers/GitPython/blob/main/doc/source/changes.rst)\n- [Commits](https://github.com/gitpython-developers/GitPython/compare/3.1.62...3.2.1)\n\n---\nupdated-dependencies:\n- dependency-name: gitpython\n dependency-version: 3.2.1\n dependency-type: direct:production\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "b926ec692752e306225a21c2490fed949f47dbf7", "tree": "1dca62715580a0d8cc4ee5acf3ef8991b792e987", "parents": [ "3baecdaabd49943bdf49e627faf10aee9065bf58" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Mon Oct 26 19:29:10 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 26 19:29:10 2026 -0700" }, "message": "Bump black in /tools/workload/benchmark_velox/analysis (#13465)\n\nBumps [black](https://github.com/psf/black) from 26.3.1 to 26.10.1.\n- [Release notes](https://github.com/psf/black/releases)\n- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)\n- [Commits](https://github.com/psf/black/compare/26.3.1...26.10.1)\n\n---\nupdated-dependencies:\n- dependency-name: black\n dependency-version: 26.10.1\n dependency-type: direct:production\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "3baecdaabd49943bdf49e627faf10aee9065bf58", "tree": "38535500f334428d3982538318e2cc4eae1e106b", "parents": [ "1e896874b0ef4ddf2f8ceecd3c771d0970209836" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Mon Oct 26 19:28:46 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 26 19:28:46 2026 -0700" }, "message": "Bump nbconvert in /tools/workload/benchmark_velox/analysis (#13466)\n\nBumps [nbconvert](https://github.com/jupyter/nbconvert) from 7.17.1 to 7.17.2.\n- [Release notes](https://github.com/jupyter/nbconvert/releases)\n- [Changelog](https://github.com/jupyter/nbconvert/blob/main/CHANGELOG.md)\n- [Commits](https://github.com/jupyter/nbconvert/compare/v7.17.1...v7.17.2)\n\n---\nupdated-dependencies:\n- dependency-name: nbconvert\n dependency-version: 7.17.2\n dependency-type: direct:production\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "1e896874b0ef4ddf2f8ceecd3c771d0970209836", "tree": "006118d09aab3439454df8f43813893de2be7fa8", "parents": [ "e6ea497d210ec0e5a812b72cb10e77ce3a2efe7a" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Mon Oct 26 15:38:47 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 26 15:38:47 2026 -0700" }, "message": "[GLUTEN-13179][VL] Combine pushed-down filters into a balanced And tree and re-enable GlutenWholeStageCodegenSuite for Spark 4.2 (#13435)" }, { "commit": "e6ea497d210ec0e5a812b72cb10e77ce3a2efe7a", "tree": "0e06f3d4056067656560aba9d933c96a7e1bf0f5", "parents": [ "1fdabb14a00c74df016cd16aca3b29b60c13f457" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Mon Oct 26 15:14:59 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 26 15:14:59 2026 -0700" }, "message": "[GLUTEN-13179][VL] Fix Spark 4.2 function-registry and error-wrapping test failures (#13442)" }, { "commit": "1fdabb14a00c74df016cd16aca3b29b60c13f457", "tree": "31f1359b5ec686e806aa608583d98090c7fdb115", "parents": [ "6639cbcdd33f540956f6c227f8c949bcdfd7654e" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Mon Oct 26 15:06:55 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 26 15:06:55 2026 -0700" }, "message": "[GLUTEN-13180][VL] Re-enable the Spark 4.2 bucketed scan AE and Hive ADD JAR tests (#13432)" }, { "commit": "6639cbcdd33f540956f6c227f8c949bcdfd7654e", "tree": "e6e1daeca46bf52ddba2b8ff5b24fcd20f557eb5", "parents": [ "531f76a3193ead8119f2167961dd0c3438574e73" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Mon Oct 26 14:54:41 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 26 14:54:41 2026 -0700" }, "message": "[GLUTEN-13178][VL] Re-enable the Spark 4.2 broadcast, hash aggregate, acosh and XML TIME array tests (#13431)" }, { "commit": "531f76a3193ead8119f2167961dd0c3438574e73", "tree": "28b84790e7283e004eb83fbe08c3c39671d29f18", "parents": [ "3ade2f2eb9d89d20cd2fb7fcf0c46cc8aad49d0a" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Mon Oct 26 14:49:02 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 26 14:49:02 2026 -0700" }, "message": "[GLUTEN-13177][VL] Refresh the remaining Spark 4.2 SQLQueryTest golden files (#13444)" }, { "commit": "3ade2f2eb9d89d20cd2fb7fcf0c46cc8aad49d0a", "tree": "7725fb1920e7b7b8057ff1d92af47bfa388205cb", "parents": [ "258bf8ec8b164f64dc80b986d7d76b2a1daa4c18" ], "author": { "name": "Gluten Performance Bot", "email": "137994563+GlutenPerfBot@users.noreply.github.com", "time": "Mon Oct 26 20:54:43 2026 +0000" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 26 20:54:43 2026 +0000" }, "message": "[GLUTEN-6887][VL] Daily Update Velox Version (2026_10_26) (#13453)\n\n* [GLUTEN-6887][VL] Daily Update Velox Version (dft-2026_10_26)\n\nUpstream Velox\u0027s New Commits:\nf8b036346 by tanjialiang, fix(nimble): Embed resume keys for never-reached requests (#19798)\n0293112b3 by Maria Basmanova, docs: Add ROW and test-writing rules to CLAUDE.md\n1a2c8c860 by oerling, feat(torchwave): Run merge-and-dedup in TorchWave kernels (#19781)\na19b05709 by zhichenxu-meta, feat(rpc): Retry capacity rejections through the shared rate limiter (#19786)\nda92167f3 by oerling, feat(torchwave): Run and fuse the ads trp:: ops (#19774)\n0db5599b0 by tanjialiang, fix(nimble): Set resume keys for requests past a stripe gap (#19797)\n73b948699 by macduan, feat(paimon): Introduces PaimonTableHandle with schema and field IDs\n413c06969 by Richard Barnes, build(tpcds): Build dsdgen with -Werror\ne0810919f by Sahil Gandhi, fix(exec): Start no sub-task once a fixed point\u0027s task ends (#19763)\nef573a1bc by Xiaoxuan, feat(nimble): Let cluster index factories set the key order (#19780)\n1b0f2fb18 by Richard Barnes, refactor: Replace folly::sformat and folly::format with fmt::format\nf2f7cd6f6 by Wei He, fix: Correct empty row and NaN hashes in HiveHash (#19673)\n39d788e63 by Sahil Gandhi, fix(exec): Abandon an unstarted fixed point when its task ends (#19762)\naa4dc59c8 by tanjialiang, feat(nimble): Reject files that need unsupported features (#19599)\n90ac7a1d6 by macduan, refactor(hive): Encapsulate mutable file scan state\n479ee06da by Richard Barnes, build: Include fmt/format.h in CompareFlags.h\n44872691f by XiaoDu, feat(nimble): Deduplicate stripe-group metadata arrays (#19759)\n79708dd0f by Richard Barnes, refactor(dwrf): Remove unreachable break after DWIO_RAISE in StatisticsBuilder\n118936c08 by Richard Barnes, refactor(tpcds): Remove unreachable break after exit() in decimal.cpp\n17202cf62 by Richard Barnes, refactor(exec): Remove unreachable break after FAIL() in SpillerTest\n32443e0f4 by Richard Barnes, refactor(tpcds): Remove unreachable break after exit() in scaling.cpp\n40955e7b0 by RindsSchei225e, refactor(nimble): Flatten velox/ into sibling directories (#18747)\nc3599418d by Wei He, refactor: Extract Hive hash function for reuse. (#19648)\n829147045 by Sahil Gandhi, fix(exec): Let fixed-point state outlive its sub-tasks\nc70feb2cf by GALI PREM SAGAR, feat(cudf): Run Presto\u0027s array_join on the GPU\n2b4e9a4e2 by GALI PREM SAGAR, perf(cudf): Take file sizes from the split instead of S3 HEAD calls\nab7f7f70e by Naveen Mahadevuni, feat(iceberg): Support reading geography type\nc1de6e507 by Christian Zentgraf, feat(s3): Add s3.use-fips with AWS_USE_FIPS_ENDPOINT fallback\n843a002c6 by Deepak Majeti, fix(build): Remove ranges-v3 dependency and fix image build\nea9fd20d2 by shanhao-203, perf(nimble): Index hybrid flatmap group lookups (#19764)\nd22ef33ed by tanjialiang, fix(nimble): Fix use-after-free in parallel metadata reads" }, { "commit": "258bf8ec8b164f64dc80b986d7d76b2a1daa4c18", "tree": "eddb7e7b19e8e8998f4926ef11bf40f0fc91cb22", "parents": [ "ffb8af3fedfc5c9c74244c83a4c33adb87caccd8" ], "author": { "name": "Weixin-Xu", "email": "xuweixin.rex@bytedance.com", "time": "Mon Oct 26 11:15:01 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 26 11:15:01 2026 +0800" }, "message": "[MINOR][INFRA] Fix Bolt labeling and issue templates (#13403)" }, { "commit": "ffb8af3fedfc5c9c74244c83a4c33adb87caccd8", "tree": "2fc3b0bc3323b4a5f691678c3700fe24eb0b7b1e", "parents": [ "cc74a6164ee4e38b29ca558039c30c047de67303" ], "author": { "name": "leoojiang", "email": "jianglie2023@gmail.com", "time": "Sat Oct 24 04:48:28 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Oct 24 04:48:28 2026 -0700" }, "message": "[MINOR][DOC] Fix Bolt quick-start README link (#13408)\n\nPoint the quick-start guide to the Bolt backend section in the repository README. Verified the Bolt docs scan and format preflight.\n\nCo-authored-by: Ubuntu \u003cazureuser@Main.n1l05aasmpie5onxhehb5y5gra.lx.internal.cloudapp.net\u003e" }, { "commit": "cc74a6164ee4e38b29ca558039c30c047de67303", "tree": "a836e1739525af29582ef82ebb52a56e11f25643", "parents": [ "e4ccf1a055fb8cd562b5613ffa3c7f70b684fa8e" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Sat Oct 24 01:20:57 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Oct 24 16:20:57 2026 +0800" }, "message": "[GLUTEN-13177][VL] Refresh the Spark 4.2 SQLQueryTest golden files: double-quoted-identifiers, nonansi/double-quoted-identifiers (#13292)\n\nRe-enable 2 GlutenSQLQueryTestSuite test(s) disabled by #13163:\ndouble-quoted-identifiers, nonansi/double-quoted-identifiers.\nTheir inputs and results are replaced by the Spark 4.2.0 versions.\n\nCo-authored-by: Akshay Tayal \u003cakshaytayal@users.noreply.github.com\u003e\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9" }, { "commit": "e4ccf1a055fb8cd562b5613ffa3c7f70b684fa8e", "tree": "8e94e03a6048d80142a9ee084185cfad1661b56c", "parents": [ "205ca389bb2845413dbc5187a5ba9bb296e6ba26" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Sat Oct 24 01:20:39 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Oct 24 16:20:39 2026 +0800" }, "message": "[GLUTEN-13177][VL] Refresh the Spark 4.2 SQLQueryTest golden files: cte-legacy, cte-nested, cte-nonlegacy (#13291)\n\nRe-enable 3 GlutenSQLQueryTestSuite test(s) disabled by #13163:\ncte-legacy, cte-nested, cte-nonlegacy.\nTheir inputs and results are replaced by the Spark 4.2.0 versions.\n\nCo-authored-by: Akshay Tayal \u003cakshaytayal@users.noreply.github.com\u003e\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9" }, { "commit": "205ca389bb2845413dbc5187a5ba9bb296e6ba26", "tree": "19940c65e10b742302fd397476a128901d1309d6", "parents": [ "923abe05486f5ebcb7ee064c059e01791ec43214" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Sat Oct 24 01:20:16 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Oct 24 16:20:16 2026 +0800" }, "message": "[GLUTEN-13175][VL] Add a Gluten version of the Spark 4.2 UPDATE NOT NULL test in GlutenDeltaBasedUpdateTableSuite (#13315)\n\nKeep \"update with constraint violation and transactional checks\" excluded in GlutenDeltaBasedUpdateTableSuite\n(Velox assert_not_null throws a VeloxUserError instead of SparkRuntimeException) and add\na Gluten version in GlutenUpdateTableSuiteBase that keeps the transaction checks.\n\nCo-authored-by: Akshay Tayal \u003cakshaytayal@users.noreply.github.com\u003e\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9" }, { "commit": "923abe05486f5ebcb7ee064c059e01791ec43214", "tree": "40fcdc1e00fad801f2add13300f558fa0006f543", "parents": [ "ee4289aafcf663fc8f890fa78c18ffa73841da60" ], "author": { "name": "BInwei Yang", "email": "felixybw@apache.org", "time": "Sat Oct 24 00:47:09 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Oct 24 08:47:09 2026 +0100" }, "message": "[MINOR][DOC] Document the pre-built apache/gluten Docker images (#13425)" }, { "commit": "ee4289aafcf663fc8f890fa78c18ffa73841da60", "tree": "e1150f695702b662d8497fcefadab8a654997246", "parents": [ "a3c5602dc9de58858069588d00a7468e79cec8c1" ], "author": { "name": "BInwei Yang", "email": "felixybw@apache.org", "time": "Sat Oct 24 00:45:17 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Oct 24 00:45:17 2026 -0700" }, "message": "[VL][CI] Stop building arm64 images that no arm64 job uses (#13428)\n\nNo arm64 workflow runs the centos-8-jdk* images or centos-9-jdk11/jdk17, so\nbuild them for amd64 only. arm64 stays for centos-9-jdk8 (ARM cpp-test-udf-test\nand the default dev container), vcpkg-centos-9 (ARM static build and the static\ndev container) and vcpkg-almalinux-8-gcc13 (nightly arm64 build).\n\nCo-authored-by: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e" }, { "commit": "a3c5602dc9de58858069588d00a7468e79cec8c1", "tree": "659fba8f6814e8f30759c57224df4d0e36ac730e", "parents": [ "3a2f96987f13ceab67ae8cd7eb5e71803c10d6e5" ], "author": { "name": "BInwei Yang", "email": "felixybw@apache.org", "time": "Fri Oct 23 19:31:02 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 23 19:31:02 2026 -0700" }, "message": "[VL][CI] Add a manual workflow to delete unused Docker Hub tags (#13426)\n\nDocker Hub has 36 apache/gluten tags, many of them no longer built or used.\nOnly the Docker Hub token in the repository secrets can delete them, so add a\nworkflow_dispatch workflow that does. It dry-runs by default, and refuses\nrelease tags (\u003cmajor\u003e.\u003cminor\u003e-*), tags docker_image.yml still publishes, and\ntags referenced as apache/gluten:\u003ctag\u003e on main or any branch-* release branch.\n\nCo-authored-by: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e" }, { "commit": "3a2f96987f13ceab67ae8cd7eb5e71803c10d6e5", "tree": "e827ef72d2d18a66732488a533e2a22cf0fc0342", "parents": [ "e939fd402d722e8372e8ce2cf05bb93050a8c27c" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Fri Oct 23 10:08:43 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 23 10:08:43 2026 -0700" }, "message": "[GLUTEN-13175][VL] Add Spark 4.2 UPDATE NOT NULL test (#13314)" }, { "commit": "e939fd402d722e8372e8ce2cf05bb93050a8c27c", "tree": "38461a9d43b7ba249dfffdd5bd562be844878579", "parents": [ "c6852ecd3f8dcaea57752559a5645f593fdf4a1e" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Fri Oct 23 10:07:11 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 23 10:07:11 2026 -0700" }, "message": "[GLUTEN-13177][VL] Refresh the Spark 4.2 SQLQueryTest golden files: keywords, nonansi/keywords (#13295)" }, { "commit": "c6852ecd3f8dcaea57752559a5645f593fdf4a1e", "tree": "93a6e5137a520db076fb59ff90fa649ba4211c35", "parents": [ "7b193da3fd462db56c0b66a11762d748d145269f" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Fri Oct 23 10:04:25 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 23 10:04:25 2026 -0700" }, "message": "[GLUTEN-13177][VL] Refresh the Spark 4.2 SQLQueryTest golden files: identifier-clause, identifier-clause-legacy (#13294)" }, { "commit": "7b193da3fd462db56c0b66a11762d748d145269f", "tree": "c08d472cd157f3833a8931b08706e215b708ecb7", "parents": [ "fb588bc4c965db7895e3fc2c781a62a9fb6cbd01" ], "author": { "name": "Gluten Performance Bot", "email": "137994563+GlutenPerfBot@users.noreply.github.com", "time": "Fri Oct 23 17:57:08 2026 +0100" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 23 17:57:08 2026 +0100" }, "message": "[GLUTEN-6887][VL] Daily Update Velox Version (2026_10_23) (#13418)\n\n* [GLUTEN-6887][VL] Daily Update Velox Version (dft-2026_10_23)\n\nUpstream Velox\u0027s New Commits:\n72ad30f8f by Maria Basmanova, fix(exec): Support broadcast in FixedPoint body plans (#19753)\n1afcf9079 by Pedro Pedreira, refactor(functions): Dispatch MakeRowFromMap on the map encoding (#19674)\n09bced8ca by GALI PREM SAGAR, feat(cudf): Run from_unixtime and to_unixtime on the GPU\n1831f3590 by Ke, fix(nimble): Free previous chunk before decoding the next\ne1cefcc92 by Ke, perf(nimble): Skip value decoding for null filters (#19669)\n8d6a4ad19 by Hongze Zhang, fix!: CountDistinctAggregate should override initializeNewGroupsInternal\ncbe4bec92 by inf / Hazmi, feat(iceberg): Add split offsets to commit metadata\n58d01dc99 by Mariam AlMesfer, feat(spark): Add support for unix_timestamp/to_unix_timestamp(timestamp_utc)\nf40aa0c9a by Wechar Yu, fix(spill): Delete spill inputs after successful pre-merge\nf0e1a32fd by Apurva Kumar, fix(iceberg): Omit unknown null counts from DWRF data file metrics (#19738)\n0068a37cb by GALI PREM SAGAR, fix(fuzzer): Keep nulls out of the k-RANGE bound sent to Presto\n07c29ee4e by oerling, feat(torchwave): Faster scatter_add and compiler fixes (#19679)\nb8982c7f6 by dependabot[bot], build(deps): Bump launch-editor from 2.14.1 to 2.14.2 in /website\n7ff007df4 by Misiu Godfrey, fix(cudf): Preserve 64-bit offsets when restoring VARBINARY format\n7fbd57fde by Maria Basmanova, fix(exec)!: Fail when a convergence plan produces no row\nd499fb3df by Apurva Kumar, fix(iceberg): Commit a deletion vector under its data file\u0027s partition spec\n54c5d740b by tanjialiang, refactor(nimble): Let CachedDataInput run on a cache other than AsyncDataCache (#19663)\ncca087339 by Dmitry Spikhalsky, fix(exec): Clear FixedPointLoop next-row link with an unaligned store\n3f397aff6 by Dmitry Spikhalsky, fix(dwio): Use unaligned loads in DecoderUtil bit helpers\n22ac4a306 by Dmitry Spikhalsky, fix(functions): Use unaligned loads in Murmur3 32-bit string hashes\n4c900b305 by Natasha Sehgal, test(fuzzer): Update padding skip issue references (#19647)\n241ed544d by mohsaka, feat(type): Parse Hive \"void\" type as Velox UNKNOWN\nd645ae590 by Xin Zhang, fix(docs): Fix broken TLSF paper link\n34403efaa by Krishna Pai, build(ci): Make stash saves and restores best effort\nb4957001e by Jimmy Lu, fix(exec): Default-initialize NestedFixedPoint members\nb2a92df83 by Wechar Yu, fix: Spill writer should respect writeBufferSize for large batches\n3fcdb2b33 by Anshul Baliga, docs(sparksql): Document approx_percentile\n968ec27fa by Deepak Majeti, fix(build): Prompt Boost build and fix wget_opts on MacOS\nf9327757a by peterenescu, feat(fuzzer): Add support for floating point edge cases (#19649)\n552ec51e2 by Zhenyuan Zhao, fix(nimble): Keep file read options on small-file preload\n8501e6f4a by Pedro Pedreira, feat(benchmark): Benchmark make_row_from_map over flat maps\nc57fd8592 by Wei He, fix(row): Pad fixed-width UnsafeRow arrays (#19668)\n13d9d32e3 by Apurva Kumar, fix(iceberg): Count an UPDATE once in the MERGE row count\n51d70eda8 by fzxu, feat: Add flat map support for map_subset (#18629)\n399f9d753 by Shruti Shivakumar, feat(arrow): Import 32-bit decimal arrays\ne257582f4 by Pedrum Jalali, feat(spark): Implement native transform_keys\n11c27c404 by dependabot[bot], build(deps): Bump follow-redirects from 1.16.0 to 1.16.1 in /website\n6eeb70ac7 by shanhao-203, docs(nimble): Add Hybrid FlatMap blog post (#19532)\n447e432c9 by Maria Basmanova, feat(exec)!: Simplify fixed point worker addressing (#19688)\n4a4679f80 by Maria Basmanova, test(exec): Fix data race in fixed point test cursors\nc52ff97b4 by Ke, perf(nimble): Add bulkScan to ConstantEncoding (#19581)\n3c9fd367a by Apurva Kumar, fix(iceberg): Report NIMBLE data files as NIMBLE in Iceberg commit messages\n42b5ef84a by Apurva Kumar, fix(iceberg): Keep partition data on MERGE deletion vectors\n0cba44b3c by Matt Karrmann, fix(exec): Avoid division by zero in AdaptivePrefetch (#19614)\nd0f64a95e by Xiaoxuan, feat(nimble): Add synchronous row access and point seeks to index iterators\n959199f65 by Ke, perf(nimble): Bulk-decode hooks and filter-only reads in FBW, BBP, SubIntSplit (#19572)\n0235167b2 by Maria Basmanova, feat(exec)!: Decide fixed point convergence on worker 0 (#19622)\naa55e9e6e by PRASHANT GOLASH, feat(nimble): Simplify ALP-RD size estimation and selection (#19497)\n7fdce0438 by Maria Basmanova, fix(core)!: Check the plans inside a fixed point (#19676)\n\nSigned-off-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e\n\n* optimize CI uage\n\nSigned-off-by: Yuan \u003cyuanzhou@apache.org\u003e\n\n---------\n\nSigned-off-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e\nSigned-off-by: Yuan \u003cyuanzhou@apache.org\u003e\nCo-authored-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e\nCo-authored-by: Yuan \u003cyuanzhou@apache.org\u003e" }, { "commit": "fb588bc4c965db7895e3fc2c781a62a9fb6cbd01", "tree": "975d1a5f0845c200f70dae36af19a88149b3c2c7", "parents": [ "9e52cbda69a83fec2a768385ede2bffaece30e72" ], "author": { "name": "Niels Pardon", "email": "par@zurich.ibm.com", "time": "Fri Oct 23 14:50:16 2026 +0200" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 23 13:50:16 2026 +0100" }, "message": "[GLUTEN-12597][CORE] Migrate Rel oneof to Substrait 0.98 (park GenerateRel/WindowGroupLimitRel) (#13210)" }, { "commit": "9e52cbda69a83fec2a768385ede2bffaece30e72", "tree": "13f7c6eed22b870f4f745872ed5fbf87170c1b8d", "parents": [ "094607af1e38680d26db06ce194beb459fb6bd8b" ], "author": { "name": "Reema", "email": "80041251+ReemaAlzaid@users.noreply.github.com", "time": "Fri Oct 23 11:37:15 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 23 09:37:15 2026 +0100" }, "message": "[VL] Fix host to GPU conversion in CudfVectorStream (#13067)" }, { "commit": "094607af1e38680d26db06ce194beb459fb6bd8b", "tree": "d9b3180faa2465cdd1159f09c4aa5d009531daf9", "parents": [ "4100c25b34e53d966cb5fae7f2944c3d1e5a1388" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Fri Oct 23 09:18:06 2026 +0100" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 23 09:18:06 2026 +0100" }, "message": "Bump pillow in /tools/workload/benchmark_velox/analysis (#13414)" }, { "commit": "4100c25b34e53d966cb5fae7f2944c3d1e5a1388", "tree": "5a91c899e0513886d2e783996238d6f8088299c2", "parents": [ "54b1439d2f39ccce2d512fb0e140a83b9d13479a" ], "author": { "name": "Yuan", "email": "yuanzhou@apache.org", "time": "Fri Oct 23 09:17:26 2026 +0100" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 23 09:17:26 2026 +0100" }, "message": "[VL] fix nightly build; adding spark-4.2 to nightly build (#13327)" }, { "commit": "54b1439d2f39ccce2d512fb0e140a83b9d13479a", "tree": "bac4ce921927ecfe87fbf0e2d1bd45eb60717d5a", "parents": [ "52b2099f7f300504f4ca168ed803592a8ff40a13" ], "author": { "name": "Mohammad Linjawi", "email": "linjawi.mohammad@gmail.com", "time": "Fri Oct 23 11:13:42 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 23 09:13:42 2026 +0100" }, "message": "[GLUTEN-12504][VL] Support dumping and replaying BHJ stages in the micro benchmark (#12592)" }, { "commit": "52b2099f7f300504f4ca168ed803592a8ff40a13", "tree": "018590a5ce72fd45ca134c5499e625d2e7a49eb0", "parents": [ "5ada13649a5248d76a0c1e397b991f3adc7ee568" ], "author": { "name": "inf / Hazmi", "email": "ialhazmim@gmail.com", "time": "Fri Oct 23 11:12:06 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 23 09:12:06 2026 +0100" }, "message": "[VL][Iceberg] Add global iceberg sorted write (#13199)" }, { "commit": "5ada13649a5248d76a0c1e397b991f3adc7ee568", "tree": "238c5ecf4063f37d27431b95c7684d25e38032bc", "parents": [ "73e7d569a594c266f7793c78ad50e624dcdb3391" ], "author": { "name": "Reema", "email": "80041251+ReemaAlzaid@users.noreply.github.com", "time": "Fri Oct 23 11:05:19 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 23 09:05:19 2026 +0100" }, "message": "[VL] Translate Kafka Substrait reads (#13236)" }, { "commit": "73e7d569a594c266f7793c78ad50e624dcdb3391", "tree": "e703b69c34a158123711830b473ba5a88f6a2f87", "parents": [ "1b6197dec94b2c16e101f0fa6dd4baa8efeb807b" ], "author": { "name": "Gluten Performance Bot", "email": "137994563+GlutenPerfBot@users.noreply.github.com", "time": "Thu Oct 22 22:52:27 2026 +0100" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 22 14:52:27 2026 -0700" }, "message": "[GLUTEN-6887][VL] Daily Update Velox Version (dft-2026_10_21) (#13401)\n\nUpstream Velox\u0027s New Commits:\n76bf7e9e5 by Maria Basmanova, fix(exec): Stop a fixed point\u0027s loop when its task ends (#19666)\nbe84a3153 by GALI PREM SAGAR, feat(presto): Add opt-in inf and nan spellings to CAST from VARCHAR\n911930187 by Ke, feat(nimble): Wire `nimbleScanFastPath` toggle to ChunkedDecoder (#19630)\n910348a12 by Maria Basmanova, test(exec): Run the output buffer serde tests with every serde (#19667)\n9e8bd22a9 by Maria Basmanova, fix(exec): Wait for a fixed point\u0027s peers before its first iteration (#19664)\n4ccc87a30 by GALI PREM SAGAR, feat(cudf): Support Hive partition keys in the cuDF Hive connector\n0d9c3cd00 by Pedro Pedreira, perf(functions): Group per-row element_at copies on flat maps\ne089fcc6d by macduan, fix(dwrf): Handle filter cache resets without an active stripe\nac96dd08c by Maria Basmanova, fix(exec): Serve reads of an output buffer that does not exist yet (#19662)\n9bf75aa86 by Anshul Baliga, fix(sparksql): Honor duplicate map key config in str_to_map\n99d6ba8b3 by Daniel Bauer, feat(ucx-exchange): Register the UCX transport and enable its build\n101286a45 by Maria Basmanova, fix(exec): Start a fixed point\u0027s loop after the task\u0027s drivers (#19659)\n62c048a34 by macduan, fix(paimon): Rebind runtime statistics after data source takeover\n9ffaceeeb by Maria Basmanova, docs: Add rules on one check per condition and one detail level\n581db8a2a by Xiaoxuan, perf(nimble): Add a cacheEncodingView option to encoding views\n\nSigned-off-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e\nCo-authored-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e" }, { "commit": "1b6197dec94b2c16e101f0fa6dd4baa8efeb807b", "tree": "ba2a1401cf597bd1f317fd63a6417f817ff9d7c9", "parents": [ "b7e1b6388c221a75c1256020ef09710acf3c3137" ], "author": { "name": "YangJie", "email": "yangjie01@baidu.com", "time": "Thu Oct 22 11:17:06 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 22 16:17:06 2026 +0100" }, "message": "[GLUTEN-13359][CORE] Keep alignment when shrinking an aligned block (#13360)\n\nCloses #13359" }, { "commit": "b7e1b6388c221a75c1256020ef09710acf3c3137", "tree": "e27a367ec4a87670b40011c9e56f2e96042b0bd3", "parents": [ "8a037eeb0d53ff9b796a046d67097ed763bbfa29" ], "author": { "name": "BInwei Yang", "email": "felixybw@apache.org", "time": "Thu Oct 22 00:08:39 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 22 00:08:39 2026 -0700" }, "message": "[GLUTEN-13140][VL] Replace ArrowWritableColumnVector with Spark\u0027s ArrowColumnVector and ArrowWriter (#13280)\n\n* [GLUTEN-13140][VL] Replace ArrowWritableColumnVector with Spark\u0027s ArrowColumnVector and ArrowWriter\n\nGluten\u0027s Java Arrow batches now consist of RefCountedArrowColumnVector, a\nsubclass of Spark\u0027s ArrowColumnVector that keeps Gluten\u0027s reference count.\nThe batch lifecycle is unchanged: ColumnarBatches still retains and releases\nbatches by reference count, and loads and offloads them in place.\n\n- Reading goes through Spark\u0027s ArrowColumnVector accessors, which check for\n nulls on all supported Spark versions, so the ArrowColumnarBatch /\n ArrowColumnarRow workaround for Spark\u0027s ColumnarRow before 4.0 is no\n longer needed.\n- Writing fills the Arrow vectors directly or with Spark\u0027s ArrowWriter:\n - ColumnarPartialProjectExec: Spark\u0027s MutableProjection + ArrowWriter\n (replaces ArrowProjection / GenerateArrowProjection /\n InterpretedArrowProjection).\n - ColumnarPartialGenerateExec: ArrowWriter over the generated rows.\n - ColumnarRangeExec, ExecUtil: set the Arrow vector directly.\n- ArrowColumnVectors: allocate a root from Gluten\u0027s allocator, wrap\n vectors, build a batch from rows with ArrowWriter, and adopt (move or\n copy) a vector into Gluten\u0027s allocator. ColumnarBatches.offload adopts\n batches of plain Spark ArrowColumnVectors.\n- ColumnarArrowEvalPythonExec wraps the Python output vectors with\n RefCountedArrowColumnVector.\n- VeloxColumnarWriteFilesExec reads the VARBINARY fragments column with\n getBinary (the old accessor allowed getUTF8String on binary).\n- Removed: ArrowWritableColumnVector, WritableColumnVectorShim (spark34 -\n spark42), ArrowColumnarBatch, ArrowColumnarRow, ArrowColumnVectorUtils,\n ArrowProjection and its code generators.\n\nArrowColumnVector, ArrowWriter.create(VectorSchemaRoot) and MutableProjection\nare available with the same API in Spark 3.4 - 4.2, so no shim is needed.\n\nTests: ColumnarBatchTest ported to ArrowWriter-built batches; new\nArrowColumnVectorTest cases (ref count, decimal round trip, offloading a\nSpark Arrow batch from another allocator root); new ArrowEvalPythonExecSuite\ncase with chained UDFs over many small batches with nulls.\n\nCo-Authored-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e\n\n* [GLUTEN-13140][VL] Add explicit return type to fix scalastyle in ArrowEvalPythonExecSuite\n\nCo-Authored-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e\n\n* [GLUTEN-13140][VL] Migrate LanceScanTransformer to ArrowColumnVectors\n\nCo-Authored-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e\n\n* [GLUTEN-13140][DOC] Update ArrowColumnarConversion.md for RefCountedArrowColumnVector\n\nCo-Authored-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e\n\n* [GLUTEN-13140][VL] Address review: safe adopt, fromRows cleanup, atomic ref count\n\n- ColumnarBatches.adopt copies instead of moving a vector when moving would\n empty it for someone else: a Gluten RefCountedArrowColumnVector passed\n through in a mixed batch, or a vector used by more than one column.\n- ArrowColumnVectors.fromRows closes the root if writing fails.\n- RefCountedArrowColumnVector updates its count with compare-and-set, never\n goes below 0, and rejects retain() after release.\n\nCo-Authored-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e\n\n---------\n\nCo-authored-by: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e" }, { "commit": "8a037eeb0d53ff9b796a046d67097ed763bbfa29", "tree": "87b2ce50ff9430717b763c5cc200d91e33dd03c5", "parents": [ "2d7179a874a7ecbf9fd9c3fd1b6603feb033e5da" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Wed Oct 21 10:17:02 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 21 10:17:02 2026 -0700" }, "message": "[GLUTEN-13175][VL] Share the Spark 4.2 MERGE shuffle-partition setting and re-enable the MERGE summary tests (#13317)" }, { "commit": "2d7179a874a7ecbf9fd9c3fd1b6603feb033e5da", "tree": "c44541149bef30b79586028790de6007086ea977", "parents": [ "08c302ca6b74f34220c75e2f3ebc3dff6096ad66" ], "author": { "name": "rdtr", "email": "norio@apache.org", "time": "Wed Oct 21 10:13:36 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 21 10:13:36 2026 -0700" }, "message": "[GLUTEN-11914][VL] Enable Parquet missing struct field tests for Spark 4.1 and 4.2 (#13398)" }, { "commit": "08c302ca6b74f34220c75e2f3ebc3dff6096ad66", "tree": "2a9e5b89f8b2f1e3d7fed050488f24075b288a7c", "parents": [ "39d80db5545113d5ca38c01eb7d3fceb9300d0fb" ], "author": { "name": "Mohammad Linjawi", "email": "linjawi.mohammad@gmail.com", "time": "Wed Oct 21 16:54:04 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 21 14:54:04 2026 +0100" }, "message": "[VL] Stop relocating flatbuffers in the Spark 3.x bundle (#13393)" }, { "commit": "39d80db5545113d5ca38c01eb7d3fceb9300d0fb", "tree": "ff2e827b9128e652b996fabe66ae6118adca21c3", "parents": [ "ff571bf1a075dec8357c0ced6922fe3b8a2a8726" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Wed Oct 21 00:14:00 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 21 15:14:00 2026 +0800" }, "message": "[GLUTEN-13174][VL] Add Gluten versions of the Spark 4.2 SPJ sorted-merge tests (#13246)\n\nSpark 4.2\u0027s GroupPartitionsExec can coalesce partitions with a k-way\nsorted merge (SortedMergeCoalescedRDD) that keeps the scan ordering, but\nonly when every node below it is SafeForKWayMerge, a short list of\nvanilla nodes. Gluten offloads the Filter/Project between the scan and\nGroupPartitionsExec, so the sorted merge is not used: GroupPartitionsExec\nruns columnar, concatenates the coalesced splits and reports only the\nordering that survives that. Gluten\u0027s EnsureLocalSortRequirements then\nadds the sort the sort-merge join needs, so the results stay correct.\n\nThe 4 Spark tests check the sorted-merge path and look for vanilla\nSortMergeJoinExec / ShuffledHashJoinExec nodes. Keep them excluded and\nadd Gluten versions that:\n- check the Gluten join transformers and SortExec / SortExecTransformer,\n- run the sorted-merge checks with filter and project offload turned\n off, so the subtree under GroupPartitionsExec stays vanilla, and\n- for the two preserve-ordering tests, also run with the Gluten defaults\n and check that a sort is added before the join.\n\nCo-authored-by: Akshay Tayal \u003cakshaytayal@users.noreply.github.com\u003e\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9" }, { "commit": "ff571bf1a075dec8357c0ced6922fe3b8a2a8726", "tree": "3b52a2cbe2a4a531b36121dec22bfcc504a324a0", "parents": [ "4876f4e551b21a2db4995c0d7f4023ca6481d88f" ], "author": { "name": "YangJie", "email": "yangjie01@baidu.com", "time": "Wed Oct 21 03:09:58 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 21 15:09:58 2026 +0800" }, "message": "[GLUTEN-13341][VL] Remove the unused veloxToArrow metric (#13344)" }, { "commit": "4876f4e551b21a2db4995c0d7f4023ca6481d88f", "tree": "894b7c148675de530b55fa94a1d3a6e11cd73ec4", "parents": [ "6fbdb0eb0891604ce4f8e6599584cd9009b71f1b" ], "author": { "name": "YangJie", "email": "yangjie01@baidu.com", "time": "Wed Oct 21 03:09:20 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 21 15:09:20 2026 +0800" }, "message": "[GLUTEN-13093][CORE] Fix two dynamic off-heap sizing memory bugs (#13094)" }, { "commit": "6fbdb0eb0891604ce4f8e6599584cd9009b71f1b", "tree": "3951fda4160851c521870b532cc6b49de14273da", "parents": [ "197c985ce0a1bd2ce9208d60da8c1bef8880599a" ], "author": { "name": "YangJie", "email": "yangjie01@baidu.com", "time": "Wed Oct 21 03:08:57 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 21 15:08:57 2026 +0800" }, "message": "[CORE] Run task resource release outside the global registry lock (#13166)" }, { "commit": "197c985ce0a1bd2ce9208d60da8c1bef8880599a", "tree": "162cb9102664086819ea21913447d615a87cb592", "parents": [ "dac0d1d9d498f598690c215fe9bd14e945edb173" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Wed Oct 21 00:04:55 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 21 15:04:55 2026 +0800" }, "message": "[GLUTEN-13175][VL] Re-enable the Spark 4.2 DELETE tests in GlutenGroupBasedDeleteFromTableSuite (#13244)\n\nRe-enable the 12 DELETE tests in GlutenGroupBasedDeleteFromTableSuite disabled by #13163. Spark\u0027s own tests use\n5 shuffle partitions; with Gluten\u0027s 1, AQE replaces the ReplaceDataExec input with an empty\nrelation when no rows are copied and numDeletedRows becomes -1 (vanilla Spark too).\n\nCo-authored-by: Akshay Tayal \u003cakshaytayal@users.noreply.github.com\u003e\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9" }, { "commit": "dac0d1d9d498f598690c215fe9bd14e945edb173", "tree": "a31c5cf7aed53291d6781e11dc5f3f51c68c3ec8", "parents": [ "c4b3cf7861ccadab10ecec17d0f346b7629111f2" ], "author": { "name": "BInwei Yang", "email": "felixybw@apache.org", "time": "Tue Oct 20 23:42:41 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 20 23:42:41 2026 -0700" }, "message": "[MINOR][DOC] Organize backend-specific docs under docs/backends (#13372)\n\n* [MINOR][DOC] Organize backend-specific docs under docs/backends\n\nMove Velox, Bolt and ClickHouse specific documents into per-backend\ndirectories so common docs and backend docs are clearly separated:\n\n docs/backends/{velox,bolt,clickhouse}/\n *.md configuration, function support, reference\n get-started/ build and integration guides\n developers/ backend-specific developer docs\n\nCommon docs stay in docs/, docs/get-started and docs/developers.\nRelative links, mkdocs nav, README/AGENTS/CONTRIBUTING, and the doc\ngenerators (AllVeloxConfiguration, AllBoltConfiguration,\ngen/diff-function-support-docs.py) are updated to the new paths.\n\n- Move MicroBenchmarks.md, PartialProject.md and QueryTrace.md to\n docs/backends/velox/developers.\n- Move ArrowColumnarConversion.md to docs/backends/velox/developers and fix\n its image paths.\n- Remove QAT docs (VeloxQAT.md, docs/qat/4x16.conf) and references.\n- Remove SLF4J/startup warnings output from Work-with-pyspark.ipynb.\n- Config docs: show only the emoji in the Type column and add a legend\n before each table; regenerate Configuration.md and velox-configuration.md.\n\nWhen REV predates the move to docs/backends, look up the function support\ndoc at its old location, and fail explicitly if neither path exists instead\nof treating a failed git show as an empty document.\n\nCo-Authored-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e" }, { "commit": "c4b3cf7861ccadab10ecec17d0f346b7629111f2", "tree": "d099681a006f79c068f39f22fc6cfe76f9387bfe", "parents": [ "06ac8a1c5b7687a7b327b43d88b94ca3a2cacb7d" ], "author": { "name": "YangJie", "email": "yangjie01@baidu.com", "time": "Tue Oct 20 23:34:01 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 21 11:34:01 2026 +0800" }, "message": "[GLUTEN-13342][VL] Remove the unused ShuffleSchema.h (#13345)" }, { "commit": "06ac8a1c5b7687a7b327b43d88b94ca3a2cacb7d", "tree": "1b47a3f99022a3733412d4d2945be836f8a8d673", "parents": [ "dcac144c31cd816201a86279335a4934e1e8ce91" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Tue Oct 20 20:24:47 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Oct 21 11:24:47 2026 +0800" }, "message": "[GLUTEN-13175][VL] Re-enable the Spark 4.2 MERGE summary tests in GlutenDeltaBasedMergeIntoTableUpdateAsDeleteAndInsertSuite (#13318)" }, { "commit": "dcac144c31cd816201a86279335a4934e1e8ce91", "tree": "48ccf37e3bdcb572f080b3b6bfe83d704a4f5602", "parents": [ "ea08752e653acba49c417fee9e99592dd61f4c9a" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Tue Oct 20 15:35:12 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 20 15:35:12 2026 -0700" }, "message": "Bump pip from 26.1 to 26.2 in /tools/workload/benchmark_velox/analysis (#13397)\n\nBumps [pip](https://github.com/pypa/pip) from 26.1 to 26.2.\n- [Changelog](https://github.com/pypa/pip/blob/main/NEWS.rst)\n- [Commits](https://github.com/pypa/pip/compare/26.1...26.2)\n\n---\nupdated-dependencies:\n- dependency-name: pip\n dependency-version: \u002726.2\u0027\n dependency-type: direct:production\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "ea08752e653acba49c417fee9e99592dd61f4c9a", "tree": "3e6f033c36bf3b761552d19b9b198998f4245cf2", "parents": [ "01f3c69403da02a38de15e229a08d6061c80ce6a" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Tue Oct 20 11:27:43 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 20 11:27:43 2026 -0700" }, "message": "[GLUTEN-13175][VL] Re-enable the Spark 4.2 DELETE tests in GlutenDeleteFromTableSuite (#13316)" }, { "commit": "01f3c69403da02a38de15e229a08d6061c80ce6a", "tree": "e80fcebb2e37b9c9e1a630d80b11917ad045752b", "parents": [ "96ec5a2e416aa7035b783cbcfc16a50efe3adf8b" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Tue Oct 20 11:25:48 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 20 11:25:48 2026 -0700" }, "message": "[GLUTEN-13175][VL] Add Spark 4.2 UPDATE NOT NULL test (#13313)" }, { "commit": "96ec5a2e416aa7035b783cbcfc16a50efe3adf8b", "tree": "ff352492ae94d12524dab3fc337e2ccb1ce622eb", "parents": [ "38686d5b9267f8f63ed3a1c862bdd0f0077ae048" ], "author": { "name": "Gluten Performance Bot", "email": "137994563+GlutenPerfBot@users.noreply.github.com", "time": "Tue Oct 20 18:42:15 2026 +0100" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 20 18:42:15 2026 +0100" }, "message": "[GLUTEN-6887][VL] Daily Update Velox Version (2026_10_20) (#13376)\n\n* [GLUTEN-6887][VL] Daily Update Velox Version (dft-2026_10_20)\n\nUpstream Velox\u0027s New Commits:\n303d71497 by Apurva Kumar, fix: Fix rowsRemaining check for merge-on-read inserts with remaining filter (#19626)\n394244a9a by Natasha Sehgal, fix(binary): Pad VARBINARY by bytes in lpad and rpad (#19597)\na26ac7c3e by Apurva Kumar, test(iceberg): Build Iceberg tests on both Buck and CMake (#19608)\n7390907bb by Pedro Pedreira, fix(vector): Make `FlatMapVector::projectKey()` null rows without the key (#19641)\nc01832afa by zhichenxu-meta, fix(dwio): Make dictionary copy bounds explicit for GCC 12\nb4c248801 by Jimmy Lu, refactor(exec): Remove FixedPointOptions::upstreamExchangeUri (#19616)\n0f8e409d5 by RindsSchei225e, fix(fuzzer): Quote column names in table evolution fuzzer projections\ncecd67084 by dependabot[bot], build(ci): Bump actions/download-artifact from 8.0.1 to 8.0.2\nb1e90d2ee by dependabot[bot], build(ci): Bump dorny/paths-filter from 3.0.2 to 4.0.3\n90eb9aa4e by dependabot[bot], build(ci): Bump actions/labeler from 5.0.0 to 7.0.0\n090ba6f90 by Henry Dikeman, fix(serializer): Own encoding backing buffers across decode calls (#19601)\n2a81dfe23 by Zhenyuan Zhao, feat(arrow): Import fixed-size binary arrays as VARBINARY\n204f6e8d7 by Zhu, Lipeng, perf(functions): Inline Murmur3 mixing steps into the header\nc04fdbf6b by Felipe Pessoto, fix(spark): Preserve TIMESTAMP_UTC values in to_json\n88268ae2b by Apurva Kumar, test(dwio): Fix flaky CachedBufferedInputTest.readGapTracking\n9096a3af2 by Xiaoxuan, perf(nimble): Make index projector clock reads optional\n48926011f by Maria Basmanova, fix: Handle throwing close() and backward clock steps (#19640)\n4a5aa0acd by oerling, test(torchwave): Add the in-place copy and scatter_add fixtures\nead429b50 by Maria Basmanova, fix(exec): Fix hang when Exchange driver 0 stops early (#19629)\n26d8dde90 by Suryadev Sahadevan Rajesh, feat(nimble): Pin subintsplit per column (#19529)\ne1da302cf by Pedrum Jalali, fix(build): Add missing unistd.h includes\n330f845c3 by Maria Basmanova, fix(exec): Fixed point splits and one-plan parallel bodies (#19618)\ne96cf371e by Xiaoxuan, fix(test): Fix flaky HashJoinTest.reclaimDuringReserve\nbc5a3e1c9 by Maria Basmanova, refactor(exec): Run fixed-point tasks as driver pipelines (#19610)\nac868ec8a by zhichenxu-meta, fix(rpc): Preserve shared pacing during BATCH recovery (#19594)\n16b674c86 by Xiaoxuan, perf(nimble): Batch nullable encoding view reads\n90c32c5ee by Karthikeyan, refactor: Make Timestamp and TimeUtils zone conversions device-callable\n214422750 by Maria Basmanova, feat(exec): Run a non-shuffling fixed point on many tasks (#19606)\n2d4abf673 by oerling, fix(torchwave): Wave and TorchWave fixes from tw_inplace2\n06a713c97 by Maria Basmanova, feat(functions): Add functions for the Axiom optimizer (#19615)\n6d6979b4d by kevinwilfong, fix(exec): Record in the spilled row whether a distinct key was emitted (#19179)\n5a303a3f3 by Karthikeyan, fix(decimal): Stop Presto mod from narrowing long operands\n44d4a44fc by Shruti Shivakumar, perf(cudf): Finalize buffered group-by states directly\n180d8270e by Shruti Shivakumar, build(cudf): Update cuDF pin and memory resource usage\n6c88e53e2 by Zoltán Arnold Nagy, build(ci): Install UCX in the adapters image for cuDF builds\ne6b1de834 by Shruti Shivakumar, perf(cudf): Reduce Decimal64 global aggregates without widening\n81a48ef0a by Maria Basmanova, fix(exec): Fix hangs in FixedPoint plans (#19603)\nf33d11cf3 by XiaoDu, perf(nimble): Vectorize cosine normalization (#19438)\n90f9d4172 by XiaoDu, feat(nimble): Report vector search work (#19432)\n0166f69b6 by Apurva Kumar, fix(dwio): Bounds-check selective dictionary indices (#19263)\n\nSigned-off-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e\n\n* do cache re-build on native changes\n\nSigned-off-by: Yuan \u003cyuanzhou@apache.org\u003e\n\n---------\n\nSigned-off-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e\nSigned-off-by: Yuan \u003cyuanzhou@apache.org\u003e\nCo-authored-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e\nCo-authored-by: Yuan \u003cyuanzhou@apache.org\u003e" }, { "commit": "38686d5b9267f8f63ed3a1c862bdd0f0077ae048", "tree": "411fc42f38fa9c5844e7e9c0416e38489fa35478", "parents": [ "2c32a92af7908c1a2f27560f53f2c1ce52508f95" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Tue Oct 20 09:28:59 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 20 09:28:59 2026 -0700" }, "message": "[GLUTEN-13177][VL] Refresh the Spark 4.2 TPC-DS approved plans (#13312)" }, { "commit": "2c32a92af7908c1a2f27560f53f2c1ce52508f95", "tree": "a94fd39ce08dad5ced56f394b2c59d9160def158", "parents": [ "693d70506a5b0d1b9f9ad81af26599c8f5500f13" ], "author": { "name": "Mohammad Linjawi", "email": "linjawi.mohammad@gmail.com", "time": "Tue Oct 20 12:26:42 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 20 10:26:42 2026 +0100" }, "message": "[VL] Make ValueStreamDataSource::next() iterative instead of recursive (#13346)" }, { "commit": "693d70506a5b0d1b9f9ad81af26599c8f5500f13", "tree": "202676745577e55927a42974accefd32fdcfc019", "parents": [ "0ef7c9325a5dd8e7785146d9ca45980b0d941b50" ], "author": { "name": "Mohammad Linjawi", "email": "linjawi.mohammad@gmail.com", "time": "Tue Oct 20 12:20:23 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 20 10:20:23 2026 +0100" }, "message": "[GLUTEN-12025][DOC] Rewrite the Velox Delta Lake support page (#13338)" }, { "commit": "0ef7c9325a5dd8e7785146d9ca45980b0d941b50", "tree": "2b8c5b87e4d5b0d3bae04f82d7eaabaf49e0a987", "parents": [ "7edf7f6d587a7c92e4f69767e3b13edb36dbf62f" ], "author": { "name": "BInwei Yang", "email": "felixybw@apache.org", "time": "Tue Oct 20 02:15:44 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 20 10:15:44 2026 +0100" }, "message": "[MINOR][DOC] Add developer doc on Arrow columnar conversion (#13374)\n\n* [MINOR][DOC] Add developer doc on Arrow columnar conversion\n\nDescribe how Gluten converts between rows, Spark batches, Java Arrow,\nnative Arrow and Velox batches today (conventions, transition graph,\nwhat each transition does, copies, batch lifecycle), and how to add an\nArrow-based columnar operator, using the Lance scan offload as example.\n\nCo-Authored-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e\n\n* [MINOR][DOC] Add Arrow UDF buffer lifecycle example and diagrams\n\n- Add section 4: a Velox BIGINT column through LoadArrowData, an Arrow\n UDF and back to Velox, with the reference counts at each step and the\n release callbacks back to Velox\u0027s memory (PlantUML sequence diagram).\n- Explain how an ArrowBuf is released depending on whether it was\n allocated in Java or imported through C Data, and the memory\n accounting of imported buffers (second PlantUML diagram).\n- Replace the mermaid transition graph with rendered images.\n\nCo-Authored-By: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e\n\n---------\n\nCo-authored-by: Claude Opus 5.5 \u003cnoreply@anthropic.com\u003e" }, { "commit": "7edf7f6d587a7c92e4f69767e3b13edb36dbf62f", "tree": "62df26ae0eb9a8235d328a306c7631159f2b874a", "parents": [ "08417cfb18459a6f3f40e1427745a2a78f5d6b06" ], "author": { "name": "inf / Hazmi", "email": "ialhazmim@gmail.com", "time": "Tue Oct 20 11:57:46 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 20 09:57:46 2026 +0100" }, "message": "[VL][CI] Show positive changes in Iceberg CI log (#13365)" }, { "commit": "08417cfb18459a6f3f40e1427745a2a78f5d6b06", "tree": "3b175a3dc9759f40e096a8aaa5a55fc9507c9e65", "parents": [ "cf2bd18de72644196a90ff77d64e7809f1d74f9d" ], "author": { "name": "Reema", "email": "80041251+ReemaAlzaid@users.noreply.github.com", "time": "Tue Oct 20 11:28:21 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 20 09:28:21 2026 +0100" }, "message": "[VL] Add native Kafka connector (#13183)" }, { "commit": "cf2bd18de72644196a90ff77d64e7809f1d74f9d", "tree": "1a0f63294601e230ad03cd65223fe083328ceef7", "parents": [ "454d9c648bb97618a3bf9ca75c11cb198bd510ba" ], "author": { "name": "YangJie", "email": "yangjie01@baidu.com", "time": "Tue Oct 20 02:59:41 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Oct 20 07:59:41 2026 +0100" }, "message": "[VL] Remap generated build key aliases in the driver-side hash table build (#13332)" }, { "commit": "454d9c648bb97618a3bf9ca75c11cb198bd510ba", "tree": "b823fbe2b99e8705385a8dfe5f6522f874f8df66", "parents": [ "c982171094c49b5c6d0d1acbbdb83c09a4c8a09d" ], "author": { "name": "jiangjiangtian", "email": "97602666+jiangjiangtian@users.noreply.github.com", "time": "Tue Oct 20 07:54:59 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 19 16:54:59 2026 -0700" }, "message": "[GLUTEN-12413][VL] Enable ReusedExchange by normalizing Generate pre-project expressions as Alias (#12945)" }, { "commit": "c982171094c49b5c6d0d1acbbdb83c09a4c8a09d", "tree": "4fec0bf55d8173e3284d83db140334809a2b8804", "parents": [ "ab610b8a588731ffa2f2633c1722b31f3662ef20" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Mon Oct 19 14:34:06 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 19 14:34:06 2026 -0700" }, "message": "[GLUTEN-13175][VL] Add Spark 4.2 NOT NULL INSERT tests (#13248)" }, { "commit": "ab610b8a588731ffa2f2633c1722b31f3662ef20", "tree": "ae4eaf519bb24445d09dd2a076116e3532bcff01", "parents": [ "e920e70e453fe9d035e84fd75b635ba03324f387" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Mon Oct 19 14:28:31 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 19 14:28:31 2026 -0700" }, "message": "[GLUTEN-13177][VL] Refresh the Spark 4.2 SQLQueryTest golden files: postgreSQL/int4, postgreSQL/int8, try_aggregates (#13245)" }, { "commit": "e920e70e453fe9d035e84fd75b635ba03324f387", "tree": "015fb5a7d9944a206f7609c4873959319f0731df", "parents": [ "b9dda4b95536b2a4c72b3820234ca2fffc79c9a1" ], "author": { "name": "Mohammad Linjawi", "email": "linjawi.mohammad@gmail.com", "time": "Mon Oct 19 17:23:07 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 19 15:23:07 2026 +0100" }, "message": "[GLUTEN-12569][VL] Enable Delta on Spark 4.2: one Delta 4.4.1 release for Spark 4.x, following Delta\u0027s write path (#13271)" }, { "commit": "b9dda4b95536b2a4c72b3820234ca2fffc79c9a1", "tree": "88e3e82968814605a0997a66e33bf52c50d0af9d", "parents": [ "99404d9d84b683f3e4b38d8032479e488b2a506d" ], "author": { "name": "YangJie", "email": "yangjie01@baidu.com", "time": "Mon Oct 19 07:31:16 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 19 12:31:16 2026 +0100" }, "message": "[GLUTEN-13349][VL] Fix flushable aggregation double-counting distinct keys (#13350)" }, { "commit": "99404d9d84b683f3e4b38d8032479e488b2a506d", "tree": "e356cfb9f514e82a6cd760deb146e17398831d5e", "parents": [ "d9b84ee762d275d616c58cad2ea29600ee421ddd" ], "author": { "name": "YangJie", "email": "yangjie01@baidu.com", "time": "Mon Oct 19 05:31:50 2026 -0400" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 19 10:31:50 2026 +0100" }, "message": "[GLUTEN-13329][VL] Match the TencentOS Server os-release name in the shared library loaders (#13331)" }, { "commit": "d9b84ee762d275d616c58cad2ea29600ee421ddd", "tree": "624e10a4f6129cf3b6b3e2cf393e62c8d9b7f88f", "parents": [ "cb80c4406c12a227c7ef981878350ba687c9be06" ], "author": { "name": "EJ Song", "email": "51077614+sezruby@users.noreply.github.com", "time": "Mon Oct 19 02:21:34 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 19 10:21:34 2026 +0100" }, "message": "[VL] Add Velox offload for Lance scans via Arrow C stream (#12958)\n\n* [VL] Add Velox offload for Lance scans via Arrow C stream\n\nOffloads read-only Lance BatchScanExec to the Velox backend through the\nArrow C Data Interface, using lance-spark\u0027s LanceArrowStreamScanner\n(lance-spark#778). Only the Arrow C-struct address crosses the classloader\nboundary, so Gluten\u0027s Arrow version stays decoupled from lance-spark\u0027s.\n\nThe offload is gated on being a plain columnar Lance scan (no pushed\naggregation, no full-text query); anything else falls back to vanilla\nSpark. A velox-lance Component registers the rule and only engages when\nlance-spark is on the classpath.\n\nCode lives under backends-velox/src-lance behind a new `lance` Maven\nprofile, mirroring the src-iceberg velox-integration layout. The read path\nis entirely velox-specific (ArrowWritableColumnVector, ArrowBufferAllocators,\nValidatablePlan), so no top-level gluten-lance module is added yet; one can\nbe introduced when an engine-agnostic write path lands." }, { "commit": "cb80c4406c12a227c7ef981878350ba687c9be06", "tree": "7a1bd20d90992bf67076ac3a18a77c219b062677", "parents": [ "f64be861560e6228d8653e14dc65063c8b3925c9" ], "author": { "name": "Gluten Performance Bot", "email": "137994563+GlutenPerfBot@users.noreply.github.com", "time": "Sun Oct 18 07:52:55 2026 +0100" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sun Oct 18 07:52:55 2026 +0100" }, "message": "[GLUTEN-6887][VL] Daily Update Velox Version (dft-2026_10_17) (#13348)\n\nUpstream Velox\u0027s New Commits:\nea84d5760 by XiaoDu, perf(nimble): Decode fixed-bit values into output (#19549)\n9f11c6f6a by XiaoDu, fix(nimble): Correct PFOR estimates and scratch reuse (#19545)\n75f38d93e by Wannuc-M, fix(rpc): Retry per-row client errors once\n4ed96f483 by dependabot[bot], build(ci): Bump apache/infrastructure-actions/stash/save from 3354c1565d4b0e335b78a76aedd82153a9e144d4 to b1b1ab191498302581efdef55279706f79249c0e\na8214abaa by Apurva Kumar, fix(type): Answer the int64 filter tests from HugeintRange\n38eeae40b by Krishna Pai, fix(function): Stop inverse_poisson_cdf hanging on large lambda\n07093317c by xiaodouchen, fix(cudf): Bind scan workers to the selected GPU\n17a4983e2 by Mohammad Linjawi, fix(exec)!: Forward constants to window and companion aggregates\neba8b804f by dependabot[bot], build(ci): Bump docker/login-action from 4.2.0 to 4.6.0\ned7bd9129 by Natasha Sehgal, fix(datetime): Preserve AT TIME ZONE wall clock (#19528)\n6be32f4d2 by Apurva Kumar, feat(iceberg): Skip filters that a split\u0027s partition values enforce\n7e79f30f5 by Maria Basmanova, fix(fixed-point): Reject table reads outside initial plans\nb048d7482 by Muhammad Haseeb, fix(cudf): Use connector config in cudf DV reader\n2152a46dd by peterenescu, fix(sparksql): Accept more than 10 key-value pairs in map\n349e414fd by zhichenxu-meta, feat(rpc): Grow BATCH window on clean no-latency completions (#19582)\n38da698c8 by zhichenxu-meta, feat(rpc): Configure BATCH congestion start window (#19569)\n\nSigned-off-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e\nCo-authored-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e" }, { "commit": "f64be861560e6228d8653e14dc65063c8b3925c9", "tree": "5fc441561e0b834695ce71d217c1e98883b2ee86", "parents": [ "8040c1567993ed5265b76c8b4fa92f53490fa14b" ], "author": { "name": "Reema", "email": "80041251+ReemaAlzaid@users.noreply.github.com", "time": "Sun Oct 18 09:51:11 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sun Oct 18 07:51:11 2026 +0100" }, "message": "[MINOR][DOC] Correct the C++ formatter script path (#13354)" }, { "commit": "8040c1567993ed5265b76c8b4fa92f53490fa14b", "tree": "9ef4942e26ef64aac2cf68773bb28010819c36aa", "parents": [ "bf964154c0854afa6496069e0524df5bd24ccf40" ], "author": { "name": "Reema", "email": "80041251+ReemaAlzaid@users.noreply.github.com", "time": "Sun Oct 18 09:50:24 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sun Oct 18 07:50:24 2026 +0100" }, "message": "[MINOR][BUILD] Respect custom Spark resource installation directories (#13353)" }, { "commit": "bf964154c0854afa6496069e0524df5bd24ccf40", "tree": "647af2fa32431a56421cc7beff87e98d406af8c4", "parents": [ "a5ceb3793b7d22be5ec777ae322db3838d9fee67" ], "author": { "name": "Chang Chen", "email": "changchen@microsoft.com", "time": "Sat Oct 17 21:04:02 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Oct 17 21:04:02 2026 +0800" }, "message": "[GLUTEN-13167][VL] Fix macOS builds and decouple native tests from JNI libraries (#13168)\n\n* [VL] Fix macOS OpenSSL 3 setup and GoogleTest installation\n\n* build(vcpkg): own Roaring and resolve packages without shell setup\n\n* fix(native): link tests to implementation objects instead of JNI DSOs\n\n* [MINOR] Format native CMake targets\n\n* [GLUTEN-13167][VL] Expose imported Arrow targets across native directories\n\n* [GLUTEN-13167][VL] Preserve shared-library linkage for UDF integration tests\n\n* [MINOR][CI] Trigger weekly native platform validation" }, { "commit": "a5ceb3793b7d22be5ec777ae322db3838d9fee67", "tree": "ccd424484738c1c44b88d9c9b4c48d28865695a3", "parents": [ "a256c510832206e7d3ddb3223139d3412a7f2664" ], "author": { "name": "rdtr", "email": "norio@apache.org", "time": "Fri Oct 16 20:51:01 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 16 20:51:01 2026 -0700" }, "message": "[GLUTEN-7628][VL] Support monotonically_increasing_id function (#13268)" }, { "commit": "a256c510832206e7d3ddb3223139d3412a7f2664", "tree": "7c0a277b95f2eb4f975bdf3cfb7d105f6c16aa00", "parents": [ "844241f500d65b31b41e7d9f9616324e2f48be9c" ], "author": { "name": "Mohammad Linjawi", "email": "linjawi.mohammad@gmail.com", "time": "Sat Oct 17 01:27:42 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 16 15:27:42 2026 -0700" }, "message": "[MINOR][INFRA] Use the CI license header check instead of dev/check.py header (#13339)" }, { "commit": "844241f500d65b31b41e7d9f9616324e2f48be9c", "tree": "97c88273dd7a94f59d234120393488c51fd0827f", "parents": [ "8f465236f3529f8bde006d87a22d9251e164acb3" ], "author": { "name": "Kaifei Yi", "email": "yikaifei@apache.org", "time": "Sat Oct 17 05:09:06 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 16 14:09:06 2026 -0700" }, "message": "[VL] Use a dedicated CPU thread pool for parallel broadcast hash-table build (#13115)" }, { "commit": "8f465236f3529f8bde006d87a22d9251e164acb3", "tree": "9154119e5974a11cae25a0f47c0fe81bd4899337", "parents": [ "1cf2c0d956b555422785d673dfb76c9ca9046269" ], "author": { "name": "Mohammad Linjawi", "email": "linjawi.mohammad@gmail.com", "time": "Fri Oct 16 20:35:11 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 16 10:35:11 2026 -0700" }, "message": "[GLUTEN-13290][VL] Remove remaining CentOS 7 support and references (#13336)" }, { "commit": "1cf2c0d956b555422785d673dfb76c9ca9046269", "tree": "344b65cc5550558d4047abe2d3169d0162b2d25b", "parents": [ "11d1e8766bf4905a956d44470d74bafd056fa8c9" ], "author": { "name": "Hongze Zhang", "email": "hongze.zzz123@gmail.com", "time": "Fri Oct 16 16:38:54 2026 +0100" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 16 16:38:54 2026 +0100" }, "message": "[VL] Eager aggregation: Refactor option partialMerge.mode (#13328)" }, { "commit": "11d1e8766bf4905a956d44470d74bafd056fa8c9", "tree": "1391091d1a40c54cb6e7f651d2cf3d86feaa2a50", "parents": [ "60200390f473e93203cae1781521705b7cc251c2" ], "author": { "name": "Kaifei Yi", "email": "yikaifei@apache.org", "time": "Fri Oct 16 18:59:44 2026 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 16 11:59:44 2026 +0100" }, "message": "[VL][CH] Preserve broadcast resource IDs during concurrent registration (#13320)" }, { "commit": "60200390f473e93203cae1781521705b7cc251c2", "tree": "68a9e815d7f5f088863e682bba968c54bd1763cd", "parents": [ "9e9638d54fab22b56d819ceec8e421bc9cfef808" ], "author": { "name": "DongjunHwang", "email": "33440277+HwangDongJun@users.noreply.github.com", "time": "Fri Oct 16 17:23:17 2026 +0900" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 16 09:23:17 2026 +0100" }, "message": "[GLUTEN-12712][CORE] Derive Iceberg getRootPathsInternal from actual scanned file paths (#12833)" }, { "commit": "9e9638d54fab22b56d819ceec8e421bc9cfef808", "tree": "661912eab6028647be705bd9d8a555f253b18980", "parents": [ "eb2f5ac006730d41089f3c6556f58fe0c31d025a" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Fri Oct 16 09:21:37 2026 +0100" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 16 09:21:37 2026 +0100" }, "message": "Bump org.apache.spark:spark-core_2.12 in /tools/gluten-it (#13325)" }, { "commit": "eb2f5ac006730d41089f3c6556f58fe0c31d025a", "tree": "dbfca058378157fc9b92963116e0c932415cae3b", "parents": [ "822c6da2bbc53e285c55e4c55a6afe2d44017b35" ], "author": { "name": "Gluten Performance Bot", "email": "137994563+GlutenPerfBot@users.noreply.github.com", "time": "Fri Oct 16 09:20:59 2026 +0100" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 16 09:20:59 2026 +0100" }, "message": "[GLUTEN-6887][VL] Daily Update Velox Version (2026_10_15) (#13289)\n\n* [GLUTEN-6887][VL] Daily Update Velox Version (dft-2026_10_15)\n\nUpstream Velox\u0027s New Commits:\n41498076c by Andrii Rosa, feat(config): Register task_partitioned_writer_count session property\n1ae10f98c by Karthikeyan, refactor(functions): Move calendar functions to a device-compatible header\n4d0721399 by RindsSchei225e, test(nimble): Let the writer fuzzer force Huffman (#18929)\nf0276a8b6 by Andrii Rosa, fix: Paste test name in disabled DEBUG_ONLY_CO_TEST_F\ne162d0b21 by Muhammad Haseeb, feat(cudf): Velox-cuDF table scan ignores missing files when specified\n9e111b510 by xiaodouchen, fix(type): Stop integer filter loops at INT64_MAX\n3e035c893 by Karthikeyan, fix(type): Reject out-of-range time zone ids in locateZone\n409cf8d63 by Wechar Yu, fix: Unify Presto serialized-size estimation\nff8dfcdfe by adheer-araokar, fix(memory): Fix free capacity underflow in shrink\na2ff335a3 by Ke, refactor(fuzzer): Remove Nimble stats from TableEvolutionFuzzer (#19559)\n9db3317a9 by Pedro Pedreira, feat(vector): Add `FlatMapVector::getKeyChannels()` (#19537)\n53ece251d by adheer-araokar, fix(memory): Fix stray placeholder in free capacity error\nb0706264b by Karthikeyan, feat(decimal): Make decimal call() bodies device-compatible\n3286f4d29 by kevinwilfong, test: Reduce io_uring memory in DataInput tests\n05cc69aa0 by Karthikeyan, perf(functions): Compute round()\u0027s scale factor once for constant digits\n\nSigned-off-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e\n\n* misc fix rhel setup\n\nSigned-off-by: Yuan \u003cyuanzhou@apache.org\u003e\n\n* remove centos7 scripts\n\nSigned-off-by: Yuan \u003cyuanzhou@apache.org\u003e\n\n* remove centos7 script\n\nSigned-off-by: Yuan \u003cyuanzhou@apache.org\u003e\n\n* fix java setup on kafka\n\nSigned-off-by: Yuan \u003cyuanzhou@apache.org\u003e\n\n---------\n\nSigned-off-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e\nSigned-off-by: Yuan \u003cyuanzhou@apache.org\u003e\nCo-authored-by: glutenperfbot \u003cglutenperfbot@glutenproject-internal.com\u003e\nCo-authored-by: Yuan \u003cyuanzhou@apache.org\u003e" }, { "commit": "822c6da2bbc53e285c55e4c55a6afe2d44017b35", "tree": "ae0222ba75137a8a9f8990392b6a27be9f2f86a9", "parents": [ "d3325b000053c48864442e0452af64ef26ac9b61" ], "author": { "name": "Pedrum Jalali", "email": "pedrum.jalali@gmail.com", "time": "Fri Oct 16 01:17:59 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 16 09:17:59 2026 +0100" }, "message": "[CORE] Keep struct field names in udfLibraryPaths UDF signatures (#13274)" }, { "commit": "d3325b000053c48864442e0452af64ef26ac9b61", "tree": "35b1019724914892cf1d7a134f8aba73ac28d534", "parents": [ "27aa68207aff4d734cec2605044f0e25801309df" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Thu Oct 15 19:32:41 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 16 10:32:41 2026 +0800" }, "message": "[GLUTEN-13174][VL] Re-enable the Spark 4.2 storage-partitioned join tests (#13243)\n\n* [GLUTEN-13174][VL] Re-enable the Spark 4.2 storage-partitioned join tests\n\nRemove the 29 GlutenKeyGroupedPartitioningSuite excludes that #13163 added for\nSpark 4.2, so the spark42 UT CI shows which of them still fail on current\nmain. Fixes for the remaining failures follow in this PR.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9\n\n* [GLUTEN-13174][VL] Leave the SPJ sorted-merge coalescing tests to a separate PR\n\nKeep the 4 SPARK-55715 / SPARK-56549 tests excluded here. They need a\ndifferent fix and have their own PR.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9\n\n* [GLUTEN-13174][VL] Look for GroupPartitionsExec under SortMergeJoinExecTransformer\n\nSpark 4.2 collectGroupPartitions only looks under a vanilla SortMergeJoinExec,\nwhich Gluten replaces with SortMergeJoinExecTransformer. Override it the same\nway as collectShuffles. This fixes 8 of the re-enabled tests.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9\n\n* [GLUTEN-13174][VL] Port the Gluten SPJ test copies to Spark 4.2\n\nSpark 4.2 moved SPJ partition grouping from BatchScanExec into\nGroupPartitionsExec and updated its KeyGroupedPartitioningSuite for it.\nBring the Gluten copies in GlutenKeyGroupedPartitioningSuite in line:\n\n- SPARK-42038 (8 tests), SPARK-44647 (3 tests) and SPARK-47094: check\n the GroupPartitionsExec output partitions instead of the raw scan\n split counts, with the Spark 4.2 expectations (no GroupPartitionsExec\n when SPJ is not used). The SPARK-42038 checks also require a\n non-empty result.\n- SPARK-53322: Spark 4.2 uses checkpointed scans for SPJ. Replace the\n \"aren\u0027t used for SPJ\" and \"can\u0027t shuffle other children on SPJ\" copies\n with the Spark 4.2 \"are used for SPJ\" and \"can shuffle other children\n on SPJ\" tests, and remove \"can be shuffled by children on SPJ\", which\n Spark 4.2 dropped. Add the Spark 4.2 GroupPartitionsExec checks to\n \"avoid shuffles for aggregates\".\n- collectAllShuffles returns vanilla and columnar shuffles again, as the\n inherited tests expect. Native-only checks use the new\n collectAllColumnarShuffles.\n- Add Gluten variants of the two SPARK-56241 tests, which look for a\n vanilla SortMergeJoinExec, and of \"SPARK-55848: checkpointed\n partially-clustered join with dedup\", which looks for a vanilla\n RDDScanExec that Gluten offloads to an RDDScanTransformer. The vanilla\n tests are excluded.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9\n\n---------\n\nCo-authored-by: Akshay Tayal \u003cakshaytayal@users.noreply.github.com\u003e\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9" }, { "commit": "27aa68207aff4d734cec2605044f0e25801309df", "tree": "b4df33437cfce6570f584e09d485f50500fbe7df", "parents": [ "020211a4b4c132f8133d0b3786b2d1928d28f0bf" ], "author": { "name": "Akshay Tayal", "email": "500399+akshaytayal@users.noreply.github.com", "time": "Thu Oct 15 19:32:29 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Oct 16 10:32:29 2026 +0800" }, "message": "[GLUTEN-12569][CORE] Add enhanced DSv2 filter test suites for Spark 4.2 (#13164)\n\n* [GLUTEN-12569][CORE] Add enhanced runtime partition filter tests for Spark 4.2\n\nWrap Spark 4.2\u0027s DataSourceV2EnhancedRuntimePartitionFilterSuite with the\nGluten test trait and enable it for Velox. This adds 15 upstream tests for\nthe iterative runtime partition-filter path used by BatchScanExec.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 8ae7d6bd-f561-417a-a635-40fe24dc67a5\n(cherry picked from commit ed127752e36e74820f061da7478cfee0c105c571)\n\n* [GLUTEN-12569][CORE] Add enhanced partition and delete filter tests for Spark 4.2\n\nWrap Spark 4.2\u0027s DataSourceV2EnhancedPartitionFilterSuite and\nDataSourceV2EnhancedDeleteFilterSuite with the Gluten test trait and enable\nthem for Velox, completing coverage of the three new enhanced DSv2 filter\nsuites.\n\nThis adds 37 upstream tests covering first/second-pass partition filter\npushdown, untranslatable and nested partition predicates, post-scan filter\nretention, and delete filter handling.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 8ae7d6bd-f561-417a-a635-40fe24dc67a5\n(cherry picked from commit 6a8449800685c72cdffefe0593a38828e3f8497a)\n\n* [GLUTEN-12569][VL] Disable 2 CI-failing enhanced DSv2 partition-filter UTs for Spark 4.2\n\nGlutenDataSourceV2EnhancedPartitionFilterSuite \u0027case 9 ...\u0027 (+ nested variant):\nthe test asserts a post-scan vanilla FilterExec remains, but Gluten replaces it\n(columnar transform) / the InMemoryEnhancedPartitionFilterBatchScan falls back,\nso the plan-shape assertion fails. CI-confirmed (apache #13164 group3). Disabled\nvia VeloxTestSettings.exclude; the other enhanced suites (Delete, RuntimePartition)\npass.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 413738d2-8cf1-4c70-9331-6c5e4314f9fa\n\n* [GLUTEN-12569][VL] Rewrite the 2 case-9 enhanced partition-filter UTs as Gluten tests\n\nThe vanilla \"case 9\" tests in DataSourceV2EnhancedPartitionFilterSuite assert\nthat a Spark FilterExec remains above the scan for the partition filter the\nsource returns in the first pass. With Gluten, the in-memory DSv2 scan falls\nback but that post-scan filter is offloaded to a native\nFilterExecTransformer, so the vanilla assertion fails even though the plan is\ncorrect.\n\nKeep the vanilla tests excluded and add Gluten variants that keep the answer,\nsecond-pass pushdown and scanned-partition checks, and accept either a Spark\nFilterExec or a Gluten FilterExecTransformerBase as the post-scan filter.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9\n\n* [GLUTEN-12569][VL] Check the returned equality in the Gluten case-9 post-scan filter\n\nAddress review feedback: the Gluten case-9 tests only checked that a\npost-scan filter exists. Require the query\u0027s filter condition, taken from\nthe analyzed plan, to be one of the conjuncts of the post-scan filter, so\na filter that keeps only IsNotNull no longer passes. Works for both\nFilterExec and FilterExecTransformerBase.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9\n\n---------\n\nCo-authored-by: Akshay Tayal \u003cakshaytayal@users.noreply.github.com\u003e\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\nCopilot-Session: 8ae7d6bd-f561-417a-a635-40fe24dc67a5\nCopilot-Session: 413738d2-8cf1-4c70-9331-6c5e4314f9fa\nCopilot-Session: 3a1d1e30-b32c-45b1-9f0e-d7aa4654eab9" }, { "commit": "020211a4b4c132f8133d0b3786b2d1928d28f0bf", "tree": "3c3d341f9005fa681d1989b5451c2d1f209f0a07", "parents": [ "57a95dc847c11e0e6103ae167a5246f358ac3417" ], "author": { "name": "Felipe Pessoto", "email": "fepessot@microsoft.com", "time": "Thu Oct 15 08:31:57 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 15 16:31:57 2026 +0100" }, "message": "[GLUTEN-12743][VL] Preserve transaction observers in native Delta commands (#13134)\n\n* [GLUTEN-12743][VL] Preserve transaction observers in native Delta commands\n\nDelegate native transaction creation and lifecycle callbacks to the existing Delta observer. Preserve next-observer advancement and nested command scopes, and add lifecycle and native DELETE regressions.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\n\n* [GLUTEN-12743][VL] Keep observer integration coverage in gluten-delta\n\nExport the native DELETE regression as an abstract gluten-delta suite and run it through the existing Velox Delta test-jar dependency. Remove the added gluten-ut dependency and use a concrete Velox suite instead.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\n\n* [GLUTEN-12743][VL] Share observer tests and refresh Delta baseline\n\nRemove the 64 observer-related cases confirmed passing in Delta CI run 36155269247, retaining the remaining footer-metadata failure. Share the unchanged observer regression suites from src-delta33 through existing-style src-delta40 symlinks, with both Spark 3.5 and Spark 4.0 coverage.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\n\n* [GLUTEN-12743][VL] Address observer regression review comments\n\nKeep native DELETE coverage in the Velox suite with typed references and assert a data-file rewrite. Tag both suites for extended CI, use the supported transaction API, and document the Delta 4.3 observer getter upgrade requirement.\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e\n\n---------\n\nCo-authored-by: Copilot \u003c223556219+Copilot@users.noreply.github.com\u003e" }, { "commit": "57a95dc847c11e0e6103ae167a5246f358ac3417", "tree": "68ffa3c5bbaab1d5792ba7f8beb43fdafcd75163", "parents": [ "30548fe6df1a1d781c047179108127a787d6f2b9" ], "author": { "name": "Felipe Pessoto", "email": "fepessot@microsoft.com", "time": "Thu Oct 15 08:31:03 2026 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 15 16:31:03 2026 +0100" }, "message": "[GLUTEN-12743][VL] Fall back Delta scans requiring generated DV metadata (#13042)\n\n* [GLUTEN-12743][VL] Fall back generated Delta DV metadata scans\n\nCo-authored-by: Copilot App \u003c223556219+Copilot@users.noreply.github.com\u003e\n\n* [GLUTEN-12743][VL] Fix Delta fallback CI\n\nCo-authored-by: Copilot App \u003c223556219+Copilot@users.noreply.github.com\u003e\n\n* [GLUTEN-12743][VL] Strengthen Delta row-index assertions\n\nCo-authored-by: Copilot App \u003c223556219+Copilot@users.noreply.github.com\u003e\n\n* [GLUTEN-12743][CI] Remove fixed Delta scan failures\n\nCo-authored-by: Copilot App \u003c223556219+Copilot@users.noreply.github.com\u003e\n\n* [GLUTEN-12743][VL] Reorder Delta scan helpers\n\nCo-authored-by: Copilot App \u003c223556219+Copilot@users.noreply.github.com\u003e\n\n* [GLUTEN-12743][VL] Preserve native Spark metadata row-index scans\n\nLimit the generated DV metadata fallback to Delta internal columns, exercise native metadata row-index reads through SQL, explain the disabled-config path, and refresh test snapshots once.\n\nCo-authored-by: Copilot App \u003c223556219+Copilot@users.noreply.github.com\u003e\n\n---------\n\nCo-authored-by: Copilot App \u003c223556219+Copilot@users.noreply.github.com\u003e" }, { "commit": "30548fe6df1a1d781c047179108127a787d6f2b9", "tree": "78318dda368a366bf0a7fc7d8f37a18601723c1b", "parents": [ "96883745bc01d478fdf4718696824415d60bbe99" ], "author": { "name": "inf / Hazmi", "email": "ialhazmim@gmail.com", "time": "Thu Oct 15 16:16:51 2026 +0300" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Oct 15 14:16:51 2026 +0100" }, "message": "[VL] Add iceberg UT CI whitelist (#13323)" } ], "next": "96883745bc01d478fdf4718696824415d60bbe99" }