chore: Update to latest main on upstream DataFusion (#1662)
* Update to main on the upstream datafusion to prepare for DF55 release
* Deprecate singular external table location
AI Disclosure: This code was written in part by an AI agent.:
* Fix tests for updated DataFusion behavior
AI Disclosure: This code was written in part by an AI agent.:
* create table requires a location
* Remove unused Python token from Substrait deserialization
AI Disclosure: This code was written in part by an AI agent.:
* test: cover external table locations binding
AI Disclosure: This code was written in part by an AI agent.:
diff --git a/Cargo.lock b/Cargo.lock
index 15895ae..d34862a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -99,9 +99,9 @@
[[package]]
name = "arrow"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471"
+checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -121,9 +121,9 @@
[[package]]
name = "arrow-arith"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738"
+checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -135,9 +135,9 @@
[[package]]
name = "arrow-array"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e"
+checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192"
dependencies = [
"ahash",
"arrow-buffer",
@@ -154,9 +154,9 @@
[[package]]
name = "arrow-avro"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376"
+checksum = "2e4f9b23a0d7b613acb59fa20bdbe0f80ffdae6411498378340b3915e45f5b84"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -178,9 +178,9 @@
[[package]]
name = "arrow-buffer"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0"
+checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434"
dependencies = [
"bytes",
"half",
@@ -190,9 +190,9 @@
[[package]]
name = "arrow-cast"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f"
+checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -201,7 +201,7 @@
"arrow-schema",
"arrow-select",
"atoi",
- "base64",
+ "base64 0.22.1",
"chrono",
"comfy-table",
"half",
@@ -212,9 +212,9 @@
[[package]]
name = "arrow-csv"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de"
+checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -227,9 +227,9 @@
[[package]]
name = "arrow-data"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0"
+checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -240,9 +240,9 @@
[[package]]
name = "arrow-ipc"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f"
+checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -256,9 +256,9 @@
[[package]]
name = "arrow-json"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b"
+checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -281,9 +281,9 @@
[[package]]
name = "arrow-ord"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0"
+checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -294,9 +294,9 @@
[[package]]
name = "arrow-pyarrow"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce"
+checksum = "bf8d967bdece4fa5a0199706730175b3df3448b87e350250a86eb6c22639e445"
dependencies = [
"arrow-array",
"arrow-data",
@@ -306,9 +306,9 @@
[[package]]
name = "arrow-row"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c"
+checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -319,9 +319,9 @@
[[package]]
name = "arrow-schema"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
+checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5"
dependencies = [
"bitflags",
"serde_core",
@@ -330,9 +330,9 @@
[[package]]
name = "arrow-select"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222"
+checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9"
dependencies = [
"ahash",
"arrow-array",
@@ -344,9 +344,9 @@
[[package]]
name = "arrow-string"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42"
+checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -385,7 +385,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -396,7 +396,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -427,6 +427,12 @@
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
+name = "base64"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9"
+
+[[package]]
name = "bigdecimal"
version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -514,12 +520,6 @@
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
-name = "byteorder"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
-
-[[package]]
name = "bytes"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -790,9 +790,8 @@
[[package]]
name = "datafusion"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"arrow-schema",
@@ -828,7 +827,7 @@
"flate2",
"futures",
"indexmap",
- "itertools",
+ "itertools 0.15.0",
"liblzma",
"log",
"object_store",
@@ -844,9 +843,8 @@
[[package]]
name = "datafusion-catalog"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"async-trait",
@@ -860,7 +858,7 @@
"datafusion-physical-plan",
"datafusion-session",
"futures",
- "itertools",
+ "itertools 0.15.0",
"log",
"object_store",
"parking_lot",
@@ -869,9 +867,8 @@
[[package]]
name = "datafusion-catalog-listing"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"async-trait",
@@ -885,16 +882,16 @@
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"futures",
- "itertools",
+ "itertools 0.15.0",
"log",
"object_store",
+ "percent-encoding",
]
[[package]]
name = "datafusion-common"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"arrow-ipc",
@@ -904,9 +901,10 @@
"half",
"hashbrown 0.17.1",
"indexmap",
- "itertools",
+ "itertools 0.15.0",
"libc",
"log",
+ "num-traits",
"object_store",
"parquet",
"recursive",
@@ -918,9 +916,8 @@
[[package]]
name = "datafusion-common-runtime"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"futures",
"log",
@@ -929,9 +926,8 @@
[[package]]
name = "datafusion-datasource"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"async-compression",
@@ -947,11 +943,12 @@
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-proto-models",
"datafusion-session",
"flate2",
"futures",
"glob",
- "itertools",
+ "itertools 0.15.0",
"liblzma",
"log",
"object_store",
@@ -965,9 +962,8 @@
[[package]]
name = "datafusion-datasource-arrow"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"arrow-ipc",
@@ -982,16 +978,15 @@
"datafusion-physical-plan",
"datafusion-session",
"futures",
- "itertools",
+ "itertools 0.15.0",
"object_store",
"tokio",
]
[[package]]
name = "datafusion-datasource-avro"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"arrow-avro",
@@ -1008,9 +1003,8 @@
[[package]]
name = "datafusion-datasource-csv"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"async-trait",
@@ -1031,9 +1025,8 @@
[[package]]
name = "datafusion-datasource-json"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"async-trait",
@@ -1054,11 +1047,11 @@
[[package]]
name = "datafusion-datasource-parquet"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
+ "arrow-schema",
"async-trait",
"bytes",
"datafusion-common",
@@ -1075,7 +1068,7 @@
"datafusion-pruning",
"datafusion-session",
"futures",
- "itertools",
+ "itertools 0.15.0",
"log",
"object_store",
"parking_lot",
@@ -1085,19 +1078,18 @@
[[package]]
name = "datafusion-doc"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
[[package]]
name = "datafusion-execution"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"arrow-buffer",
"async-trait",
+ "bytes",
"dashmap",
"datafusion-common",
"datafusion-expr",
@@ -1106,16 +1098,18 @@
"log",
"object_store",
"parking_lot",
+ "pin-project-lite",
"rand 0.9.4",
"tempfile",
+ "tokio",
+ "tokio-util",
"url",
]
[[package]]
name = "datafusion-expr"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"arrow-schema",
@@ -1128,7 +1122,7 @@
"datafusion-functions-window-common",
"datafusion-physical-expr-common",
"indexmap",
- "itertools",
+ "itertools 0.15.0",
"recursive",
"serde_json",
"sqlparser",
@@ -1136,21 +1130,19 @@
[[package]]
name = "datafusion-expr-common"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"datafusion-common",
"indexmap",
- "itertools",
+ "itertools 0.15.0",
]
[[package]]
name = "datafusion-ffi"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5660e8fa79fd51e29ce46f3026b67317ef738ebd633e106beb1a1907a406152"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"arrow-schema",
@@ -1203,13 +1195,12 @@
[[package]]
name = "datafusion-functions"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"arrow-buffer",
- "base64",
+ "base64 0.23.0",
"blake2",
"blake3",
"chrono",
@@ -1222,7 +1213,7 @@
"datafusion-macros",
"datafusion-physical-expr-common",
"hex",
- "itertools",
+ "itertools 0.15.0",
"log",
"md-5 0.11.0",
"memchr",
@@ -1235,9 +1226,8 @@
[[package]]
name = "datafusion-functions-aggregate"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"datafusion-common",
@@ -1248,17 +1238,16 @@
"datafusion-macros",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
- "foldhash 0.2.0",
"half",
+ "hashbrown 0.17.1",
"log",
"num-traits",
]
[[package]]
name = "datafusion-functions-aggregate-common"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"datafusion-common",
@@ -1268,9 +1257,8 @@
[[package]]
name = "datafusion-functions-nested"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"arrow-ord",
@@ -1285,7 +1273,7 @@
"datafusion-macros",
"datafusion-physical-expr-common",
"hashbrown 0.17.1",
- "itertools",
+ "itertools 0.15.0",
"itoa",
"log",
"memchr",
@@ -1293,9 +1281,8 @@
[[package]]
name = "datafusion-functions-table"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"async-trait",
@@ -1309,9 +1296,8 @@
[[package]]
name = "datafusion-functions-window"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"datafusion-common",
@@ -1326,9 +1312,8 @@
[[package]]
name = "datafusion-functions-window-common"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -1336,20 +1321,18 @@
[[package]]
name = "datafusion-macros"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"datafusion-doc",
"quote",
- "syn",
+ "syn 3.0.3",
]
[[package]]
name = "datafusion-optimizer"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"chrono",
@@ -1358,7 +1341,7 @@
"datafusion-expr-common",
"datafusion-physical-expr",
"indexmap",
- "itertools",
+ "itertools 0.15.0",
"log",
"recursive",
"regex",
@@ -1367,9 +1350,8 @@
[[package]]
name = "datafusion-physical-expr"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"datafusion-common",
@@ -1377,10 +1359,11 @@
"datafusion-expr-common",
"datafusion-functions-aggregate-common",
"datafusion-physical-expr-common",
+ "datafusion-proto-models",
"half",
"hashbrown 0.17.1",
"indexmap",
- "itertools",
+ "itertools 0.15.0",
"parking_lot",
"petgraph",
"recursive",
@@ -1389,9 +1372,8 @@
[[package]]
name = "datafusion-physical-expr-adapter"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"datafusion-common",
@@ -1399,31 +1381,30 @@
"datafusion-functions",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
- "itertools",
+ "itertools 0.15.0",
]
[[package]]
name = "datafusion-physical-expr-common"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"chrono",
"datafusion-common",
"datafusion-expr-common",
+ "datafusion-proto-models",
"hashbrown 0.17.1",
"indexmap",
- "itertools",
+ "itertools 0.15.0",
"parking_lot",
"pin-project",
]
[[package]]
name = "datafusion-physical-optimizer"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"datafusion-common",
@@ -1434,15 +1415,15 @@
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-pruning",
- "itertools",
+ "datafusion-session",
+ "itertools 0.15.0",
"recursive",
]
[[package]]
name = "datafusion-physical-plan"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"arrow-data",
@@ -1450,6 +1431,7 @@
"arrow-ord",
"arrow-schema",
"async-trait",
+ "bytes",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-execution",
@@ -1459,23 +1441,25 @@
"datafusion-functions-window-common",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
+ "datafusion-proto-common",
+ "datafusion-proto-models",
"futures",
"half",
"hashbrown 0.17.1",
"indexmap",
- "itertools",
+ "itertools 0.15.0",
"log",
"num-traits",
"parking_lot",
"pin-project-lite",
+ "serde_json",
"tokio",
]
[[package]]
name = "datafusion-proto"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"chrono",
@@ -1494,15 +1478,15 @@
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-proto-common",
+ "datafusion-proto-models",
"object_store",
"prost",
]
[[package]]
name = "datafusion-proto-common"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"datafusion-common",
@@ -1510,10 +1494,18 @@
]
[[package]]
+name = "datafusion-proto-models"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
+dependencies = [
+ "datafusion-proto-common",
+ "prost",
+]
+
+[[package]]
name = "datafusion-pruning"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"datafusion-common",
@@ -1572,10 +1564,10 @@
[[package]]
name = "datafusion-session"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
+ "arrow-schema",
"async-trait",
"datafusion-common",
"datafusion-execution",
@@ -1586,14 +1578,14 @@
[[package]]
name = "datafusion-spark"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"bigdecimal",
"chrono",
"crc32fast",
+ "datafusion",
"datafusion-catalog",
"datafusion-common",
"datafusion-execution",
@@ -1615,9 +1607,8 @@
[[package]]
name = "datafusion-sql"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"arrow",
"bigdecimal",
@@ -1630,20 +1621,20 @@
"recursive",
"regex",
"sqlparser",
+ "stacker",
]
[[package]]
name = "datafusion-substrait"
-version = "54.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b22c8f8c72d317e54fad6f85c0ef6d1e1da53cc7faadc7eea8daf0f8d86d4f2"
+version = "54.1.0"
+source = "git+https://github.com/apache/datafusion?rev=dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48#dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48"
dependencies = [
"async-recursion",
"async-trait",
"chrono",
"datafusion",
"half",
- "itertools",
+ "itertools 0.15.0",
"object_store",
"pbjson-types",
"prost",
@@ -1682,7 +1673,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -1835,7 +1826,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -2099,7 +2090,7 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
dependencies = [
- "base64",
+ "base64 0.22.1",
"bytes",
"futures-channel",
"futures-util",
@@ -2256,12 +2247,6 @@
]
[[package]]
-name = "integer-encoding"
-version = "3.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
-
-[[package]]
name = "ipnet"
version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2277,6 +2262,15 @@
]
[[package]]
+name = "itertools"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
+dependencies = [
+ "either",
+]
+
+[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2575,7 +2569,7 @@
checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49"
dependencies = [
"async-trait",
- "base64",
+ "base64 0.22.1",
"bytes",
"chrono",
"form_urlencoded",
@@ -2587,7 +2581,7 @@
"httparse",
"humantime",
"hyper",
- "itertools",
+ "itertools 0.14.0",
"md-5 0.10.6",
"parking_lot",
"percent-encoding",
@@ -2621,15 +2615,6 @@
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
[[package]]
-name = "ordered-float"
-version = "2.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c"
-dependencies = [
- "num-traits",
-]
-
-[[package]]
name = "parking_lot"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2654,9 +2639,9 @@
[[package]]
name = "parquet"
-version = "58.3.0"
+version = "59.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908"
+checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd"
dependencies = [
"ahash",
"arrow-array",
@@ -2665,7 +2650,7 @@
"arrow-ipc",
"arrow-schema",
"arrow-select",
- "base64",
+ "base64 0.22.1",
"brotli",
"bytes",
"chrono",
@@ -2682,7 +2667,6 @@
"seq-macro",
"simdutf8",
"snap",
- "thrift",
"tokio",
"twox-hash",
"zstd",
@@ -2700,7 +2684,7 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3"
dependencies = [
- "base64",
+ "base64 0.22.1",
"serde",
]
@@ -2711,7 +2695,7 @@
checksum = "af22d08a625a2213a78dbb0ffa253318c5c79ce3133d32d296655a7bdfb02095"
dependencies = [
"heck",
- "itertools",
+ "itertools 0.14.0",
"prost",
"prost-types",
]
@@ -2784,7 +2768,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -2830,7 +2814,7 @@
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -2868,7 +2852,7 @@
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
dependencies = [
"heck",
- "itertools",
+ "itertools 0.14.0",
"log",
"multimap",
"petgraph",
@@ -2876,7 +2860,7 @@
"prost",
"prost-types",
"regex",
- "syn",
+ "syn 2.0.118",
"tempfile",
]
@@ -2887,10 +2871,10 @@
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
dependencies = [
"anyhow",
- "itertools",
+ "itertools 0.14.0",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -2923,9 +2907,9 @@
[[package]]
name = "pyo3"
-version = "0.28.3"
+version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12"
+checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c"
dependencies = [
"libc",
"once_cell",
@@ -2937,9 +2921,9 @@
[[package]]
name = "pyo3-async-runtimes"
-version = "0.28.0"
+version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e7364a95bf00e8377bbf9b0f09d7ff9715a29d8fcf93b47d1a967363b973178"
+checksum = "b3ef68daa7316a3fac65e5e18b2203f010346de1c1c53456811a2624673ab046"
dependencies = [
"futures-channel",
"futures-util",
@@ -2951,19 +2935,18 @@
[[package]]
name = "pyo3-build-config"
-version = "0.28.3"
+version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e"
+checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078"
dependencies = [
- "python3-dll-a",
"target-lexicon",
]
[[package]]
name = "pyo3-ffi"
-version = "0.28.3"
+version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e"
+checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b"
dependencies = [
"libc",
"pyo3-build-config",
@@ -2982,36 +2965,26 @@
[[package]]
name = "pyo3-macros"
-version = "0.28.3"
+version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813"
+checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
name = "pyo3-macros-backend"
-version = "0.28.3"
+version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb"
+checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362"
dependencies = [
"heck",
"proc-macro2",
- "pyo3-build-config",
"quote",
- "syn",
-]
-
-[[package]]
-name = "python3-dll-a"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d80ba7540edb18890d444c5aa8e1f1f99b1bdf26fb26ae383135325f4a36042b"
-dependencies = [
- "cc",
+ "syn 2.0.118",
]
[[package]]
@@ -3163,7 +3136,7 @@
checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b"
dependencies = [
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -3220,7 +3193,7 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
dependencies = [
- "base64",
+ "base64 0.22.1",
"bytes",
"futures-core",
"futures-util",
@@ -3396,7 +3369,7 @@
"proc-macro2",
"quote",
"serde_derive_internals",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -3471,7 +3444,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -3482,7 +3455,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -3508,7 +3481,7 @@
"proc-macro2",
"quote",
"serde",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -3635,7 +3608,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -3669,7 +3642,7 @@
"proc-macro-crate",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -3700,7 +3673,7 @@
"heck",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -3725,7 +3698,7 @@
"serde",
"serde_json",
"serde_yaml",
- "syn",
+ "syn 2.0.118",
"typify",
"walkdir",
]
@@ -3748,6 +3721,17 @@
]
[[package]]
+name = "syn"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
name = "sync_wrapper"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3764,7 +3748,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -3803,18 +3787,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
-]
-
-[[package]]
-name = "thrift"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09"
-dependencies = [
- "byteorder",
- "integer-encoding",
- "ordered-float",
+ "syn 2.0.118",
]
[[package]]
@@ -3874,7 +3847,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -4006,7 +3979,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -4064,7 +4037,7 @@
"semver",
"serde",
"serde_json",
- "syn",
+ "syn 2.0.118",
"thiserror",
"unicode-ident",
]
@@ -4082,7 +4055,7 @@
"serde",
"serde_json",
"serde_tokenstream",
- "syn",
+ "syn 2.0.118",
"typify-impl",
]
@@ -4227,7 +4200,7 @@
"bumpalo",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
"wasm-bindgen-shared",
]
@@ -4303,7 +4276,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -4314,7 +4287,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -4537,7 +4510,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
"synstructure",
]
@@ -4558,7 +4531,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
@@ -4578,7 +4551,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
"synstructure",
]
@@ -4618,7 +4591,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.118",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index b37d653..3621599 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -32,24 +32,24 @@
[workspace.dependencies]
tokio = { version = "1.52" }
-pyo3 = { version = "0.28" }
-pyo3-async-runtimes = { version = "0.28" }
+pyo3 = { version = "0.29" }
+pyo3-async-runtimes = { version = "0.29" }
pyo3-log = "0.13.3"
chrono = { version = "0.4", default-features = false }
-arrow = { version = "58" }
-arrow-array = { version = "58" }
-arrow-schema = { version = "58" }
-arrow-select = { version = "58" }
-datafusion = { version = "54" }
-datafusion-substrait = { version = "54" }
-datafusion-proto = { version = "54" }
-datafusion-ffi = { version = "54" }
-datafusion-catalog = { version = "54", default-features = false }
-datafusion-common = { version = "54", default-features = false }
-datafusion-functions-aggregate = { version = "54" }
-datafusion-functions-window = { version = "54" }
-datafusion-spark = { version = "54" }
-datafusion-expr = { version = "54" }
+arrow = { version = "59" }
+arrow-array = { version = "59" }
+arrow-schema = { version = "59" }
+arrow-select = { version = "59" }
+datafusion = { version = "54.1.0" }
+datafusion-substrait = { version = "54.1.0" }
+datafusion-proto = { version = "54.1.0" }
+datafusion-ffi = { version = "54.1.0" }
+datafusion-catalog = { version = "54.1.0", default-features = false }
+datafusion-common = { version = "54.1.0", default-features = false }
+datafusion-functions-aggregate = { version = "54.1.0" }
+datafusion-functions-window = { version = "54.1.0" }
+datafusion-spark = { version = "54.1.0" }
+datafusion-expr = { version = "54.1.0" }
prost = "0.14.3"
serde_json = "1"
uuid = { version = "1.23" }
@@ -62,7 +62,7 @@
log = "0.4.29"
parking_lot = "0.12"
prost-types = "0.14.3" # keep in line with `datafusion-substrait`
-pyo3-build-config = "0.28"
+pyo3-build-config = "0.29"
datafusion-python-util = { path = "crates/util", version = "54.0.0" }
[profile.release]
@@ -72,3 +72,13 @@
# We cannot publish to crates.io with any patches in the below section. Developers
# must remove any entries in this section before creating a release candidate.
[patch.crates-io]
+datafusion = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-substrait = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-proto = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-ffi = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-catalog = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-common = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-functions-window = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
+datafusion-expr = { git = "https://github.com/apache/datafusion", rev = "dbcb5c0f729e9ef6b0ab4c79253fe3b657929f48" }
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index e2e922a..c5f1e01 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -53,7 +53,7 @@
datafusion-substrait = { workspace = true, optional = true }
datafusion-proto = { workspace = true }
datafusion-ffi = { workspace = true }
-datafusion-spark = { workspace = true }
+datafusion-spark = { workspace = true, features = ["core"] }
prost = { workspace = true } # keep in line with `datafusion-substrait`
serde_json = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
diff --git a/crates/core/src/array.rs b/crates/core/src/array.rs
index f284fa9..dfe9631 100644
--- a/crates/core/src/array.rs
+++ b/crates/core/src/array.rs
@@ -63,10 +63,10 @@
};
let ffi_schema = FFI_ArrowSchema::try_from(&field)?;
- let schema_capsule = PyCapsule::new(py, ffi_schema, Some(cr"arrow_schema".into()))?;
+ let schema_capsule = PyCapsule::new_with_value(py, ffi_schema, cr"arrow_schema")?;
let ffi_array = FFI_ArrowArray::new(&self.array.to_data());
- let array_capsule = PyCapsule::new(py, ffi_array, Some(cr"arrow_array".into()))?;
+ let array_capsule = PyCapsule::new_with_value(py, ffi_array, cr"arrow_array")?;
Ok((schema_capsule, array_capsule))
}
diff --git a/crates/core/src/codec.rs b/crates/core/src/codec.rs
index a6ea667..26853e6 100644
--- a/crates/core/src/codec.rs
+++ b/crates/core/src/codec.rs
@@ -100,9 +100,13 @@
TypeSignature, Volatility, WindowUDF, WindowUDFImpl,
};
use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
+use datafusion::physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
use datafusion::physical_plan::ExecutionPlan;
use datafusion_proto::logical_plan::{DefaultLogicalExtensionCodec, LogicalExtensionCodec};
-use datafusion_proto::physical_plan::{DefaultPhysicalExtensionCodec, PhysicalExtensionCodec};
+use datafusion_proto::physical_plan::{
+ DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalProtoConverterExtension,
+};
use pyo3::prelude::*;
use pyo3::sync::PyOnceLock;
use pyo3::types::{PyBytes, PyTuple};
@@ -483,12 +487,18 @@
buf: &[u8],
inputs: &[Arc<dyn ExecutionPlan>],
ctx: &TaskContext,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- self.inner.try_decode(buf, inputs, ctx)
+ self.inner.try_decode(buf, inputs, ctx, proto_converter)
}
- fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()> {
- self.inner.try_encode(node, buf)
+ fn try_encode(
+ &self,
+ node: Arc<dyn ExecutionPlan>,
+ buf: &mut Vec<u8>,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()> {
+ self.inner.try_encode(node, buf, proto_converter)
}
fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec<u8>) -> Result<()> {
@@ -509,16 +519,22 @@
self.inner.try_decode_udf(name, buf)
}
- fn try_encode_expr(&self, node: &Arc<dyn PhysicalExpr>, buf: &mut Vec<u8>) -> Result<()> {
- self.inner.try_encode_expr(node, buf)
+ fn try_encode_expr(
+ &self,
+ node: &Arc<dyn PhysicalExpr>,
+ buf: &mut Vec<u8>,
+ ctx: &PhysicalExprEncodeCtx<'_>,
+ ) -> Result<()> {
+ self.inner.try_encode_expr(node, buf, ctx)
}
fn try_decode_expr(
&self,
buf: &[u8],
inputs: &[Arc<dyn PhysicalExpr>],
+ ctx: &PhysicalExprDecodeCtx<'_>,
) -> Result<Arc<dyn PhysicalExpr>> {
- self.inner.try_decode_expr(buf, inputs)
+ self.inner.try_decode_expr(buf, inputs, ctx)
}
fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec<u8>) -> Result<()> {
diff --git a/crates/core/src/context.rs b/crates/core/src/context.rs
index 6184c7c..7bbeed2 100644
--- a/crates/core/src/context.rs
+++ b/crates/core/src/context.rs
@@ -1371,12 +1371,10 @@
&self,
py: Python<'py>,
) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_task_context_provider".into();
-
let ctx_provider = Arc::clone(&self.ctx) as Arc<dyn TaskContextProvider>;
let ffi_ctx_provider = FFI_TaskContextProvider::from(&ctx_provider);
- PyCapsule::new(py, ffi_ctx_provider, Some(name))
+ PyCapsule::new_with_value(py, ffi_ctx_provider, cr"datafusion_task_context_provider")
}
pub fn __datafusion_logical_extension_codec__<'py>(
diff --git a/crates/core/src/dataframe.rs b/crates/core/src/dataframe.rs
index 8f1a20d..b1f3055 100644
--- a/crates/core/src/dataframe.rs
+++ b/crates/core/src/dataframe.rs
@@ -16,7 +16,7 @@
// under the License.
use std::collections::HashMap;
-use std::ffi::{CStr, CString};
+use std::ffi::CStr;
use std::ptr::NonNull;
use std::str::FromStr;
use std::sync::Arc;
@@ -1237,8 +1237,7 @@
// destructor provided by PyO3 will drop the stream unless ownership is
// transferred to PyArrow during import.
let stream = FFI_ArrowArrayStream::new(reader);
- let name = CString::new(ARROW_ARRAY_STREAM_NAME.to_bytes()).unwrap();
- let capsule = PyCapsule::new(py, stream, Some(name))?;
+ let capsule = PyCapsule::new_with_value(py, stream, ARROW_ARRAY_STREAM_NAME)?;
Ok(capsule)
}
@@ -1317,7 +1316,8 @@
None => Vec::new(), // Empty vector means fill null for all columns
};
- let df = self.df.as_ref().clone().fill_null(scalar_value.0, cols)?;
+ let cols = cols.iter().map(String::as_str).collect::<Vec<_>>();
+ let df = self.df.as_ref().fill_null(&scalar_value.0, &cols)?;
Ok(Self::new(df))
}
}
diff --git a/crates/core/src/expr/create_external_table.rs b/crates/core/src/expr/create_external_table.rs
index 980eea1..e78b836 100644
--- a/crates/core/src/expr/create_external_table.rs
+++ b/crates/core/src/expr/create_external_table.rs
@@ -88,7 +88,7 @@
let create = CreateExternalTable {
schema: Arc::new(schema.into()),
name: name.into(),
- location,
+ locations: vec![location],
file_type,
table_partition_cols,
if_not_exists,
@@ -118,8 +118,8 @@
Ok(self.create.name.to_string())
}
- pub fn location(&self) -> String {
- self.create.location.clone()
+ pub fn locations(&self) -> Vec<String> {
+ self.create.locations.clone()
}
pub fn file_type(&self) -> String {
diff --git a/crates/core/src/expr/dml.rs b/crates/core/src/expr/dml.rs
index 26f9758..5967d18 100644
--- a/crates/core/src/expr/dml.rs
+++ b/crates/core/src/expr/dml.rs
@@ -18,6 +18,7 @@
use datafusion::logical_expr::dml::InsertOp;
use datafusion::logical_expr::{DmlStatement, WriteOp};
use pyo3::IntoPyObjectExt;
+use pyo3::exceptions::PyNotImplementedError;
use pyo3::prelude::*;
use super::logical_node::LogicalNode;
@@ -71,8 +72,8 @@
})
}
- pub fn op(&self) -> PyWriteOp {
- self.dml.op.clone().into()
+ pub fn op(&self) -> PyResult<PyWriteOp> {
+ self.dml.op.clone().try_into()
}
pub fn input(&self) -> PyLogicalPlan {
@@ -112,16 +113,21 @@
Truncate,
}
-impl From<WriteOp> for PyWriteOp {
- fn from(write_op: WriteOp) -> Self {
+impl TryFrom<WriteOp> for PyWriteOp {
+ type Error = PyErr;
+
+ fn try_from(write_op: WriteOp) -> Result<Self, Self::Error> {
match write_op {
- WriteOp::Insert(InsertOp::Append) => PyWriteOp::Append,
- WriteOp::Insert(InsertOp::Overwrite) => PyWriteOp::Overwrite,
- WriteOp::Insert(InsertOp::Replace) => PyWriteOp::Replace,
- WriteOp::Update => PyWriteOp::Update,
- WriteOp::Delete => PyWriteOp::Delete,
- WriteOp::Ctas => PyWriteOp::Ctas,
- WriteOp::Truncate => PyWriteOp::Truncate,
+ WriteOp::Insert(InsertOp::Append) => Ok(PyWriteOp::Append),
+ WriteOp::Insert(InsertOp::Overwrite) => Ok(PyWriteOp::Overwrite),
+ WriteOp::Insert(InsertOp::Replace) => Ok(PyWriteOp::Replace),
+ WriteOp::Update => Ok(PyWriteOp::Update),
+ WriteOp::Delete => Ok(PyWriteOp::Delete),
+ WriteOp::Ctas => Ok(PyWriteOp::Ctas),
+ WriteOp::Truncate => Ok(PyWriteOp::Truncate),
+ unsupported => Err(PyNotImplementedError::new_err(format!(
+ "DataFusion write operation {unsupported:?} is not supported"
+ ))),
}
}
}
diff --git a/crates/core/src/expr/drop_catalog_schema.rs b/crates/core/src/expr/drop_catalog_schema.rs
index fd51053..f349098 100644
--- a/crates/core/src/expr/drop_catalog_schema.rs
+++ b/crates/core/src/expr/drop_catalog_schema.rs
@@ -18,9 +18,8 @@
use std::fmt::{self, Display, Formatter};
use std::sync::Arc;
-use datafusion::common::SchemaReference;
+use datafusion::common::{SchemaReference, TableReference};
use datafusion::logical_expr::DropCatalogSchema;
-use datafusion::sql::TableReference;
use pyo3::IntoPyObjectExt;
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
diff --git a/crates/core/src/expr/recursive_query.rs b/crates/core/src/expr/recursive_query.rs
index e03137b..0b198a1 100644
--- a/crates/core/src/expr/recursive_query.rs
+++ b/crates/core/src/expr/recursive_query.rs
@@ -22,6 +22,7 @@
use pyo3::prelude::*;
use super::logical_node::LogicalNode;
+use crate::errors::PyDataFusionResult;
use crate::sql::logical::PyLogicalPlan;
#[pyclass(
@@ -67,15 +68,15 @@
static_term: PyLogicalPlan,
recursive_term: PyLogicalPlan,
is_distinct: bool,
- ) -> Self {
- Self {
- query: RecursiveQuery {
+ ) -> PyDataFusionResult<Self> {
+ Ok(Self {
+ query: RecursiveQuery::try_new(
name,
- static_term: static_term.plan(),
- recursive_term: recursive_term.plan(),
+ static_term.plan(),
+ recursive_term.plan(),
is_distinct,
- },
- }
+ )?,
+ })
}
fn name(&self) -> PyResult<String> {
diff --git a/crates/core/src/sql/logical.rs b/crates/core/src/sql/logical.rs
index 647c3fa..12fc43b 100644
--- a/crates/core/src/sql/logical.rs
+++ b/crates/core/src/sql/logical.rs
@@ -135,7 +135,7 @@
LogicalPlan::Dml(plan) => PyDmlStatement::from(plan.clone()).to_variant(py),
LogicalPlan::Ddl(plan) => match plan {
DdlStatement::CreateExternalTable(plan) => {
- PyCreateExternalTable::from(plan.clone()).to_variant(py)
+ PyCreateExternalTable::from(plan.as_ref().clone()).to_variant(py)
}
DdlStatement::CreateMemoryTable(plan) => {
PyCreateMemoryTable::from(plan.clone()).to_variant(py)
@@ -154,7 +154,7 @@
PyDropCatalogSchema::from(plan.clone()).to_variant(py)
}
DdlStatement::CreateFunction(plan) => {
- PyCreateFunction::from(plan.clone()).to_variant(py)
+ PyCreateFunction::from(plan.as_ref().clone()).to_variant(py)
}
DdlStatement::DropFunction(plan) => {
PyDropFunction::from(plan.clone()).to_variant(py)
diff --git a/crates/core/src/substrait.rs b/crates/core/src/substrait.rs
index 27e446f..dcb5587 100644
--- a/crates/core/src/substrait.rs
+++ b/crates/core/src/substrait.rs
@@ -109,7 +109,7 @@
) -> PyDataFusionResult<PyPlan> {
PySubstraitSerializer::serialize_bytes(sql, ctx, py).and_then(|proto_bytes| {
let proto_bytes = proto_bytes.bind(py).cast::<PyBytes>().unwrap();
- PySubstraitSerializer::deserialize_bytes(proto_bytes.as_bytes().to_vec(), py)
+ PySubstraitSerializer::deserialize_bytes(proto_bytes.as_bytes().to_vec())
})
}
@@ -131,8 +131,8 @@
}
#[staticmethod]
- pub fn deserialize_bytes(proto_bytes: Vec<u8>, py: Python) -> PyDataFusionResult<PyPlan> {
- let plan = wait_for_future(py, serializer::deserialize_bytes(proto_bytes))??;
+ pub fn deserialize_bytes(proto_bytes: Vec<u8>) -> PyDataFusionResult<PyPlan> {
+ let plan = serializer::deserialize_bytes(&proto_bytes)?;
Ok(PyPlan { plan: *plan })
}
}
diff --git a/crates/util/src/lib.rs b/crates/util/src/lib.rs
index 07aa0a2..9327d7f 100644
--- a/crates/util/src/lib.rs
+++ b/crates/util/src/lib.rs
@@ -164,7 +164,8 @@
)));
}
- let capsule_name = unsafe { capsule_name.unwrap().as_cstr().to_str()? };
+ let capsule_name = unsafe { capsule_name.unwrap().as_cstr().to_str() }
+ .map_err(|err| PyValueError::new_err(err.to_string()))?;
if capsule_name != name {
return Err(PyValueError::new_err(format!(
"Expected name '{name}' in PyCapsule, instead got '{capsule_name}'"
@@ -208,10 +209,9 @@
py: Python<'py>,
codec: &FFI_LogicalExtensionCodec,
) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_logical_extension_codec".into();
let codec = codec.clone();
- PyCapsule::new(py, codec, Some(name))
+ PyCapsule::new_with_value(py, codec, cr"datafusion_logical_extension_codec")
}
pub fn ffi_logical_codec_from_pycapsule(obj: Bound<PyAny>) -> PyResult<FFI_LogicalExtensionCodec> {
@@ -235,10 +235,9 @@
py: Python<'py>,
codec: &FFI_PhysicalExtensionCodec,
) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_physical_extension_codec".into();
let codec = codec.clone();
- PyCapsule::new(py, codec, Some(name))
+ PyCapsule::new_with_value(py, codec, cr"datafusion_physical_extension_codec")
}
/// Define a `<fn_name>(obj) -> PyResult<Arc<$output_type>>` extractor that
diff --git a/examples/datafusion-ffi-example/python/tests/_test_table_provider_factory.py b/examples/datafusion-ffi-example/python/tests/_test_table_provider_factory.py
index b1e94ec..0c0f1a5 100644
--- a/examples/datafusion-ffi-example/python/tests/_test_table_provider_factory.py
+++ b/examples/datafusion-ffi-example/python/tests/_test_table_provider_factory.py
@@ -32,7 +32,7 @@
CREATE EXTERNAL TABLE
foo
STORED AS my_format
- LOCATION '';
+ LOCATION 'unused';
""").collect()
# Query the pre-populated table
diff --git a/examples/datafusion-ffi-example/src/aggregate_udf.rs b/examples/datafusion-ffi-example/src/aggregate_udf.rs
index 8673777..ea15183 100644
--- a/examples/datafusion-ffi-example/src/aggregate_udf.rs
+++ b/examples/datafusion-ffi-example/src/aggregate_udf.rs
@@ -50,12 +50,10 @@
&self,
py: Python<'py>,
) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_aggregate_udf".into();
-
let func = Arc::new(AggregateUDF::from(self.clone()));
let provider = FFI_AggregateUDF::from(func);
- PyCapsule::new(py, provider, Some(name))
+ PyCapsule::new_with_value(py, provider, cr"datafusion_aggregate_udf")
}
}
diff --git a/examples/datafusion-ffi-example/src/catalog_provider.rs b/examples/datafusion-ffi-example/src/catalog_provider.rs
index 6131ab0..a56b585 100644
--- a/examples/datafusion-ffi-example/src/catalog_provider.rs
+++ b/examples/datafusion-ffi-example/src/catalog_provider.rs
@@ -33,8 +33,8 @@
use pyo3::{Bound, PyAny, PyResult, Python, pyclass, pymethods};
pub fn my_table() -> Arc<dyn TableProvider + 'static> {
+ use arrow::array::record_batch;
use arrow::datatypes::{DataType, Field};
- use datafusion_common::record_batch;
let schema = Arc::new(Schema::new(vec![
Field::new("units", DataType::Int32, true),
@@ -92,14 +92,12 @@
py: Python<'py>,
session: Bound<PyAny>,
) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_schema_provider".into();
-
let provider = Arc::clone(&self.inner) as Arc<dyn SchemaProvider + Send>;
let codec = ffi_logical_codec_from_pycapsule(session)?;
let provider = FFI_SchemaProvider::new_with_ffi_codec(provider, None, codec);
- PyCapsule::new(py, provider, Some(name))
+ PyCapsule::new_with_value(py, provider, cr"datafusion_schema_provider")
}
}
@@ -186,14 +184,12 @@
py: Python<'py>,
session: Bound<PyAny>,
) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_catalog_provider".into();
-
let provider = Arc::clone(&self.inner) as Arc<dyn CatalogProvider + Send>;
let codec = ffi_logical_codec_from_pycapsule(session)?;
let provider = FFI_CatalogProvider::new_with_ffi_codec(provider, None, codec);
- PyCapsule::new(py, provider, Some(name))
+ PyCapsule::new_with_value(py, provider, cr"datafusion_catalog_provider")
}
}
@@ -247,13 +243,11 @@
py: Python<'py>,
session: Bound<PyAny>,
) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_catalog_provider_list".into();
-
let provider = Arc::clone(&self.inner) as Arc<dyn CatalogProviderList + Send>;
let codec = ffi_logical_codec_from_pycapsule(session)?;
let provider = FFI_CatalogProviderList::new_with_ffi_codec(provider, None, codec);
- PyCapsule::new(py, provider, Some(name))
+ PyCapsule::new_with_value(py, provider, cr"datafusion_catalog_provider_list")
}
}
diff --git a/examples/datafusion-ffi-example/src/config.rs b/examples/datafusion-ffi-example/src/config.rs
index 6cdb8aa..9cf704b 100644
--- a/examples/datafusion-ffi-example/src/config.rs
+++ b/examples/datafusion-ffi-example/src/config.rs
@@ -53,14 +53,12 @@
&self,
py: Python<'py>,
) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_extension_options".into();
-
let mut config = FFI_ExtensionOptions::default();
config
.add_config(self)
.map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
- PyCapsule::new(py, config, Some(name))
+ PyCapsule::new_with_value(py, config, cr"datafusion_extension_options")
}
}
diff --git a/examples/datafusion-ffi-example/src/logical_extension_codec.rs b/examples/datafusion-ffi-example/src/logical_extension_codec.rs
index da9efb2..8c3976d 100644
--- a/examples/datafusion-ffi-example/src/logical_extension_codec.rs
+++ b/examples/datafusion-ffi-example/src/logical_extension_codec.rs
@@ -147,7 +147,6 @@
let ctx_provider = bare_session as Arc<dyn TaskContextProvider>;
let ffi = FFI_LogicalExtensionCodec::new(inner, Some(runtime), &ctx_provider);
- let name = cr"datafusion_logical_extension_codec".into();
- PyCapsule::new(py, ffi, Some(name))
+ PyCapsule::new_with_value(py, ffi, cr"datafusion_logical_extension_codec")
}
}
diff --git a/examples/datafusion-ffi-example/src/physical_extension_codec.rs b/examples/datafusion-ffi-example/src/physical_extension_codec.rs
index b1a586d..35ef77f 100644
--- a/examples/datafusion-ffi-example/src/physical_extension_codec.rs
+++ b/examples/datafusion-ffi-example/src/physical_extension_codec.rs
@@ -24,7 +24,9 @@
use datafusion::physical_plan::ExecutionPlan;
use datafusion::prelude::SessionContext;
use datafusion_ffi::proto::physical_extension_codec::FFI_PhysicalExtensionCodec;
-use datafusion_proto::physical_plan::{DefaultPhysicalExtensionCodec, PhysicalExtensionCodec};
+use datafusion_proto::physical_plan::{
+ DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalProtoConverterExtension,
+};
use datafusion_python_util::get_tokio_runtime;
use pyo3::prelude::*;
use pyo3::types::PyCapsule;
@@ -51,12 +53,18 @@
buf: &[u8],
inputs: &[Arc<dyn ExecutionPlan>],
ctx: &TaskContext,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- self.inner.try_decode(buf, inputs, ctx)
+ self.inner.try_decode(buf, inputs, ctx, proto_converter)
}
- fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()> {
- self.inner.try_encode(node, buf)
+ fn try_encode(
+ &self,
+ node: Arc<dyn ExecutionPlan>,
+ buf: &mut Vec<u8>,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()> {
+ self.inner.try_encode(node, buf, proto_converter)
}
fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result<Arc<ScalarUDF>> {
@@ -113,7 +121,6 @@
let ctx_provider = bare_session as Arc<dyn TaskContextProvider>;
let ffi = FFI_PhysicalExtensionCodec::new(inner, Some(runtime), &ctx_provider);
- let name = cr"datafusion_physical_extension_codec".into();
- PyCapsule::new(py, ffi, Some(name))
+ PyCapsule::new_with_value(py, ffi, cr"datafusion_physical_extension_codec")
}
}
diff --git a/examples/datafusion-ffi-example/src/physical_optimizer.rs b/examples/datafusion-ffi-example/src/physical_optimizer.rs
index 0acd1bb..e175104 100644
--- a/examples/datafusion-ffi-example/src/physical_optimizer.rs
+++ b/examples/datafusion-ffi-example/src/physical_optimizer.rs
@@ -92,7 +92,6 @@
let runtime = get_tokio_runtime().handle().clone();
let ffi = FFI_PhysicalOptimizerRule::new(rule, Some(runtime));
- let name = cr"datafusion_physical_optimizer_rule".into();
- PyCapsule::new(py, ffi, Some(name))
+ PyCapsule::new_with_value(py, ffi, cr"datafusion_physical_optimizer_rule")
}
}
diff --git a/examples/datafusion-ffi-example/src/scalar_udf.rs b/examples/datafusion-ffi-example/src/scalar_udf.rs
index a3c65e8..85b884e 100644
--- a/examples/datafusion-ffi-example/src/scalar_udf.rs
+++ b/examples/datafusion-ffi-example/src/scalar_udf.rs
@@ -50,12 +50,10 @@
}
fn __datafusion_scalar_udf__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_scalar_udf".into();
-
let func = Arc::new(ScalarUDF::from(self.clone()));
let provider = FFI_ScalarUDF::from(func);
- PyCapsule::new(py, provider, Some(name))
+ PyCapsule::new_with_value(py, provider, cr"datafusion_scalar_udf")
}
}
diff --git a/examples/datafusion-ffi-example/src/table_function.rs b/examples/datafusion-ffi-example/src/table_function.rs
index ed3ef14..55543cb 100644
--- a/examples/datafusion-ffi-example/src/table_function.rs
+++ b/examples/datafusion-ffi-example/src/table_function.rs
@@ -47,13 +47,11 @@
py: Python<'py>,
session: Bound<PyAny>,
) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_table_function".into();
-
let func = self.clone();
let codec = ffi_logical_codec_from_pycapsule(session)?;
let provider = FFI_TableFunction::new_with_ffi_codec(Arc::new(func), None, codec);
- PyCapsule::new(py, provider, Some(name))
+ PyCapsule::new_with_value(py, provider, cr"datafusion_table_function")
}
}
diff --git a/examples/datafusion-ffi-example/src/table_provider.rs b/examples/datafusion-ffi-example/src/table_provider.rs
index 358ef74..5756e6d 100644
--- a/examples/datafusion-ffi-example/src/table_provider.rs
+++ b/examples/datafusion-ffi-example/src/table_provider.rs
@@ -99,8 +99,6 @@
py: Python<'py>,
session: Bound<PyAny>,
) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_table_provider".into();
-
let provider = self
.create_table()
.map_err(|e: DataFusionError| PyRuntimeError::new_err(e.to_string()))?;
@@ -109,6 +107,6 @@
let provider =
FFI_TableProvider::new_with_ffi_codec(Arc::new(provider), false, None, codec);
- PyCapsule::new(py, provider, Some(name))
+ PyCapsule::new_with_value(py, provider, cr"datafusion_table_provider")
}
}
diff --git a/examples/datafusion-ffi-example/src/table_provider_factory.rs b/examples/datafusion-ffi-example/src/table_provider_factory.rs
index 53248a9..71dfd73 100644
--- a/examples/datafusion-ffi-example/src/table_provider_factory.rs
+++ b/examples/datafusion-ffi-example/src/table_provider_factory.rs
@@ -77,11 +77,10 @@
py: Python<'py>,
codec: Bound<PyAny>,
) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_table_provider_factory".into();
let codec = ffi_logical_codec_from_pycapsule(codec)?;
let factory = Arc::clone(&self.inner) as Arc<dyn TableProviderFactory + Send>;
let factory = FFI_TableProviderFactory::new_with_ffi_codec(factory, None, codec);
- PyCapsule::new(py, factory, Some(name))
+ PyCapsule::new_with_value(py, factory, cr"datafusion_table_provider_factory")
}
}
diff --git a/examples/datafusion-ffi-example/src/window_udf.rs b/examples/datafusion-ffi-example/src/window_udf.rs
index f33a166..2956ad6 100644
--- a/examples/datafusion-ffi-example/src/window_udf.rs
+++ b/examples/datafusion-ffi-example/src/window_udf.rs
@@ -45,12 +45,10 @@
}
fn __datafusion_window_udf__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyCapsule>> {
- let name = cr"datafusion_window_udf".into();
-
let func = Arc::new(WindowUDF::from(self.clone()));
let provider = FFI_WindowUDF::from(func);
- PyCapsule::new(py, provider, Some(name))
+ PyCapsule::new_with_value(py, provider, cr"datafusion_window_udf")
}
}
diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py
index a086cd3..c198b64 100644
--- a/python/datafusion/expr.py
+++ b/python/datafusion/expr.py
@@ -49,6 +49,11 @@
from collections.abc import Callable, Iterable, Sequence
from typing import TYPE_CHECKING, Any, ClassVar
+try:
+ from warnings import deprecated # Python 3.13+
+except ImportError:
+ from typing_extensions import deprecated # Python 3.12
+
import pyarrow as pa
from ._internal import expr as expr_internal
@@ -90,6 +95,27 @@
CreateCatalog = expr_internal.CreateCatalog
CreateCatalogSchema = expr_internal.CreateCatalogSchema
CreateExternalTable = expr_internal.CreateExternalTable
+
+
+@deprecated("CreateExternalTable.location() is deprecated; use locations() instead.")
+def _create_external_table_location(self: Any) -> str:
+ """Return the first external table location.
+
+ Use :meth:`CreateExternalTable.locations` instead.
+
+ Examples:
+ >>> class Command:
+ ... def locations(self) -> list[str]:
+ ... return ["data.csv"]
+ >>> Command().locations()
+ ['data.csv']
+ """
+ locations = self.locations()
+ return locations[0] if locations else ""
+
+
+CreateExternalTable.location = _create_external_table_location
+
CreateFunction = expr_internal.CreateFunction
CreateFunctionBody = expr_internal.CreateFunctionBody
CreateIndex = expr_internal.CreateIndex
diff --git a/python/datafusion/functions/spark.py b/python/datafusion/functions/spark.py
index 5163a67..0a0f414 100644
--- a/python/datafusion/functions/spark.py
+++ b/python/datafusion/functions/spark.py
@@ -273,12 +273,10 @@
Examples:
>>> ctx = dfn.SessionContext()
- >>> df = ctx.from_pydict({"x": [1]})
+ >>> df = ctx.from_pydict({"x": [[1, 2, 3, 4]]})
>>> r = df.select(
... dfn.functions.spark.slice(
- ... dfn.functions.spark.array(
- ... dfn.lit(1), dfn.lit(2), dfn.lit(3), dfn.lit(4)),
- ... 2, 2,
+ ... dfn.col("x"), 2, 2,
... ).alias("v")
... )
>>> r.collect_column("v")[0].as_py()
diff --git a/python/tests/test_catalog.py b/python/tests/test_catalog.py
index c89da36..97a8988 100644
--- a/python/tests/test_catalog.py
+++ b/python/tests/test_catalog.py
@@ -123,6 +123,11 @@
class CustomTableProviderFactory(dfn.catalog.TableProviderFactory):
def create(self, cmd: dfn.expr.CreateExternalTable):
assert cmd.name() == "test_table_factory"
+ assert cmd.locations() == ["foo"]
+
+ with pytest.warns(DeprecationWarning, match=r"location\(\).+deprecated"):
+ assert cmd.location() == "foo"
+
return create_dataset()
diff --git a/python/tests/test_context.py b/python/tests/test_context.py
index 2f068b0..7d038c7 100644
--- a/python/tests/test_context.py
+++ b/python/tests/test_context.py
@@ -17,6 +17,7 @@
import datetime as dt
import gzip
import pathlib
+import shutil
import pyarrow as pa
import pyarrow.dataset as ds
@@ -799,16 +800,15 @@
csv_df.select(column("c1")).show()
-def test_read_csv_list(ctx):
- csv_df = ctx.read_csv(path=["testing/data/csv/aggregate_test_100.csv"])
+def test_read_csv_list(ctx, tmp_path):
+ source_path = pathlib.Path("testing/data/csv/aggregate_test_100.csv")
+ copied_path = tmp_path / source_path.name
+ shutil.copy(source_path, copied_path)
+
+ csv_df = ctx.read_csv(path=[source_path])
expected = csv_df.count() * 2
- double_csv_df = ctx.read_csv(
- path=[
- "testing/data/csv/aggregate_test_100.csv",
- "testing/data/csv/aggregate_test_100.csv",
- ]
- )
+ double_csv_df = ctx.read_csv(path=[source_path, copied_path])
actual = double_csv_df.count()
double_csv_df.select(column("c1")).show()
diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py
index 08bec60..fabe6df 100644
--- a/python/tests/test_functions.py
+++ b/python/tests/test_functions.py
@@ -145,7 +145,7 @@
f.pow(col_v, literal(pa.scalar(4))),
f.round(col_v),
f.round(col_v, literal(pa.scalar(3))),
- f.sqrt(col_v),
+ f.sqrt(f.abs(col_v)),
f.signum(col_v),
f.trunc(col_v),
f.asinh(col_v),
@@ -189,7 +189,7 @@
np.testing.assert_array_almost_equal(result.column(16), np.power(values, 4))
np.testing.assert_array_almost_equal(result.column(17), np.round(values))
np.testing.assert_array_almost_equal(result.column(18), np.round(values, 3))
- np.testing.assert_array_almost_equal(result.column(19), np.sqrt(values))
+ np.testing.assert_array_almost_equal(result.column(19), np.sqrt(np.abs(values)))
np.testing.assert_array_almost_equal(result.column(20), np.sign(values))
np.testing.assert_array_almost_equal(result.column(21), np.trunc(values))
np.testing.assert_array_almost_equal(result.column(22), np.arcsinh(values))
@@ -215,6 +215,14 @@
)
+def test_sqrt_rejects_negative_input():
+ ctx = SessionContext()
+ df = ctx.from_pydict({"value": [-1.0]})
+
+ with pytest.raises(Exception, match="cannot take square root of a negative number"):
+ df.select(f.sqrt(column("value"))).collect()
+
+
def test_is_nan_alias():
ctx = SessionContext()
df = ctx.from_pydict({"value": [1.0, np.nan, None]})
diff --git a/python/tests/test_io.py b/python/tests/test_io.py
index 9f56f74..f0a9c3a 100644
--- a/python/tests/test_io.py
+++ b/python/tests/test_io.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import shutil
from pathlib import Path
import pyarrow as pa
@@ -72,16 +73,15 @@
csv_df.select(column("c1")).show()
-def test_read_csv_list():
- csv_df = read_csv(path=["testing/data/csv/aggregate_test_100.csv"])
+def test_read_csv_list(tmp_path):
+ source_path = Path("testing/data/csv/aggregate_test_100.csv")
+ copied_path = tmp_path / source_path.name
+ shutil.copy(source_path, copied_path)
+
+ csv_df = read_csv(path=[source_path])
expected = csv_df.count() * 2
- double_csv_df = read_csv(
- path=[
- "testing/data/csv/aggregate_test_100.csv",
- "testing/data/csv/aggregate_test_100.csv",
- ]
- )
+ double_csv_df = read_csv(path=[source_path, copied_path])
actual = double_csv_df.count()
double_csv_df.select(column("c1")).show()
diff --git a/python/tests/test_lambda.py b/python/tests/test_lambda.py
index 68be22a..ce37546 100644
--- a/python/tests/test_lambda.py
+++ b/python/tests/test_lambda.py
@@ -16,6 +16,8 @@
# under the License.
"""Tests for lambda expressions and higher-order array functions."""
+import pickle
+
import pytest
from datafusion import SessionConfig, SessionContext, col, lit
from datafusion import functions as f
@@ -137,8 +139,9 @@
assert result.to_pylist() == [[2, 4, 6]]
-def test_pickle_lambda_expr_not_supported():
- # v1 limitation: upstream proto serialization rejects lambda expressions.
+def test_pickle_lambda_expr_round_trip(df):
expr = f.array_transform(col("a"), lambda v: v * 2)
- with pytest.raises(Exception, match="Lambda not implemented"):
- expr.to_bytes()
+ decoded = pickle.loads(pickle.dumps(expr)) # noqa: S301
+
+ assert decoded.canonical_name() == expr.canonical_name()
+ assert _column(df, decoded, "r") == [[2, 4, 6], [8, 10]]
diff --git a/python/tests/test_spark_functions.py b/python/tests/test_spark_functions.py
index aed46d7..78b86c9 100644
--- a/python/tests/test_spark_functions.py
+++ b/python/tests/test_spark_functions.py
@@ -172,6 +172,14 @@
def test_slice(df):
+ assert _val(df, spark.slice(col("a"), lit(2), lit(2))) == [2, 3]
+
+
+@pytest.mark.xfail(
+ strict=True,
+ reason="Upstream array_slice does not preserve Spark array field names",
+)
+def test_slice_spark_array(df):
arr = spark.array(lit(1), lit(2), lit(3), lit(4))
assert _val(df, spark.slice(arr, lit(2), lit(2))) == [2, 3]
diff --git a/python/tests/test_sql.py b/python/tests/test_sql.py
index 6089c37..57d22bd 100644
--- a/python/tests/test_sql.py
+++ b/python/tests/test_sql.py
@@ -107,6 +107,7 @@
def test_register_csv_list(ctx, tmp_path):
path = tmp_path / "test.csv"
+ second_path = tmp_path / "test2.csv"
int_values = [1, 2, 3, 4]
table = pa.Table.from_arrays(
@@ -118,6 +119,7 @@
names=["int", "str", "float"],
)
write_csv(table, path)
+ write_csv(table, second_path)
ctx.register_csv("csv", path)
csv_df = ctx.table("csv")
@@ -126,7 +128,7 @@
"double_csv",
path=[
path,
- path,
+ second_path,
],
)
