chore: update datafusion to 51.0 (#1345)
* cherry pick Marko's work
* update code and test based on new DF apis
* fmt toml and bump rust min version
* update benchmark tests
* fmt
* Using task ctx directly
* update protos
* fmt
* remove session context from few more places
* remove unnecessary clone
---------
Co-authored-by: Marko Milenković <milenkovicm@users.noreply.github.com>
diff --git a/Cargo.lock b/Cargo.lock
index 0aed463..d9ed1e7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -24,9 +24,9 @@
[[package]]
name = "aho-corasick"
-version = "1.1.3"
+version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
dependencies = [
"memchr",
]
@@ -93,22 +93,22 @@
[[package]]
name = "anstyle-query"
-version = "1.1.4"
+version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
+checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
]
[[package]]
name = "anstyle-wincon"
-version = "3.0.10"
+version = "3.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"
+checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -138,8 +138,8 @@
"serde_bytes",
"serde_json",
"snap",
- "strum 0.27.2",
- "strum_macros 0.27.2",
+ "strum",
+ "strum_macros",
"thiserror 2.0.17",
"uuid",
"xz2",
@@ -147,6 +147,15 @@
]
[[package]]
+name = "ar_archive_writer"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a"
+dependencies = [
+ "object",
+]
+
+[[package]]
name = "arrayref"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -160,9 +169,9 @@
[[package]]
name = "arrow"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc"
+checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -181,23 +190,23 @@
[[package]]
name = "arrow-arith"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8"
+checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
- "num",
+ "num-traits",
]
[[package]]
name = "arrow-array"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d"
+checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31"
dependencies = [
"ahash",
"arrow-buffer",
@@ -206,26 +215,29 @@
"chrono",
"chrono-tz",
"half",
- "hashbrown 0.16.0",
- "num",
+ "hashbrown 0.16.1",
+ "num-complex",
+ "num-integer",
+ "num-traits",
]
[[package]]
name = "arrow-buffer"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc"
+checksum = "76885a2697a7edf6b59577f568b456afc94ce0e2edc15b784ce3685b6c3c5c27"
dependencies = [
"bytes",
"half",
- "num",
+ "num-bigint",
+ "num-traits",
]
[[package]]
name = "arrow-cast"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023"
+checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -238,15 +250,15 @@
"comfy-table",
"half",
"lexical-core",
- "num",
+ "num-traits",
"ryu",
]
[[package]]
name = "arrow-csv"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb"
+checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -259,21 +271,22 @@
[[package]]
name = "arrow-data"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0"
+checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b"
dependencies = [
"arrow-buffer",
"arrow-schema",
"half",
- "num",
+ "num-integer",
+ "num-traits",
]
[[package]]
name = "arrow-flight"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c8b0ba0784d56bc6266b79f5de7a24b47024e7b3a0045d2ad4df3d9b686099f"
+checksum = "f70bb56412a007b0cfc116d15f24dda6adeed9611a213852a004cda20085a3b9"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -294,13 +307,14 @@
"prost",
"prost-types",
"tonic",
+ "tonic-prost",
]
[[package]]
name = "arrow-ipc"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5"
+checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -314,9 +328,9 @@
[[package]]
name = "arrow-json"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b"
+checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -325,20 +339,22 @@
"arrow-schema",
"chrono",
"half",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
+ "itoa",
"lexical-core",
"memchr",
- "num",
- "serde",
+ "num-traits",
+ "ryu",
+ "serde_core",
"serde_json",
"simdutf8",
]
[[package]]
name = "arrow-ord"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f"
+checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -349,9 +365,9 @@
[[package]]
name = "arrow-row"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753"
+checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -362,33 +378,33 @@
[[package]]
name = "arrow-schema"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe"
+checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5"
dependencies = [
- "serde",
+ "serde_core",
"serde_json",
]
[[package]]
name = "arrow-select"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a"
+checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47"
dependencies = [
"ahash",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
- "num",
+ "num-traits",
]
[[package]]
name = "arrow-string"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d"
+checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -396,12 +412,28 @@
"arrow-schema",
"arrow-select",
"memchr",
- "num",
+ "num-traits",
"regex",
"regex-syntax",
]
[[package]]
+name = "astral-tokio-tar"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec179a06c1769b1e42e1e2cbe74c7dcdb3d6383c838454d063eaac5bbb7ebbe5"
+dependencies = [
+ "filetime",
+ "futures-core",
+ "libc",
+ "portable-atomic",
+ "rustc-hash",
+ "tokio",
+ "tokio-stream",
+ "xattr",
+]
+
+[[package]]
name = "async-compression"
version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -437,7 +469,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -448,7 +480,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -474,9 +506,9 @@
[[package]]
name = "aws-config"
-version = "1.8.8"
+version = "1.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37cf2b6af2a95a20e266782b4f76f1a5e12bf412a9db2de9c1e9123b9d8c0ad8"
+checksum = "a0149602eeaf915158e14029ba0c78dedb8c08d554b024d54c8f239aab46511d"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -504,9 +536,9 @@
[[package]]
name = "aws-credential-types"
-version = "1.2.8"
+version = "1.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "faf26925f4a5b59eb76722b63c2892b1d70d06fa053c72e4a100ec308c1d47bc"
+checksum = "b01c9521fa01558f750d183c8c68c81b0155b9d193a4ba7f84c36bd1b6d04a06"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
@@ -516,9 +548,9 @@
[[package]]
name = "aws-lc-rs"
-version = "1.14.1"
+version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "879b6c89592deb404ba4dc0ae6b58ffd1795c78991cbb5b8bc441c48a070440d"
+checksum = "5932a7d9d28b0d2ea34c6b3779d35e3dd6f6345317c34e73438c4f1f29144151"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -526,9 +558,9 @@
[[package]]
name = "aws-lc-sys"
-version = "0.32.3"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "107a4e9d9cab9963e04e84bb8dee0e25f2a987f9a8bad5ed054abd439caa8f8c"
+checksum = "1826f2e4cfc2cd19ee53c42fbf68e2f81ec21108e0b7ecf6a71cf062137360fc"
dependencies = [
"bindgen",
"cc",
@@ -539,9 +571,9 @@
[[package]]
name = "aws-runtime"
-version = "1.5.12"
+version = "1.5.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa006bb32360ed90ac51203feafb9d02e3d21046e1fd3a450a404b90ea73e5d"
+checksum = "7ce527fb7e53ba9626fc47824f25e256250556c40d8f81d27dd92aa38239d632"
dependencies = [
"aws-credential-types",
"aws-sigv4",
@@ -563,9 +595,9 @@
[[package]]
name = "aws-sdk-sso"
-version = "1.86.0"
+version = "1.90.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a0abbfab841446cce6e87af853a3ba2cc1bc9afcd3f3550dd556c43d434c86d"
+checksum = "4f18e53542c522459e757f81e274783a78f8c81acdfc8d1522ee8a18b5fb1c66"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -585,9 +617,9 @@
[[package]]
name = "aws-sdk-ssooidc"
-version = "1.89.0"
+version = "1.92.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "695dc67bb861ccb8426c9129b91c30e266a0e3d85650cafdf62fcca14c8fd338"
+checksum = "532f4d866012ffa724a4385c82e8dd0e59f0ca0e600f3f22d4c03b6824b34e4a"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -607,9 +639,9 @@
[[package]]
name = "aws-sdk-sts"
-version = "1.88.0"
+version = "1.94.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d30990923f4f675523c51eb1c0dec9b752fb267b36a61e83cbc219c9d86da715"
+checksum = "1be6fbbfa1a57724788853a623378223fe828fc4c09b146c992f0c95b6256174"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -630,9 +662,9 @@
[[package]]
name = "aws-sigv4"
-version = "1.3.5"
+version = "1.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bffc03068fbb9c8dd5ce1c6fb240678a5cffb86fb2b7b1985c999c4b83c8df68"
+checksum = "c35452ec3f001e1f2f6db107b6373f1f48f05ec63ba2c5c9fa91f07dad32af11"
dependencies = [
"aws-credential-types",
"aws-smithy-http",
@@ -663,15 +695,16 @@
[[package]]
name = "aws-smithy-http"
-version = "0.62.4"
+version = "0.62.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3feafd437c763db26aa04e0cc7591185d0961e64c61885bece0fb9d50ceac671"
+checksum = "445d5d720c99eed0b4aa674ed00d835d9b1427dd73e04adaf2f94c6b2d6f9fca"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
"bytes",
"bytes-utils",
"futures-core",
+ "futures-util",
"http 0.2.12",
"http 1.3.1",
"http-body 0.4.6",
@@ -683,9 +716,9 @@
[[package]]
name = "aws-smithy-http-client"
-version = "1.1.3"
+version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1053b5e587e6fa40ce5a79ea27957b04ba660baa02b28b7436f64850152234f1"
+checksum = "623254723e8dfd535f566ee7b2381645f8981da086b5c4aa26c0c41582bb1d2c"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
@@ -707,9 +740,9 @@
[[package]]
name = "aws-smithy-json"
-version = "0.61.6"
+version = "0.61.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cff418fc8ec5cadf8173b10125f05c2e7e1d46771406187b2c878557d4503390"
+checksum = "2db31f727935fc63c6eeae8b37b438847639ec330a9161ece694efba257e0c54"
dependencies = [
"aws-smithy-types",
]
@@ -735,9 +768,9 @@
[[package]]
name = "aws-smithy-runtime"
-version = "1.9.3"
+version = "1.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40ab99739082da5347660c556689256438defae3bcefd66c52b095905730e404"
+checksum = "0bbe9d018d646b96c7be063dd07987849862b0e6d07c778aad7d93d1be6c1ef0"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
@@ -759,9 +792,9 @@
[[package]]
name = "aws-smithy-runtime-api"
-version = "1.9.1"
+version = "1.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3683c5b152d2ad753607179ed71988e8cfd52964443b4f74fd8e552d0bbfeb46"
+checksum = "ec7204f9fd94749a7c53b26da1b961b4ac36bf070ef1e0b94bb09f79d4f6c193"
dependencies = [
"aws-smithy-async",
"aws-smithy-types",
@@ -776,9 +809,9 @@
[[package]]
name = "aws-smithy-types"
-version = "1.3.3"
+version = "1.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f5b3a7486f6690ba25952cabf1e7d75e34d69eaff5081904a47bc79074d6457"
+checksum = "25f535879a207fce0db74b679cfc3e91a3159c8144d717d55f5832aea9eef46e"
dependencies = [
"base64-simd",
"bytes",
@@ -799,18 +832,18 @@
[[package]]
name = "aws-smithy-xml"
-version = "0.60.11"
+version = "0.60.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9c34127e8c624bc2999f3b657e749c1393bedc9cd97b92a804db8ced4d2e163"
+checksum = "eab77cdd036b11056d2a30a7af7b775789fb024bf216acc13884c6c97752ae56"
dependencies = [
"xmlparser",
]
[[package]]
name = "aws-types"
-version = "1.3.9"
+version = "1.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2fd329bf0e901ff3f60425691410c69094dc2a1f34b331f37bfc4e9ac1565a1"
+checksum = "d79fb68e3d7fe5d4833ea34dc87d2e97d26d3086cb3da660bb6b1f76d98680b6"
dependencies = [
"aws-credential-types",
"aws-smithy-async",
@@ -822,9 +855,9 @@
[[package]]
name = "axum"
-version = "0.8.6"
+version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871"
+checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425"
dependencies = [
"axum-core",
"bytes",
@@ -916,7 +949,7 @@
version = "50.0.0"
dependencies = [
"ballista",
- "clap 4.5.50",
+ "clap 4.5.53",
"datafusion",
"datafusion-cli",
"dirs",
@@ -935,7 +968,7 @@
"aws-config",
"aws-credential-types",
"chrono",
- "clap 4.5.50",
+ "clap 4.5.53",
"datafusion",
"datafusion-proto",
"datafusion-proto-common",
@@ -955,6 +988,8 @@
"tokio-stream",
"tonic",
"tonic-build",
+ "tonic-prost",
+ "tonic-prost-build",
"url",
"uuid",
]
@@ -986,7 +1021,7 @@
"arrow-flight",
"async-trait",
"ballista-core",
- "clap 4.5.50",
+ "clap 4.5.53",
"dashmap",
"datafusion",
"datafusion-proto",
@@ -1014,7 +1049,7 @@
"async-trait",
"axum",
"ballista-core",
- "clap 4.5.50",
+ "clap 4.5.53",
"dashmap",
"datafusion",
"datafusion-proto",
@@ -1093,7 +1128,7 @@
"regex",
"rustc-hash",
"shlex",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -1141,9 +1176,9 @@
[[package]]
name = "bollard"
-version = "0.19.1"
+version = "0.19.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "899ca34eb6924d6ec2a77c6f7f5c7339e60fd68235eaf91edd5a15f12958bb06"
+checksum = "87a52479c9237eb04047ddb94788c41ca0d26eaff8b697ecfbb4c32f7fdc3b1b"
dependencies = [
"async-stream",
"base64 0.22.1",
@@ -1188,21 +1223,22 @@
[[package]]
name = "bollard-buildkit-proto"
-version = "0.6.1"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40b3e79f8bd0f25f32660e3402afca46fd91bebaf135af017326d905651f8107"
+checksum = "85a885520bf6249ab931a764ffdb87b0ceef48e6e7d807cfdb21b751e086e1ad"
dependencies = [
"prost",
"prost-types",
"tonic",
+ "tonic-prost",
"ureq",
]
[[package]]
name = "bollard-stubs"
-version = "1.48.3-rc.28.0.4"
+version = "1.49.1-rc.28.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64ea257e555d16a2c01e5593f40b73865cdf12efbceda33c6d14a2d8d1490368"
+checksum = "5731fe885755e92beff1950774068e0cae67ea6ec7587381536fca84f1779623"
dependencies = [
"base64 0.22.1",
"bollard-buildkit-proto",
@@ -1237,7 +1273,7 @@
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -1275,9 +1311,9 @@
[[package]]
name = "bytes"
-version = "1.10.1"
+version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
+checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3"
[[package]]
name = "bytes-utils"
@@ -1319,9 +1355,9 @@
[[package]]
name = "cc"
-version = "1.2.42"
+version = "1.2.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81bbf3b3619004ad9bd139f62a9ab5cfe467f307455a0d307b0cf58bf070feaa"
+checksum = "b97463e1064cb1b1c1384ad0a0b9c8abd0988e2a91f52606c80ef14aadb63e36"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -1396,9 +1432,9 @@
[[package]]
name = "clap"
-version = "4.5.50"
+version = "4.5.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c2cfd7bf8a6017ddaa4e32ffe7403d547790db06bd171c1c53926faab501623"
+checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1406,9 +1442,9 @@
[[package]]
name = "clap_builder"
-version = "4.5.50"
+version = "4.5.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a4c05b9e80c5ccd3a7ef080ad7b6ba7d6fc00a985b8b157197075677c82c7a0"
+checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00"
dependencies = [
"anstream",
"anstyle",
@@ -1425,7 +1461,7 @@
"heck 0.5.0",
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -1460,12 +1496,11 @@
[[package]]
name = "comfy-table"
-version = "7.1.2"
+version = "7.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56"
+checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b"
dependencies = [
- "strum 0.26.3",
- "strum_macros 0.26.4",
+ "unicode-segmentation",
"unicode-width 0.2.2",
]
@@ -1552,9 +1587,9 @@
[[package]]
name = "crypto-common"
-version = "0.1.6"
+version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"typenum",
@@ -1618,7 +1653,7 @@
"proc-macro2",
"quote",
"strsim",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -1629,7 +1664,7 @@
dependencies = [
"darling_core",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -1648,12 +1683,11 @@
[[package]]
name = "datafusion"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2af15bb3c6ffa33011ef579f6b0bcbe7c26584688bd6c994f548e44df67f011a"
+checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8"
dependencies = [
"arrow",
- "arrow-ipc",
"arrow-schema",
"async-trait",
"bytes",
@@ -1664,6 +1698,7 @@
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
+ "datafusion-datasource-arrow",
"datafusion-datasource-avro",
"datafusion-datasource-csv",
"datafusion-datasource-json",
@@ -1686,7 +1721,6 @@
"datafusion-sql",
"flate2",
"futures",
- "hex",
"itertools 0.14.0",
"log",
"object_store",
@@ -1694,6 +1728,7 @@
"parquet",
"rand",
"regex",
+ "rstest",
"sqlparser",
"tempfile",
"tokio",
@@ -1705,9 +1740,9 @@
[[package]]
name = "datafusion-catalog"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "187622262ad8f7d16d3be9202b4c1e0116f1c9aa387e5074245538b755261621"
+checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d"
dependencies = [
"arrow",
"async-trait",
@@ -1720,7 +1755,6 @@
"datafusion-physical-expr",
"datafusion-physical-plan",
"datafusion-session",
- "datafusion-sql",
"futures",
"itertools 0.14.0",
"log",
@@ -1731,9 +1765,9 @@
[[package]]
name = "datafusion-catalog-listing"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9657314f0a32efd0382b9a46fdeb2d233273ece64baa68a7c45f5a192daf0f83"
+checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0"
dependencies = [
"arrow",
"async-trait",
@@ -1743,10 +1777,11 @@
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-expr",
+ "datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
- "datafusion-session",
"futures",
+ "itertools 0.14.0",
"log",
"object_store",
"tokio",
@@ -1754,16 +1789,18 @@
[[package]]
name = "datafusion-cli"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a0b9c821d14e79070f42ea3a6d6618ced04d94277f0a32301918d7a022c250f"
+checksum = "fab982df44f818a749cb5200504ccb919f4608cb9808daf8b3fb98aa7955fd1e"
dependencies = [
"arrow",
"async-trait",
"aws-config",
"aws-credential-types",
- "clap 4.5.50",
+ "chrono",
+ "clap 4.5.53",
"datafusion",
+ "datafusion-common",
"dirs",
"env_logger",
"futures",
@@ -1780,20 +1817,19 @@
[[package]]
name = "datafusion-common"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a83760d9a13122d025fbdb1d5d5aaf93dd9ada5e90ea229add92aa30898b2d1"
+checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0"
dependencies = [
"ahash",
"apache-avro",
"arrow",
"arrow-ipc",
- "base64 0.22.1",
"chrono",
"half",
"hashbrown 0.14.5",
"hex",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
"libc",
"log",
"object_store",
@@ -1807,9 +1843,9 @@
[[package]]
name = "datafusion-common-runtime"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b6234a6c7173fe5db1c6c35c01a12b2aa0f803a3007feee53483218817f8b1e"
+checksum = "b92065bbc6532c6651e2f7dd30b55cba0c7a14f860c7e1d15f165c41a1868d95"
dependencies = [
"futures",
"log",
@@ -1818,9 +1854,9 @@
[[package]]
name = "datafusion-datasource"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7256c9cb27a78709dd42d0c80f0178494637209cac6e29d5c93edd09b6721b86"
+checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6"
dependencies = [
"arrow",
"async-compression",
@@ -1843,9 +1879,7 @@
"itertools 0.14.0",
"log",
"object_store",
- "parquet",
"rand",
- "tempfile",
"tokio",
"tokio-util",
"url",
@@ -1854,46 +1888,63 @@
]
[[package]]
-name = "datafusion-datasource-avro"
-version = "50.3.0"
+name = "datafusion-datasource-arrow"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10d40b6953ebc9099b37adfd12fde97eb73ff0cee44355c6dea64b8a4537d561"
+checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea"
+dependencies = [
+ "arrow",
+ "arrow-ipc",
+ "async-trait",
+ "bytes",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-session",
+ "futures",
+ "itertools 0.14.0",
+ "object_store",
+ "tokio",
+]
+
+[[package]]
+name = "datafusion-datasource-avro"
+version = "51.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388ed8be535f562cc655b9c3d22edbfb0f1a50a25c242647a98b6d92a75b55a1"
dependencies = [
"apache-avro",
"arrow",
"async-trait",
"bytes",
- "chrono",
- "datafusion-catalog",
"datafusion-common",
"datafusion-datasource",
- "datafusion-execution",
- "datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-session",
"futures",
"num-traits",
"object_store",
- "tokio",
]
[[package]]
name = "datafusion-datasource-csv"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64533a90f78e1684bfb113d200b540f18f268134622d7c96bbebc91354d04825"
+checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568"
dependencies = [
"arrow",
"async-trait",
"bytes",
- "datafusion-catalog",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
- "datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-session",
@@ -1905,74 +1956,67 @@
[[package]]
name = "datafusion-datasource-json"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d7ebeb12c77df0aacad26f21b0d033aeede423a64b2b352f53048a75bf1d6e6"
+checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85"
dependencies = [
"arrow",
"async-trait",
"bytes",
- "datafusion-catalog",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
- "datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-session",
"futures",
"object_store",
- "serde_json",
"tokio",
]
[[package]]
name = "datafusion-datasource-parquet"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09e783c4c7d7faa1199af2df4761c68530634521b176a8d1331ddbc5a5c75133"
+checksum = "43d0b60ffd66f28bfb026565d62b0a6cbc416da09814766a3797bba7d85a3cd9"
dependencies = [
"arrow",
"async-trait",
"bytes",
- "datafusion-catalog",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
- "datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
"datafusion-physical-expr",
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
- "datafusion-physical-optimizer",
"datafusion-physical-plan",
"datafusion-pruning",
"datafusion-session",
"futures",
- "hex",
"itertools 0.14.0",
"log",
"object_store",
"parking_lot",
"parquet",
- "rand",
"tokio",
]
[[package]]
name = "datafusion-doc"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99ee6b1d9a80d13f9deb2291f45c07044b8e62fb540dbde2453a18be17a36429"
+checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07"
[[package]]
name = "datafusion-execution"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4cec0a57653bec7b933fb248d3ffa3fa3ab3bd33bd140dc917f714ac036f531"
+checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178"
dependencies = [
"arrow",
"async-trait",
@@ -1991,9 +2035,9 @@
[[package]]
name = "datafusion-expr"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef76910bdca909722586389156d0aa4da4020e1631994d50fadd8ad4b1aa05fe"
+checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91"
dependencies = [
"arrow",
"async-trait",
@@ -2004,7 +2048,8 @@
"datafusion-functions-aggregate-common",
"datafusion-functions-window-common",
"datafusion-physical-expr-common",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
+ "itertools 0.14.0",
"paste",
"recursive",
"serde_json",
@@ -2013,22 +2058,22 @@
[[package]]
name = "datafusion-expr-common"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d155ccbda29591ca71a1344dd6bed26c65a4438072b400df9db59447f590bb6"
+checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc"
dependencies = [
"arrow",
"datafusion-common",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
"itertools 0.14.0",
"paste",
]
[[package]]
name = "datafusion-functions"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7de2782136bd6014670fd84fe3b0ca3b3e4106c96403c3ae05c0598577139977"
+checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f"
dependencies = [
"arrow",
"arrow-buffer",
@@ -2046,6 +2091,7 @@
"itertools 0.14.0",
"log",
"md-5",
+ "num-traits",
"rand",
"regex",
"sha2",
@@ -2055,9 +2101,9 @@
[[package]]
name = "datafusion-functions-aggregate"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07331fc13603a9da97b74fd8a273f4238222943dffdbbed1c4c6f862a30105bf"
+checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305"
dependencies = [
"ahash",
"arrow",
@@ -2076,9 +2122,9 @@
[[package]]
name = "datafusion-functions-aggregate-common"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5951e572a8610b89968a09b5420515a121fbc305c0258651f318dc07c97ab17"
+checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d"
dependencies = [
"ahash",
"arrow",
@@ -2089,9 +2135,9 @@
[[package]]
name = "datafusion-functions-nested"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdacca9302c3d8fc03f3e94f338767e786a88a33f5ebad6ffc0e7b50364b9ea3"
+checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4"
dependencies = [
"arrow",
"arrow-ord",
@@ -2099,6 +2145,7 @@
"datafusion-doc",
"datafusion-execution",
"datafusion-expr",
+ "datafusion-expr-common",
"datafusion-functions",
"datafusion-functions-aggregate",
"datafusion-functions-aggregate-common",
@@ -2111,9 +2158,9 @@
[[package]]
name = "datafusion-functions-table"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c37ff8a99434fbbad604a7e0669717c58c7c4f14c472d45067c4b016621d981"
+checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5"
dependencies = [
"arrow",
"async-trait",
@@ -2127,9 +2174,9 @@
[[package]]
name = "datafusion-functions-window"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48e2aea7c79c926cffabb13dc27309d4eaeb130f4a21c8ba91cdd241c813652b"
+checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1"
dependencies = [
"arrow",
"datafusion-common",
@@ -2145,9 +2192,9 @@
[[package]]
name = "datafusion-functions-window-common"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fead257ab5fd2ffc3b40fda64da307e20de0040fe43d49197241d9de82a487f"
+checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -2155,20 +2202,20 @@
[[package]]
name = "datafusion-macros"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec6f637bce95efac05cdfb9b6c19579ed4aa5f6b94d951cfa5bb054b7bb4f730"
+checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848"
dependencies = [
- "datafusion-expr",
+ "datafusion-doc",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
name = "datafusion-optimizer"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6583ef666ae000a613a837e69e456681a9faa96347bf3877661e9e89e141d8a"
+checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a"
dependencies = [
"arrow",
"chrono",
@@ -2176,7 +2223,7 @@
"datafusion-expr",
"datafusion-expr-common",
"datafusion-physical-expr",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
"itertools 0.14.0",
"log",
"recursive",
@@ -2186,9 +2233,9 @@
[[package]]
name = "datafusion-physical-expr"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8668103361a272cbbe3a61f72eca60c9b7c706e87cc3565bcf21e2b277b84f6"
+checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d"
dependencies = [
"ahash",
"arrow",
@@ -2199,9 +2246,8 @@
"datafusion-physical-expr-common",
"half",
"hashbrown 0.14.5",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
"itertools 0.14.0",
- "log",
"parking_lot",
"paste",
"petgraph 0.8.3",
@@ -2209,9 +2255,9 @@
[[package]]
name = "datafusion-physical-expr-adapter"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "815acced725d30601b397e39958e0e55630e0a10d66ef7769c14ae6597298bb0"
+checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768"
dependencies = [
"arrow",
"datafusion-common",
@@ -2224,9 +2270,9 @@
[[package]]
name = "datafusion-physical-expr-common"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6652fe7b5bf87e85ed175f571745305565da2c0b599d98e697bcbedc7baa47c3"
+checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03"
dependencies = [
"ahash",
"arrow",
@@ -2238,9 +2284,9 @@
[[package]]
name = "datafusion-physical-optimizer"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49b7d623eb6162a3332b564a0907ba00895c505d101b99af78345f1acf929b5c"
+checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6"
dependencies = [
"arrow",
"datafusion-common",
@@ -2252,15 +2298,14 @@
"datafusion-physical-plan",
"datafusion-pruning",
"itertools 0.14.0",
- "log",
"recursive",
]
[[package]]
name = "datafusion-physical-plan"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2f7f778a1a838dec124efb96eae6144237d546945587557c9e6936b3414558c"
+checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac"
dependencies = [
"ahash",
"arrow",
@@ -2279,7 +2324,7 @@
"futures",
"half",
"hashbrown 0.14.5",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
"itertools 0.14.0",
"log",
"parking_lot",
@@ -2289,15 +2334,26 @@
[[package]]
name = "datafusion-proto"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7df9f606892e6af45763d94d210634eec69b9bb6ced5353381682ff090028a3"
+checksum = "d368093a98a17d1449b1083ac22ed16b7128e4c67789991869480d8c4a40ecb9"
dependencies = [
"arrow",
"chrono",
- "datafusion",
+ "datafusion-catalog",
+ "datafusion-catalog-listing",
"datafusion-common",
+ "datafusion-datasource",
+ "datafusion-datasource-arrow",
+ "datafusion-datasource-csv",
+ "datafusion-datasource-json",
+ "datafusion-datasource-parquet",
+ "datafusion-execution",
"datafusion-expr",
+ "datafusion-functions-table",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
"datafusion-proto-common",
"object_store",
"prost",
@@ -2305,9 +2361,9 @@
[[package]]
name = "datafusion-proto-common"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4b14f288ca4ef77743d9672cafecf3adfffff0b9b04af9af79ecbeaaf736901"
+checksum = "3b6aef3d5e5c1d2bc3114c4876730cb76a9bdc5a8df31ef1b6db48f0c1671895"
dependencies = [
"arrow",
"datafusion-common",
@@ -2316,12 +2372,11 @@
[[package]]
name = "datafusion-pruning"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd1e59e2ca14fe3c30f141600b10ad8815e2856caa59ebbd0e3e07cd3d127a65"
+checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db"
dependencies = [
"arrow",
- "arrow-schema",
"datafusion-common",
"datafusion-datasource",
"datafusion-expr-common",
@@ -2334,39 +2389,30 @@
[[package]]
name = "datafusion-session"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21ef8e2745583619bd7a49474e8f45fbe98ebb31a133f27802217125a7b3d58d"
+checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776"
dependencies = [
- "arrow",
"async-trait",
- "dashmap",
"datafusion-common",
- "datafusion-common-runtime",
"datafusion-execution",
"datafusion-expr",
- "datafusion-physical-expr",
"datafusion-physical-plan",
- "datafusion-sql",
- "futures",
- "itertools 0.14.0",
- "log",
- "object_store",
"parking_lot",
- "tokio",
]
[[package]]
name = "datafusion-sql"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89abd9868770386fede29e5a4b14f49c0bf48d652c3b9d7a8a0332329b87d50b"
+checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f"
dependencies = [
"arrow",
"bigdecimal",
+ "chrono",
"datafusion-common",
"datafusion-expr",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
"log",
"recursive",
"regex",
@@ -2375,9 +2421,9 @@
[[package]]
name = "deranged"
-version = "0.5.4"
+version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071"
+checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587"
dependencies = [
"powerfmt",
"serde_core",
@@ -2423,7 +2469,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -2454,9 +2500,9 @@
[[package]]
name = "dtor"
-version = "0.1.0"
+version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e58a0764cddb55ab28955347b45be00ade43d4d6f3ba4bf3dc354e4ec9432934"
+checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301"
dependencies = [
"dtor-proc-macro",
]
@@ -2578,9 +2624,9 @@
[[package]]
name = "find-msvc-tools"
-version = "0.1.4"
+version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127"
+checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844"
[[package]]
name = "fixedbitset"
@@ -2692,7 +2738,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -2733,9 +2779,9 @@
[[package]]
name = "generic-array"
-version = "0.14.9"
+version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
@@ -2802,7 +2848,7 @@
"futures-core",
"futures-sink",
"http 1.3.1",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
"slab",
"tokio",
"tokio-util",
@@ -2848,9 +2894,9 @@
[[package]]
name = "hashbrown"
-version = "0.16.0"
+version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d"
+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]]
name = "heck"
@@ -2967,9 +3013,9 @@
[[package]]
name = "hyper"
-version = "1.7.0"
+version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e"
+checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
dependencies = [
"atomic-waker",
"bytes",
@@ -3035,9 +3081,9 @@
[[package]]
name = "hyper-util"
-version = "0.1.17"
+version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8"
+checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -3051,7 +3097,7 @@
"libc",
"percent-encoding",
"pin-project-lite",
- "socket2 0.6.1",
+ "socket2",
"tokio",
"tower-service",
"tracing",
@@ -3098,9 +3144,9 @@
[[package]]
name = "icu_collections"
-version = "2.0.0"
+version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47"
+checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43"
dependencies = [
"displaydoc",
"potential_utf",
@@ -3111,9 +3157,9 @@
[[package]]
name = "icu_locale_core"
-version = "2.0.0"
+version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a"
+checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6"
dependencies = [
"displaydoc",
"litemap",
@@ -3124,11 +3170,10 @@
[[package]]
name = "icu_normalizer"
-version = "2.0.0"
+version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979"
+checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599"
dependencies = [
- "displaydoc",
"icu_collections",
"icu_normalizer_data",
"icu_properties",
@@ -3139,42 +3184,38 @@
[[package]]
name = "icu_normalizer_data"
-version = "2.0.0"
+version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3"
+checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
[[package]]
name = "icu_properties"
-version = "2.0.1"
+version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b"
+checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99"
dependencies = [
- "displaydoc",
"icu_collections",
"icu_locale_core",
"icu_properties_data",
"icu_provider",
- "potential_utf",
"zerotrie",
"zerovec",
]
[[package]]
name = "icu_properties_data"
-version = "2.0.1"
+version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632"
+checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899"
[[package]]
name = "icu_provider"
-version = "2.0.0"
+version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af"
+checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614"
dependencies = [
"displaydoc",
"icu_locale_core",
- "stable_deref_trait",
- "tinystr",
"writeable",
"yoke",
"zerofrom",
@@ -3222,12 +3263,12 @@
[[package]]
name = "indexmap"
-version = "2.12.0"
+version = "2.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f"
+checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2"
dependencies = [
"equivalent",
- "hashbrown 0.16.0",
+ "hashbrown 0.16.1",
"serde",
"serde_core",
]
@@ -3268,9 +3309,9 @@
[[package]]
name = "iri-string"
-version = "0.7.8"
+version = "0.7.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2"
+checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397"
dependencies = [
"memchr",
"serde",
@@ -3308,26 +3349,26 @@
[[package]]
name = "jiff"
-version = "0.2.15"
+version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49"
+checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35"
dependencies = [
"jiff-static",
"log",
"portable-atomic",
"portable-atomic-util",
- "serde",
+ "serde_core",
]
[[package]]
name = "jiff-static"
-version = "0.2.15"
+version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4"
+checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -3342,9 +3383,9 @@
[[package]]
name = "js-sys"
-version = "0.3.81"
+version = "0.3.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305"
+checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65"
dependencies = [
"once_cell",
"wasm-bindgen",
@@ -3459,7 +3500,7 @@
dependencies = [
"bitflags 2.10.0",
"libc",
- "redox_syscall 0.5.18",
+ "redox_syscall",
]
[[package]]
@@ -3485,9 +3526,9 @@
[[package]]
name = "litemap"
-version = "0.8.0"
+version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
+checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
[[package]]
name = "lock_api"
@@ -3731,6 +3772,15 @@
]
[[package]]
+name = "object"
+version = "0.32.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
name = "object_store"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3824,16 +3874,16 @@
dependencies = [
"cfg-if",
"libc",
- "redox_syscall 0.5.18",
+ "redox_syscall",
"smallvec",
"windows-link",
]
[[package]]
name = "parquet"
-version = "56.2.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27"
+checksum = "7a0f31027ef1af7549f7cec603a9a21dce706d3f8d7c2060a68f43c1773be95a"
dependencies = [
"ahash",
"arrow-array",
@@ -3850,10 +3900,11 @@
"flate2",
"futures",
"half",
- "hashbrown 0.16.0",
+ "hashbrown 0.16.1",
"lz4_flex",
- "num",
"num-bigint",
+ "num-integer",
+ "num-traits",
"object_store",
"paste",
"ring",
@@ -3888,7 +3939,7 @@
"regex",
"regex-syntax",
"structmeta",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -3933,7 +3984,7 @@
"pest_meta",
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -3953,7 +4004,7 @@
checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772"
dependencies = [
"fixedbitset",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
]
[[package]]
@@ -3964,7 +4015,7 @@
dependencies = [
"fixedbitset",
"hashbrown 0.15.5",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
"serde",
]
@@ -4003,7 +4054,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -4041,9 +4092,9 @@
[[package]]
name = "potential_utf"
-version = "0.1.3"
+version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a"
+checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77"
dependencies = [
"zerovec",
]
@@ -4070,7 +4121,7 @@
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -4156,9 +4207,9 @@
[[package]]
name = "prost"
-version = "0.13.5"
+version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"
+checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d"
dependencies = [
"bytes",
"prost-derive",
@@ -4166,9 +4217,9 @@
[[package]]
name = "prost-build"
-version = "0.13.5"
+version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
+checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1"
dependencies = [
"heck 0.5.0",
"itertools 0.14.0",
@@ -4179,29 +4230,31 @@
"prettyplease",
"prost",
"prost-types",
+ "pulldown-cmark",
+ "pulldown-cmark-to-cmark",
"regex",
- "syn 2.0.108",
+ "syn 2.0.110",
"tempfile",
]
[[package]]
name = "prost-derive"
-version = "0.13.5"
+version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
+checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"
dependencies = [
"anyhow",
"itertools 0.14.0",
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
name = "prost-types"
-version = "0.13.5"
+version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16"
+checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72"
dependencies = [
"prost",
]
@@ -4228,14 +4281,35 @@
[[package]]
name = "psm"
-version = "0.1.27"
+version = "0.1.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e66fcd288453b748497d8fb18bccc83a16b0518e3906d4b8df0a8d42d93dbb1c"
+checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01"
dependencies = [
+ "ar_archive_writer",
"cc",
]
[[package]]
+name = "pulldown-cmark"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0"
+dependencies = [
+ "bitflags 2.10.0",
+ "memchr",
+ "unicase",
+]
+
+[[package]]
+name = "pulldown-cmark-to-cmark"
+version = "21.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8246feae3db61428fd0bb94285c690b460e4517d83152377543ca802357785f1"
+dependencies = [
+ "pulldown-cmark",
+]
+
+[[package]]
name = "quad-rand"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4243,9 +4317,9 @@
[[package]]
name = "quick-xml"
-version = "0.38.3"
+version = "0.38.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89"
+checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
dependencies = [
"memchr",
"serde",
@@ -4264,7 +4338,7 @@
"quinn-udp",
"rustc-hash",
"rustls",
- "socket2 0.6.1",
+ "socket2",
"thiserror 2.0.17",
"tokio",
"tracing",
@@ -4301,16 +4375,16 @@
"cfg_aliases",
"libc",
"once_cell",
- "socket2 0.6.1",
+ "socket2",
"tracing",
"windows-sys 0.60.2",
]
[[package]]
name = "quote"
-version = "1.0.41"
+version = "1.0.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1"
+checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f"
dependencies = [
"proc-macro2",
]
@@ -4377,16 +4451,7 @@
checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b"
dependencies = [
"quote",
- "syn 2.0.108",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
-dependencies = [
- "bitflags 1.3.2",
+ "syn 2.0.110",
]
[[package]]
@@ -4426,7 +4491,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -4551,7 +4616,7 @@
"regex",
"relative-path",
"rustc_version",
- "syn 2.0.108",
+ "syn 2.0.110",
"unicode-ident",
]
@@ -4598,9 +4663,9 @@
[[package]]
name = "rustls"
-version = "0.23.34"
+version = "0.23.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a9586e9ee2b4f8fab52a0048ca7334d7024eef48e2cb9407e3497bb7cab7fa7"
+checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f"
dependencies = [
"aws-lc-rs",
"log",
@@ -4635,9 +4700,9 @@
[[package]]
name = "rustls-pki-types"
-version = "1.12.0"
+version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79"
+checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a"
dependencies = [
"web-time",
"zeroize",
@@ -4645,9 +4710,9 @@
[[package]]
name = "rustls-webpki"
-version = "0.103.7"
+version = "0.103.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10b3f4191e8a80e6b43eebabfac91e5dcecebb27a71f04e820c47ec41d314bf"
+checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52"
dependencies = [
"aws-lc-rs",
"ring",
@@ -4721,9 +4786,9 @@
[[package]]
name = "schemars"
-version = "1.0.4"
+version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0"
+checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289"
dependencies = [
"dyn-clone",
"ref-cast",
@@ -4809,7 +4874,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -4844,7 +4909,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -4861,17 +4926,17 @@
[[package]]
name = "serde_with"
-version = "3.15.1"
+version = "3.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa66c845eee442168b2c8134fec70ac50dc20e760769c8ba0ad1319ca1959b04"
+checksum = "10574371d41b0d9b2cff89418eda27da52bcaff2cc8741db26382a77c29131f1"
dependencies = [
"base64 0.22.1",
"chrono",
"hex",
"indexmap 1.9.3",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
"schemars 0.9.0",
- "schemars 1.0.4",
+ "schemars 1.1.0",
"serde_core",
"serde_json",
"serde_with_macros",
@@ -4880,14 +4945,14 @@
[[package]]
name = "serde_with_macros"
-version = "3.15.1"
+version = "3.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b91a903660542fced4e99881aa481bdbaec1634568ee02e0b8bd57c64cb38955"
+checksum = "08a72d8216842fdd57820dc78d840bef99248e35fb2554ff923319e60f2d686b"
dependencies = [
"darling",
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -4981,16 +5046,6 @@
[[package]]
name = "socket2"
-version = "0.5.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
-dependencies = [
- "libc",
- "windows-sys 0.52.0",
-]
-
-[[package]]
-name = "socket2"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881"
@@ -5001,9 +5056,9 @@
[[package]]
name = "sqlparser"
-version = "0.58.0"
+version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c"
+checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f"
dependencies = [
"log",
"recursive",
@@ -5018,7 +5073,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -5055,7 +5110,7 @@
"proc-macro2",
"quote",
"structmeta-derive",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -5066,7 +5121,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -5095,31 +5150,12 @@
[[package]]
name = "strum"
-version = "0.26.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
-
-[[package]]
-name = "strum"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
[[package]]
name = "strum_macros"
-version = "0.26.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
-dependencies = [
- "heck 0.5.0",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 2.0.108",
-]
-
-[[package]]
-name = "strum_macros"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
@@ -5127,7 +5163,7 @@
"heck 0.5.0",
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -5149,9 +5185,9 @@
[[package]]
name = "syn"
-version = "2.0.108"
+version = "2.0.110"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917"
+checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea"
dependencies = [
"proc-macro2",
"quote",
@@ -5175,7 +5211,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -5193,13 +5229,13 @@
[[package]]
name = "testcontainers"
-version = "0.25.0"
+version = "0.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b92bce247dc9260a19808321e11b51ea6a0293d02b48ab1c6578960610cfa2a7"
+checksum = "3f3ac71069f20ecfa60c396316c283fbf35e6833a53dff551a31b5458da05edc"
dependencies = [
+ "astral-tokio-tar",
"async-trait",
"bollard",
- "bollard-stubs",
"bytes",
"docker_credential",
"either",
@@ -5215,7 +5251,6 @@
"thiserror 2.0.17",
"tokio",
"tokio-stream",
- "tokio-tar",
"tokio-util",
"ulid",
"url",
@@ -5265,7 +5300,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -5276,7 +5311,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -5341,9 +5376,9 @@
[[package]]
name = "tinystr"
-version = "0.8.1"
+version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b"
+checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869"
dependencies = [
"displaydoc",
"zerovec",
@@ -5376,7 +5411,7 @@
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
- "socket2 0.6.1",
+ "socket2",
"tokio-macros",
"windows-sys 0.61.2",
]
@@ -5389,7 +5424,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -5414,25 +5449,10 @@
]
[[package]]
-name = "tokio-tar"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75"
-dependencies = [
- "filetime",
- "futures-core",
- "libc",
- "redox_syscall 0.3.5",
- "tokio",
- "tokio-stream",
- "xattr",
-]
-
-[[package]]
name = "tokio-util"
-version = "0.7.16"
+version = "0.7.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5"
+checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594"
dependencies = [
"bytes",
"futures-core",
@@ -5456,7 +5476,7 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d"
dependencies = [
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
"toml_datetime",
"toml_parser",
"winnow",
@@ -5473,9 +5493,9 @@
[[package]]
name = "tonic"
-version = "0.13.1"
+version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9"
+checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203"
dependencies = [
"async-trait",
"axum",
@@ -5490,8 +5510,8 @@
"hyper-util",
"percent-encoding",
"pin-project",
- "prost",
- "socket2 0.5.10",
+ "socket2",
+ "sync_wrapper",
"tokio",
"tokio-stream",
"tower",
@@ -5502,16 +5522,41 @@
[[package]]
name = "tonic-build"
-version = "0.13.1"
+version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847"
+checksum = "4c40aaccc9f9eccf2cd82ebc111adc13030d23e887244bc9cfa5d1d636049de3"
+dependencies = [
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.110",
+]
+
+[[package]]
+name = "tonic-prost"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67"
+dependencies = [
+ "bytes",
+ "prost",
+ "tonic",
+]
+
+[[package]]
+name = "tonic-prost-build"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4a16cba4043dc3ff43fcb3f96b4c5c154c64cbd18ca8dce2ab2c6a451d058a2"
dependencies = [
"prettyplease",
"proc-macro2",
"prost-build",
"prost-types",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
+ "tempfile",
+ "tonic-build",
]
[[package]]
@@ -5522,7 +5567,7 @@
dependencies = [
"futures-core",
"futures-util",
- "indexmap 2.12.0",
+ "indexmap 2.12.1",
"pin-project-lite",
"slab",
"sync_wrapper",
@@ -5595,7 +5640,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -5672,10 +5717,16 @@
]
[[package]]
-name = "unicode-ident"
-version = "1.0.20"
+name = "unicase"
+version = "2.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06"
+checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"
[[package]]
name = "unicode-segmentation"
@@ -5703,17 +5754,30 @@
[[package]]
name = "ureq"
-version = "2.12.1"
+version = "3.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d"
+checksum = "d39cb1dbab692d82a977c0392ffac19e188bd9186a9f32806f0aaa859d75585a"
dependencies = [
"base64 0.22.1",
"log",
- "once_cell",
+ "percent-encoding",
"rustls",
"rustls-pki-types",
- "url",
- "webpki-roots 0.26.11",
+ "ureq-proto",
+ "utf-8",
+ "webpki-roots",
+]
+
+[[package]]
+name = "ureq-proto"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60b4531c118335662134346048ddb0e54cc86bd7e81866757873055f0e38f5d2"
+dependencies = [
+ "base64 0.22.1",
+ "http 1.3.1",
+ "httparse",
+ "log",
]
[[package]]
@@ -5735,6 +5799,12 @@
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
+name = "utf-8"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+
+[[package]]
name = "utf8_iter"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -5812,9 +5882,9 @@
[[package]]
name = "wasm-bindgen"
-version = "0.2.104"
+version = "0.2.105"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d"
+checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60"
dependencies = [
"cfg-if",
"once_cell",
@@ -5824,24 +5894,10 @@
]
[[package]]
-name = "wasm-bindgen-backend"
-version = "0.2.104"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19"
-dependencies = [
- "bumpalo",
- "log",
- "proc-macro2",
- "quote",
- "syn 2.0.108",
- "wasm-bindgen-shared",
-]
-
-[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.54"
+version = "0.4.55"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e038d41e478cc73bae0ff9b36c60cff1c98b8f38f8d7e8061e79ee63608ac5c"
+checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0"
dependencies = [
"cfg-if",
"js-sys",
@@ -5852,9 +5908,9 @@
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.104"
+version = "0.2.105"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119"
+checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -5862,22 +5918,22 @@
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.104"
+version = "0.2.105"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7"
+checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc"
dependencies = [
+ "bumpalo",
"proc-macro2",
"quote",
- "syn 2.0.108",
- "wasm-bindgen-backend",
+ "syn 2.0.110",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.104"
+version = "0.2.105"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1"
+checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76"
dependencies = [
"unicode-ident",
]
@@ -5897,9 +5953,9 @@
[[package]]
name = "web-sys"
-version = "0.3.81"
+version = "0.3.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120"
+checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -5917,18 +5973,9 @@
[[package]]
name = "webpki-roots"
-version = "0.26.11"
+version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
-dependencies = [
- "webpki-roots 1.0.3",
-]
-
-[[package]]
-name = "webpki-roots"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32b130c0d2d49f8b6889abc456e795e82525204f27c42cf767cf0d7734e089b8"
+checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e"
dependencies = [
"rustls-pki-types",
]
@@ -5985,7 +6032,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -5996,7 +6043,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -6205,9 +6252,9 @@
[[package]]
name = "writeable"
-version = "0.6.1"
+version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb"
+checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
[[package]]
name = "xattr"
@@ -6236,11 +6283,10 @@
[[package]]
name = "yoke"
-version = "0.8.0"
+version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc"
+checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954"
dependencies = [
- "serde",
"stable_deref_trait",
"yoke-derive",
"zerofrom",
@@ -6248,34 +6294,34 @@
[[package]]
name = "yoke-derive"
-version = "0.8.0"
+version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6"
+checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
"synstructure",
]
[[package]]
name = "zerocopy"
-version = "0.8.27"
+version = "0.8.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c"
+checksum = "43fa6694ed34d6e57407afbccdeecfa268c470a7d2a5b0cf49ce9fcc345afb90"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
-version = "0.8.27"
+version = "0.8.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831"
+checksum = "c640b22cd9817fae95be82f0d2f90b11f7605f6c319d16705c459b27ac2cbc26"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
@@ -6295,7 +6341,7 @@
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
"synstructure",
]
@@ -6307,9 +6353,9 @@
[[package]]
name = "zerotrie"
-version = "0.2.2"
+version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595"
+checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851"
dependencies = [
"displaydoc",
"yoke",
@@ -6318,9 +6364,9 @@
[[package]]
name = "zerovec"
-version = "0.11.4"
+version = "0.11.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b"
+checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002"
dependencies = [
"yoke",
"zerofrom",
@@ -6329,13 +6375,13 @@
[[package]]
name = "zerovec-derive"
-version = "0.11.1"
+version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f"
+checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.108",
+ "syn 2.0.110",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 2e8bbcd..e9dc469 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,27 +27,28 @@
#
edition = "2021"
# we should try to follow datafusion version
-rust-version = "1.86.0"
+rust-version = "1.88.0"
[workspace.dependencies]
-arrow = { version = "56", features = ["ipc_compression"] }
-arrow-flight = { version = "56", features = ["flight-sql-experimental"] }
+arrow = { version = "57", features = ["ipc_compression"] }
+arrow-flight = { version = "57", features = ["flight-sql-experimental"] }
clap = { version = "4.5", features = ["derive", "cargo"] }
configure_me = { version = "0.4.0" }
configure_me_codegen = { version = "0.4.4" }
-datafusion = "50.3.0"
-datafusion-cli = "50.3.0"
-datafusion-proto = "50.3.0"
-datafusion-proto-common = "50.3.0"
+datafusion = "51.0.0"
+datafusion-cli = "51.0.0"
+datafusion-proto = "51.0.0"
+datafusion-proto-common = "51.0.0"
object_store = "0.12"
-prost = "0.13"
-prost-types = "0.13"
+prost = "0.14"
+prost-types = "0.14"
rstest = { version = "0.26" }
-tonic = { version = "0.13" }
-tonic-build = { version = "0.13", default-features = false, features = [
+tonic = { version = "0.14" }
+tonic-build = { version = "0.14", default-features = false, features = [
"transport",
- "prost"
] }
+tonic-prost = { version = "0.14" }
+tonic-prost-build = { version = "0.14" }
tracing = "0.1"
tracing-appender = "0.2.2"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
diff --git a/ballista-cli/src/main.rs b/ballista-cli/src/main.rs
index 0b55e6b..098d23a 100644
--- a/ballista-cli/src/main.rs
+++ b/ballista-cli/src/main.rs
@@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-use std::env;
use std::path::Path;
+use std::{env, sync::Arc};
use ballista::{extension::SessionConfigExt, prelude::SessionContextExt};
use ballista_cli::{
@@ -28,7 +28,9 @@
execution::SessionStateBuilder,
prelude::{SessionConfig, SessionContext},
};
-use datafusion_cli::print_options::MaxRows;
+use datafusion_cli::{
+ object_storage::instrumented::InstrumentedObjectStoreRegistry, print_options::MaxRows,
+};
use mimalloc::MiMalloc;
#[global_allocator]
@@ -151,6 +153,7 @@
quiet: args.quiet,
maxrows: MaxRows::Unlimited,
color: args.color,
+ instrumented_registry: Arc::new(InstrumentedObjectStoreRegistry::new()),
};
let files = args.file;
diff --git a/ballista/client/tests/context_basic.rs b/ballista/client/tests/context_basic.rs
index 5c137ee..45b0107 100644
--- a/ballista/client/tests/context_basic.rs
+++ b/ballista/client/tests/context_basic.rs
@@ -113,12 +113,14 @@
let result = df.unwrap().collect().await.unwrap();
- let expected = ["+---------------+--------------+---------------------+-------------+-----------------------------+-------------+",
- "| table_catalog | table_schema | table_name | column_name | data_type | is_nullable |",
- "+---------------+--------------+---------------------+-------------+-----------------------------+-------------+",
- "| datafusion | public | csv_with_timestamps | name | Utf8 | YES |",
- "| datafusion | public | csv_with_timestamps | ts | Timestamp(Nanosecond, None) | YES |",
- "+---------------+--------------+---------------------+-------------+-----------------------------+-------------+"];
+ let expected = [
+ "+---------------+--------------+---------------------+-------------+---------------+-------------+",
+ "| table_catalog | table_schema | table_name | column_name | data_type | is_nullable |",
+ "+---------------+--------------+---------------------+-------------+---------------+-------------+",
+ "| datafusion | public | csv_with_timestamps | name | Utf8 | YES |",
+ "| datafusion | public | csv_with_timestamps | ts | Timestamp(ns) | YES |",
+ "+---------------+--------------+---------------------+-------------+---------------+-------------+",
+ ];
datafusion::assert_batches_eq!(expected, &result);
}
@@ -359,7 +361,7 @@
let res = df.collect().await.unwrap();
let expected = vec![
"+-------------------+",
- "| var(test.id) |",
+ "| var_samp(test.id) |",
"+-------------------+",
"| 6.000000000000001 |",
"+-------------------+",
@@ -390,11 +392,11 @@
.unwrap();
let res = df.collect().await.unwrap();
let expected = vec![
- "+--------------------+",
- "| stddev(test.id) |",
- "+--------------------+",
- "| 2.4494897427831783 |",
- "+--------------------+",
+ "+----------------------+",
+ "| stddev_samp(test.id) |",
+ "+----------------------+",
+ "| 2.4494897427831783 |",
+ "+----------------------+",
];
assert_result_eq(expected, &res);
}
@@ -408,11 +410,11 @@
.unwrap();
let res = df.collect().await.unwrap();
let expected = vec![
- "+--------------------------------------+",
- "| covar_samp(test.id,test.tinyint_col) |",
- "+--------------------------------------+",
- "| 0.28571428571428586 |",
- "+--------------------------------------+",
+ "+---------------------------------+",
+ "| covar(test.id,test.tinyint_col) |",
+ "+---------------------------------+",
+ "| 0.28571428571428586 |",
+ "+---------------------------------+",
];
assert_result_eq(expected, &res);
}
diff --git a/ballista/client/tests/context_setup.rs b/ballista/client/tests/context_setup.rs
index 7806d84..e1a49bb 100644
--- a/ballista/client/tests/context_setup.rs
+++ b/ballista/client/tests/context_setup.rs
@@ -110,7 +110,9 @@
use datafusion::{
assert_batches_eq,
common::exec_err,
- execution::{context::QueryPlanner, SessionState, SessionStateBuilder},
+ execution::{
+ context::QueryPlanner, SessionState, SessionStateBuilder, TaskContext,
+ },
logical_expr::LogicalPlan,
physical_plan::ExecutionPlan,
prelude::{SessionConfig, SessionContext},
@@ -275,7 +277,7 @@
&self,
_buf: &[u8],
_inputs: &[datafusion::logical_expr::LogicalPlan],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> datafusion::error::Result<datafusion::logical_expr::Extension> {
self.invoked
.store(true, std::sync::atomic::Ordering::Relaxed);
@@ -297,7 +299,7 @@
_buf: &[u8],
_table_ref: &datafusion::sql::TableReference,
_schema: datafusion::arrow::datatypes::SchemaRef,
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> datafusion::error::Result<
std::sync::Arc<dyn datafusion::catalog::TableProvider>,
> {
@@ -320,7 +322,7 @@
fn try_decode_file_format(
&self,
_buf: &[u8],
- _ctx: &SessionContext,
+ _ctx: &TaskContext,
) -> datafusion::error::Result<
Arc<dyn datafusion::datasource::file_format::FileFormatFactory>,
> {
@@ -361,12 +363,12 @@
&self,
buf: &[u8],
inputs: &[Arc<dyn datafusion::physical_plan::ExecutionPlan>],
- registry: &dyn datafusion::execution::FunctionRegistry,
+ ctx: &TaskContext,
) -> datafusion::error::Result<Arc<dyn datafusion::physical_plan::ExecutionPlan>>
{
self.invoked
.store(true, std::sync::atomic::Ordering::Relaxed);
- self.codec.try_decode(buf, inputs, registry)
+ self.codec.try_decode(buf, inputs, ctx)
}
fn try_encode(
diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml
index 78b1b78..420b9d8 100644
--- a/ballista/core/Cargo.toml
+++ b/ballista/core/Cargo.toml
@@ -63,6 +63,7 @@
tokio = { workspace = true }
tokio-stream = { workspace = true, features = ["net"] }
tonic = { workspace = true }
+tonic-prost = { workspace = true }
url = { workspace = true }
uuid = { workspace = true }
@@ -72,3 +73,4 @@
[build-dependencies]
rustc_version = "0.4.1"
tonic-build = { workspace = true }
+tonic-prost-build = { workspace = true }
diff --git a/ballista/core/build.rs b/ballista/core/build.rs
index 4fe7e3b..21c6ad2 100644
--- a/ballista/core/build.rs
+++ b/ballista/core/build.rs
@@ -40,7 +40,7 @@
println!("cargo:rerun-if-changed=proto/datafusion_common.proto");
println!("cargo:rerun-if-changed=proto/datafusion.proto");
println!("cargo:rerun-if-changed=proto/ballista.proto");
- tonic_build::configure()
+ tonic_prost_build::configure()
.extern_path(".datafusion_common", "::datafusion_proto_common")
.extern_path(".datafusion", "::datafusion_proto::protobuf")
.protoc_arg("--experimental_allow_proto3_optional")
diff --git a/ballista/core/proto/ballista.proto b/ballista/core/proto/ballista.proto
index 2c1ce3d..48e9487 100644
--- a/ballista/core/proto/ballista.proto
+++ b/ballista/core/proto/ballista.proto
@@ -264,6 +264,18 @@
uint64 value = 2;
}
+message NamedPruningMetrics {
+ string name = 1;
+ uint64 pruned = 2;
+ uint64 matched = 3;
+}
+
+message NamedRatio {
+ string name = 1;
+ uint64 part = 2;
+ uint64 total = 3;
+}
+
message OperatorMetric {
oneof metric {
uint64 output_rows = 1;
@@ -277,6 +289,9 @@
int64 start_timestamp = 9;
int64 end_timestamp = 10;
uint64 spilled_rows = 11;
+ uint64 output_bytes = 12;
+ NamedPruningMetrics pruning_metrics = 13;
+ NamedRatio ratio = 14;
}
}
diff --git a/ballista/core/proto/datafusion.proto b/ballista/core/proto/datafusion.proto
index fc2752f..2d28777 100644
--- a/ballista/core/proto/datafusion.proto
+++ b/ballista/core/proto/datafusion.proto
@@ -167,6 +167,7 @@
datafusion_common.DfSchema schema = 4;
repeated string table_partition_cols = 5;
bool if_not_exists = 6;
+ bool or_replace = 15;
bool temporary = 14;
string definition = 7;
repeated SortExprNodeCollection order_exprs = 10;
@@ -180,6 +181,7 @@
string name = 1;
repeated datafusion_common.ArrowType data_types = 2;
LogicalPlanNode input = 3;
+ repeated datafusion_common.Field fields = 4;
}
message CreateCatalogSchemaNode {
@@ -412,6 +414,8 @@
message PlaceholderNode {
string id = 1;
datafusion_common.ArrowType data_type = 2;
+ optional bool nullable = 3;
+ map<string, string> metadata = 4;
}
message LogicalExprList {
@@ -517,6 +521,7 @@
LogicalExprNode filter = 3;
repeated SortExprNode order_by = 4;
optional bytes fun_definition = 6;
+ optional NullTreatment null_treatment = 7;
}
message ScalarUDFExprNode {
@@ -537,6 +542,9 @@
// repeated LogicalExprNode filter = 7;
WindowFrame window_frame = 8;
optional bytes fun_definition = 10;
+ optional NullTreatment null_treatment = 11;
+ bool distinct = 12;
+ LogicalExprNode filter = 13;
}
message BetweenNode {
@@ -621,6 +629,11 @@
datafusion_common.ScalarValue bound_value = 2;
}
+enum NullTreatment {
+ RESPECT_NULLS = 0;
+ IGNORE_NULLS = 1;
+}
+
///////////////////////////////////////////////////////////////////////////////////////////////////
// Arrow Data Types
///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -1032,6 +1045,7 @@
string comment = 6;
}
bool newlines_in_values = 7;
+ bool truncate_rows = 8;
}
message JsonScanExecNode {
diff --git a/ballista/core/proto/datafusion_common.proto b/ballista/core/proto/datafusion_common.proto
index 6f05dbf..68da590 100644
--- a/ballista/core/proto/datafusion_common.proto
+++ b/ballista/core/proto/datafusion_common.proto
@@ -460,6 +460,7 @@
bytes double_quote = 15; // Indicates if quotes are doubled
bytes newlines_in_values = 16; // Indicates if newlines are supported in values
bytes terminator = 17; // Optional terminator character as a byte
+ bytes truncated_rows = 18; // Indicates if truncated rows are allowed
}
// Options controlling CSV format
@@ -580,6 +581,10 @@
oneof coerce_int96_opt {
string coerce_int96 = 32;
}
+
+ oneof max_predicate_cache_size_opt {
+ uint64 max_predicate_cache_size = 33;
+ }
}
enum JoinSide {
diff --git a/ballista/core/src/execution_plans/distributed_query.rs b/ballista/core/src/execution_plans/distributed_query.rs
index aae084a..4336b1f 100644
--- a/ballista/core/src/execution_plans/distributed_query.rs
+++ b/ballista/core/src/execution_plans/distributed_query.rs
@@ -83,7 +83,8 @@
plan: LogicalPlan,
session_id: String,
) -> Self {
- let properties = Self::compute_properties(plan.schema().as_ref().clone().into());
+ let properties =
+ Self::compute_properties(plan.schema().as_arrow().clone().into());
Self {
scheduler_url,
config,
@@ -103,7 +104,8 @@
extension_codec: Arc<dyn LogicalExtensionCodec>,
session_id: String,
) -> Self {
- let properties = Self::compute_properties(plan.schema().as_ref().clone().into());
+ let properties =
+ Self::compute_properties(plan.schema().as_arrow().clone().into());
Self {
scheduler_url,
config,
@@ -157,7 +159,7 @@
}
fn schema(&self) -> SchemaRef {
- self.plan.schema().as_ref().clone().into()
+ self.plan.schema().as_arrow().clone().into()
}
fn properties(&self) -> &PlanProperties {
@@ -180,7 +182,7 @@
plan_repr: self.plan_repr,
session_id: self.session_id.clone(),
properties: Self::compute_properties(
- self.plan.schema().as_ref().clone().into(),
+ self.plan.schema().as_arrow().clone().into(),
),
metrics: ExecutionPlanMetricsSet::new(),
}))
diff --git a/ballista/core/src/serde/generated/ballista.rs b/ballista/core/src/serde/generated/ballista.rs
index 27cb4b8..f3a77a5 100644
--- a/ballista/core/src/serde/generated/ballista.rs
+++ b/ballista/core/src/serde/generated/ballista.rs
@@ -95,7 +95,7 @@
#[prost(uint64, tag = "13")]
pub queued_at: u64,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct StageAttempts {
#[prost(uint32, tag = "1")]
pub stage_id: u32,
@@ -247,7 +247,7 @@
#[prost(message, repeated, tag = "2")]
pub partition_location: ::prost::alloc::vec::Vec<PartitionLocation>,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct KeyValuePair {
#[prost(string, tag = "1")]
pub key: ::prost::alloc::string::String,
@@ -264,7 +264,7 @@
}
/// Nested message and enum types in `Action`.
pub mod action {
- #[derive(Clone, PartialEq, ::prost::Oneof)]
+ #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum ActionType {
/// Fetch a partition from an executor
#[prost(message, tag = "3")]
@@ -290,7 +290,7 @@
::datafusion_proto::protobuf::PhysicalHashRepartition,
>,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct FetchPartition {
#[prost(string, tag = "1")]
pub job_id: ::prost::alloc::string::String,
@@ -321,7 +321,7 @@
pub path: ::prost::alloc::string::String,
}
/// Unique identifier for a materialized partition of data
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct PartitionId {
#[prost(string, tag = "1")]
pub job_id: ::prost::alloc::string::String,
@@ -330,7 +330,7 @@
#[prost(uint32, tag = "4")]
pub partition_id: u32,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct TaskId {
#[prost(uint32, tag = "1")]
pub task_id: u32,
@@ -366,38 +366,56 @@
#[prost(message, repeated, tag = "1")]
pub metrics: ::prost::alloc::vec::Vec<OperatorMetric>,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NamedCount {
#[prost(string, tag = "1")]
pub name: ::prost::alloc::string::String,
#[prost(uint64, tag = "2")]
pub value: u64,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NamedGauge {
#[prost(string, tag = "1")]
pub name: ::prost::alloc::string::String,
#[prost(uint64, tag = "2")]
pub value: u64,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct NamedTime {
#[prost(string, tag = "1")]
pub name: ::prost::alloc::string::String,
#[prost(uint64, tag = "2")]
pub value: u64,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct NamedPruningMetrics {
+ #[prost(string, tag = "1")]
+ pub name: ::prost::alloc::string::String,
+ #[prost(uint64, tag = "2")]
+ pub pruned: u64,
+ #[prost(uint64, tag = "3")]
+ pub matched: u64,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct NamedRatio {
+ #[prost(string, tag = "1")]
+ pub name: ::prost::alloc::string::String,
+ #[prost(uint64, tag = "2")]
+ pub part: u64,
+ #[prost(uint64, tag = "3")]
+ pub total: u64,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct OperatorMetric {
#[prost(
oneof = "operator_metric::Metric",
- tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11"
+ tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14"
)]
pub metric: ::core::option::Option<operator_metric::Metric>,
}
/// Nested message and enum types in `OperatorMetric`.
pub mod operator_metric {
- #[derive(Clone, PartialEq, ::prost::Oneof)]
+ #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Metric {
#[prost(uint64, tag = "1")]
OutputRows(u64),
@@ -421,6 +439,12 @@
EndTimestamp(i64),
#[prost(uint64, tag = "11")]
SpilledRows(u64),
+ #[prost(uint64, tag = "12")]
+ OutputBytes(u64),
+ #[prost(message, tag = "13")]
+ PruningMetrics(super::NamedPruningMetrics),
+ #[prost(message, tag = "14")]
+ Ratio(super::NamedRatio),
}
}
/// Used by scheduler
@@ -464,7 +488,7 @@
#[prost(message, optional, tag = "4")]
pub status: ::core::option::Option<ExecutorStatus>,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ExecutorMetric {
/// TODO add more metrics
#[prost(oneof = "executor_metric::Metric", tags = "1")]
@@ -473,20 +497,20 @@
/// Nested message and enum types in `ExecutorMetric`.
pub mod executor_metric {
/// TODO add more metrics
- #[derive(Clone, Copy, PartialEq, ::prost::Oneof)]
+ #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Metric {
#[prost(uint64, tag = "1")]
AvailableMemory(u64),
}
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ExecutorStatus {
#[prost(oneof = "executor_status::Status", tags = "1, 2, 3, 4")]
pub status: ::core::option::Option<executor_status::Status>,
}
/// Nested message and enum types in `ExecutorStatus`.
pub mod executor_status {
- #[derive(Clone, PartialEq, ::prost::Oneof)]
+ #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Status {
#[prost(string, tag = "1")]
Active(::prost::alloc::string::String),
@@ -503,7 +527,7 @@
#[prost(message, repeated, tag = "1")]
pub resources: ::prost::alloc::vec::Vec<ExecutorResource>,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ExecutorResource {
/// TODO add more resources
#[prost(oneof = "executor_resource::Resource", tags = "1")]
@@ -512,13 +536,13 @@
/// Nested message and enum types in `ExecutorResource`.
pub mod executor_resource {
/// TODO add more resources
- #[derive(Clone, Copy, PartialEq, ::prost::Oneof)]
+ #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Resource {
#[prost(uint32, tag = "1")]
TaskSlots(u32),
}
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct AvailableTaskSlots {
#[prost(string, tag = "1")]
pub executor_id: ::prost::alloc::string::String,
@@ -537,19 +561,19 @@
#[prost(message, repeated, tag = "2")]
pub resources: ::prost::alloc::vec::Vec<ExecutorResourcePair>,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ExecutorResourcePair {
#[prost(message, optional, tag = "1")]
pub total: ::core::option::Option<ExecutorResource>,
#[prost(message, optional, tag = "2")]
pub available: ::core::option::Option<ExecutorResource>,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RunningTask {
#[prost(string, tag = "1")]
pub executor_id: ::prost::alloc::string::String,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct FailedTask {
#[prost(string, tag = "1")]
pub error: ::prost::alloc::string::String,
@@ -563,7 +587,7 @@
}
/// Nested message and enum types in `FailedTask`.
pub mod failed_task {
- #[derive(Clone, PartialEq, ::prost::Oneof)]
+ #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum FailedReason {
#[prost(message, tag = "4")]
ExecutionError(super::ExecutionError),
@@ -589,9 +613,9 @@
#[prost(message, repeated, tag = "2")]
pub partitions: ::prost::alloc::vec::Vec<ShuffleWritePartition>,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ExecutionError {}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct FetchPartitionError {
#[prost(string, tag = "1")]
pub executor_id: ::prost::alloc::string::String,
@@ -600,15 +624,15 @@
#[prost(uint32, tag = "3")]
pub map_partition_id: u32,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct IoError {}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ExecutorLost {}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ResultLost {}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct TaskKilled {}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ShuffleWritePartition {
#[prost(uint64, tag = "1")]
pub partition_id: u64,
@@ -728,7 +752,7 @@
#[prost(message, optional, tag = "1")]
pub metadata: ::core::option::Option<ExecutorRegistration>,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RegisterExecutorResult {
#[prost(bool, tag = "1")]
pub success: bool,
@@ -744,13 +768,13 @@
#[prost(message, optional, tag = "4")]
pub metadata: ::core::option::Option<ExecutorRegistration>,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct HeartBeatResult {
/// TODO it's from Spark for BlockManager
#[prost(bool, tag = "1")]
pub reregister: bool,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct StopExecutorParams {
#[prost(string, tag = "1")]
pub executor_id: ::prost::alloc::string::String,
@@ -761,9 +785,9 @@
#[prost(bool, tag = "3")]
pub force: bool,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct StopExecutorResult {}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ExecutorStoppedParams {
#[prost(string, tag = "1")]
pub executor_id: ::prost::alloc::string::String,
@@ -771,7 +795,7 @@
#[prost(string, tag = "2")]
pub reason: ::prost::alloc::string::String,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ExecutorStoppedResult {}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct UpdateTaskStatusParams {
@@ -781,7 +805,7 @@
#[prost(message, repeated, tag = "2")]
pub task_status: ::prost::alloc::vec::Vec<TaskStatus>,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct UpdateTaskStatusResult {
#[prost(bool, tag = "1")]
pub success: bool,
@@ -802,7 +826,7 @@
}
/// Nested message and enum types in `ExecuteQueryParams`.
pub mod execute_query_params {
- #[derive(Clone, PartialEq, ::prost::Oneof)]
+ #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Query {
#[prost(bytes, tag = "1")]
LogicalPlan(::prost::alloc::vec::Vec<u8>),
@@ -818,7 +842,7 @@
#[prost(message, repeated, tag = "1")]
pub settings: ::prost::alloc::vec::Vec<KeyValuePair>,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct CreateUpdateSessionResult {
#[prost(string, tag = "1")]
pub session_id: ::prost::alloc::string::String,
@@ -830,27 +854,27 @@
#[prost(message, repeated, tag = "2")]
pub settings: ::prost::alloc::vec::Vec<KeyValuePair>,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct UpdateSessionResult {
#[prost(bool, tag = "1")]
pub success: bool,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RemoveSessionParams {
#[prost(string, tag = "1")]
pub session_id: ::prost::alloc::string::String,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RemoveSessionResult {
#[prost(bool, tag = "1")]
pub success: bool,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ExecuteSqlParams {
#[prost(string, tag = "1")]
pub sql: ::prost::alloc::string::String,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ExecuteQueryResult {
#[prost(string, tag = "3")]
pub operation_id: ::prost::alloc::string::String,
@@ -859,7 +883,7 @@
}
/// Nested message and enum types in `ExecuteQueryResult`.
pub mod execute_query_result {
- #[derive(Clone, PartialEq, ::prost::Oneof)]
+ #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Result {
#[prost(message, tag = "1")]
Success(super::ExecuteQuerySuccessResult),
@@ -867,21 +891,21 @@
Failure(super::ExecuteQueryFailureResult),
}
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ExecuteQuerySuccessResult {
#[prost(string, tag = "1")]
pub job_id: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub session_id: ::prost::alloc::string::String,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ExecuteQueryFailureResult {
#[prost(oneof = "execute_query_failure_result::Failure", tags = "1, 2, 3")]
pub failure: ::core::option::Option<execute_query_failure_result::Failure>,
}
/// Nested message and enum types in `ExecuteQueryFailureResult`.
pub mod execute_query_failure_result {
- #[derive(Clone, PartialEq, ::prost::Oneof)]
+ #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Failure {
#[prost(string, tag = "1")]
SessionNotFound(::prost::alloc::string::String),
@@ -891,7 +915,7 @@
SqlParsingFailure(::prost::alloc::string::String),
}
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetJobStatusParams {
#[prost(string, tag = "1")]
pub job_id: ::prost::alloc::string::String,
@@ -907,13 +931,13 @@
#[prost(uint64, tag = "4")]
pub ended_at: u64,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct QueuedJob {
#[prost(uint64, tag = "1")]
pub queued_at: u64,
}
/// TODO: add progress report
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RunningJob {
#[prost(uint64, tag = "1")]
pub queued_at: u64,
@@ -922,7 +946,7 @@
#[prost(string, tag = "3")]
pub scheduler: ::prost::alloc::string::String,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct FailedJob {
#[prost(string, tag = "1")]
pub error: ::prost::alloc::string::String,
@@ -961,27 +985,27 @@
#[prost(message, optional, tag = "1")]
pub status: ::core::option::Option<JobStatus>,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct FilePartitionMetadata {
#[prost(string, repeated, tag = "1")]
pub filename: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct CancelJobParams {
#[prost(string, tag = "1")]
pub job_id: ::prost::alloc::string::String,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct CancelJobResult {
#[prost(bool, tag = "1")]
pub cancelled: bool,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct CleanJobDataParams {
#[prost(string, tag = "1")]
pub job_id: ::prost::alloc::string::String,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct CleanJobDataResult {}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct LaunchTaskParams {
@@ -999,13 +1023,13 @@
#[prost(string, tag = "2")]
pub scheduler_id: ::prost::alloc::string::String,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct LaunchTaskResult {
/// TODO when part of the task set are scheduled successfully
#[prost(bool, tag = "1")]
pub success: bool,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct LaunchMultiTaskResult {
/// TODO when part of the task set are scheduled successfully
#[prost(bool, tag = "1")]
@@ -1016,19 +1040,19 @@
#[prost(message, repeated, tag = "1")]
pub task_infos: ::prost::alloc::vec::Vec<RunningTaskInfo>,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct CancelTasksResult {
#[prost(bool, tag = "1")]
pub cancelled: bool,
}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RemoveJobDataParams {
#[prost(string, tag = "1")]
pub job_id: ::prost::alloc::string::String,
}
-#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RemoveJobDataResult {}
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RunningTaskInfo {
#[prost(uint32, tag = "1")]
pub task_id: u32,
@@ -1143,7 +1167,7 @@
format!("Service was not ready: {}", e.into()),
)
})?;
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/ballista.protobuf.SchedulerGrpc/PollWork",
);
@@ -1167,7 +1191,7 @@
format!("Service was not ready: {}", e.into()),
)
})?;
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/ballista.protobuf.SchedulerGrpc/RegisterExecutor",
);
@@ -1198,7 +1222,7 @@
format!("Service was not ready: {}", e.into()),
)
})?;
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/ballista.protobuf.SchedulerGrpc/HeartBeatFromExecutor",
);
@@ -1227,7 +1251,7 @@
format!("Service was not ready: {}", e.into()),
)
})?;
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/ballista.protobuf.SchedulerGrpc/UpdateTaskStatus",
);
@@ -1256,7 +1280,7 @@
format!("Service was not ready: {}", e.into()),
)
})?;
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/ballista.protobuf.SchedulerGrpc/CreateUpdateSession",
);
@@ -1285,7 +1309,7 @@
format!("Service was not ready: {}", e.into()),
)
})?;
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/ballista.protobuf.SchedulerGrpc/RemoveSession",
);
@@ -1311,7 +1335,7 @@
format!("Service was not ready: {}", e.into()),
)
})?;
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/ballista.protobuf.SchedulerGrpc/ExecuteQuery",
);
@@ -1337,7 +1361,7 @@
format!("Service was not ready: {}", e.into()),
)
})?;
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/ballista.protobuf.SchedulerGrpc/GetJobStatus",
);
@@ -1364,7 +1388,7 @@
format!("Service was not ready: {}", e.into()),
)
})?;
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/ballista.protobuf.SchedulerGrpc/ExecutorStopped",
);
@@ -1390,7 +1414,7 @@
format!("Service was not ready: {}", e.into()),
)
})?;
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/ballista.protobuf.SchedulerGrpc/CancelJob",
);
@@ -1414,7 +1438,7 @@
format!("Service was not ready: {}", e.into()),
)
})?;
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/ballista.protobuf.SchedulerGrpc/CleanJobData",
);
@@ -1427,227 +1451,6 @@
}
}
}
-/// Generated client implementations.
-pub mod executor_grpc_client {
- #![allow(
- unused_variables,
- dead_code,
- missing_docs,
- clippy::wildcard_imports,
- clippy::let_unit_value,
- )]
- use tonic::codegen::*;
- use tonic::codegen::http::Uri;
- #[derive(Debug, Clone)]
- pub struct ExecutorGrpcClient<T> {
- inner: tonic::client::Grpc<T>,
- }
- impl ExecutorGrpcClient<tonic::transport::Channel> {
- /// Attempt to create a new client by connecting to a given endpoint.
- pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
- where
- D: TryInto<tonic::transport::Endpoint>,
- D::Error: Into<StdError>,
- {
- let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
- Ok(Self::new(conn))
- }
- }
- impl<T> ExecutorGrpcClient<T>
- where
- T: tonic::client::GrpcService<tonic::body::Body>,
- T::Error: Into<StdError>,
- T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
- <T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
- {
- pub fn new(inner: T) -> Self {
- let inner = tonic::client::Grpc::new(inner);
- Self { inner }
- }
- pub fn with_origin(inner: T, origin: Uri) -> Self {
- let inner = tonic::client::Grpc::with_origin(inner, origin);
- Self { inner }
- }
- pub fn with_interceptor<F>(
- inner: T,
- interceptor: F,
- ) -> ExecutorGrpcClient<InterceptedService<T, F>>
- where
- F: tonic::service::Interceptor,
- T::ResponseBody: Default,
- T: tonic::codegen::Service<
- http::Request<tonic::body::Body>,
- Response = http::Response<
- <T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody,
- >,
- >,
- <T as tonic::codegen::Service<
- http::Request<tonic::body::Body>,
- >>::Error: Into<StdError> + std::marker::Send + std::marker::Sync,
- {
- ExecutorGrpcClient::new(InterceptedService::new(inner, interceptor))
- }
- /// Compress requests with the given encoding.
- ///
- /// This requires the server to support it otherwise it might respond with an
- /// error.
- #[must_use]
- pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
- self.inner = self.inner.send_compressed(encoding);
- self
- }
- /// Enable decompressing responses.
- #[must_use]
- pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
- self.inner = self.inner.accept_compressed(encoding);
- self
- }
- /// Limits the maximum size of a decoded message.
- ///
- /// Default: `4MB`
- #[must_use]
- pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
- self.inner = self.inner.max_decoding_message_size(limit);
- self
- }
- /// Limits the maximum size of an encoded message.
- ///
- /// Default: `usize::MAX`
- #[must_use]
- pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
- self.inner = self.inner.max_encoding_message_size(limit);
- self
- }
- pub async fn launch_task(
- &mut self,
- request: impl tonic::IntoRequest<super::LaunchTaskParams>,
- ) -> std::result::Result<
- tonic::Response<super::LaunchTaskResult>,
- tonic::Status,
- > {
- self.inner
- .ready()
- .await
- .map_err(|e| {
- tonic::Status::unknown(
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/ballista.protobuf.ExecutorGrpc/LaunchTask",
- );
- let mut req = request.into_request();
- req.extensions_mut()
- .insert(GrpcMethod::new("ballista.protobuf.ExecutorGrpc", "LaunchTask"));
- self.inner.unary(req, path, codec).await
- }
- pub async fn launch_multi_task(
- &mut self,
- request: impl tonic::IntoRequest<super::LaunchMultiTaskParams>,
- ) -> std::result::Result<
- tonic::Response<super::LaunchMultiTaskResult>,
- tonic::Status,
- > {
- self.inner
- .ready()
- .await
- .map_err(|e| {
- tonic::Status::unknown(
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/ballista.protobuf.ExecutorGrpc/LaunchMultiTask",
- );
- let mut req = request.into_request();
- req.extensions_mut()
- .insert(
- GrpcMethod::new("ballista.protobuf.ExecutorGrpc", "LaunchMultiTask"),
- );
- self.inner.unary(req, path, codec).await
- }
- pub async fn stop_executor(
- &mut self,
- request: impl tonic::IntoRequest<super::StopExecutorParams>,
- ) -> std::result::Result<
- tonic::Response<super::StopExecutorResult>,
- tonic::Status,
- > {
- self.inner
- .ready()
- .await
- .map_err(|e| {
- tonic::Status::unknown(
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/ballista.protobuf.ExecutorGrpc/StopExecutor",
- );
- let mut req = request.into_request();
- req.extensions_mut()
- .insert(
- GrpcMethod::new("ballista.protobuf.ExecutorGrpc", "StopExecutor"),
- );
- self.inner.unary(req, path, codec).await
- }
- pub async fn cancel_tasks(
- &mut self,
- request: impl tonic::IntoRequest<super::CancelTasksParams>,
- ) -> std::result::Result<
- tonic::Response<super::CancelTasksResult>,
- tonic::Status,
- > {
- self.inner
- .ready()
- .await
- .map_err(|e| {
- tonic::Status::unknown(
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/ballista.protobuf.ExecutorGrpc/CancelTasks",
- );
- let mut req = request.into_request();
- req.extensions_mut()
- .insert(
- GrpcMethod::new("ballista.protobuf.ExecutorGrpc", "CancelTasks"),
- );
- self.inner.unary(req, path, codec).await
- }
- pub async fn remove_job_data(
- &mut self,
- request: impl tonic::IntoRequest<super::RemoveJobDataParams>,
- ) -> std::result::Result<
- tonic::Response<super::RemoveJobDataResult>,
- tonic::Status,
- > {
- self.inner
- .ready()
- .await
- .map_err(|e| {
- tonic::Status::unknown(
- format!("Service was not ready: {}", e.into()),
- )
- })?;
- let codec = tonic::codec::ProstCodec::default();
- let path = http::uri::PathAndQuery::from_static(
- "/ballista.protobuf.ExecutorGrpc/RemoveJobData",
- );
- let mut req = request.into_request();
- req.extensions_mut()
- .insert(
- GrpcMethod::new("ballista.protobuf.ExecutorGrpc", "RemoveJobData"),
- );
- self.inner.unary(req, path, codec).await
- }
- }
-}
/// Generated server implementations.
pub mod scheduler_grpc_server {
#![allow(
@@ -1840,7 +1643,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = PollWorkSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -1886,7 +1689,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = RegisterExecutorSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -1935,7 +1738,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = HeartBeatFromExecutorSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -1981,7 +1784,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = UpdateTaskStatusSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -2027,7 +1830,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = CreateUpdateSessionSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -2072,7 +1875,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = RemoveSessionSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -2117,7 +1920,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = ExecuteQuerySvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -2162,7 +1965,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = GetJobStatusSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -2208,7 +2011,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = ExecutorStoppedSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -2253,7 +2056,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = CancelJobSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -2298,7 +2101,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = CleanJobDataSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -2353,6 +2156,227 @@
const NAME: &'static str = SERVICE_NAME;
}
}
+/// Generated client implementations.
+pub mod executor_grpc_client {
+ #![allow(
+ unused_variables,
+ dead_code,
+ missing_docs,
+ clippy::wildcard_imports,
+ clippy::let_unit_value,
+ )]
+ use tonic::codegen::*;
+ use tonic::codegen::http::Uri;
+ #[derive(Debug, Clone)]
+ pub struct ExecutorGrpcClient<T> {
+ inner: tonic::client::Grpc<T>,
+ }
+ impl ExecutorGrpcClient<tonic::transport::Channel> {
+ /// Attempt to create a new client by connecting to a given endpoint.
+ pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
+ where
+ D: TryInto<tonic::transport::Endpoint>,
+ D::Error: Into<StdError>,
+ {
+ let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
+ Ok(Self::new(conn))
+ }
+ }
+ impl<T> ExecutorGrpcClient<T>
+ where
+ T: tonic::client::GrpcService<tonic::body::Body>,
+ T::Error: Into<StdError>,
+ T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
+ <T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
+ {
+ pub fn new(inner: T) -> Self {
+ let inner = tonic::client::Grpc::new(inner);
+ Self { inner }
+ }
+ pub fn with_origin(inner: T, origin: Uri) -> Self {
+ let inner = tonic::client::Grpc::with_origin(inner, origin);
+ Self { inner }
+ }
+ pub fn with_interceptor<F>(
+ inner: T,
+ interceptor: F,
+ ) -> ExecutorGrpcClient<InterceptedService<T, F>>
+ where
+ F: tonic::service::Interceptor,
+ T::ResponseBody: Default,
+ T: tonic::codegen::Service<
+ http::Request<tonic::body::Body>,
+ Response = http::Response<
+ <T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody,
+ >,
+ >,
+ <T as tonic::codegen::Service<
+ http::Request<tonic::body::Body>,
+ >>::Error: Into<StdError> + std::marker::Send + std::marker::Sync,
+ {
+ ExecutorGrpcClient::new(InterceptedService::new(inner, interceptor))
+ }
+ /// Compress requests with the given encoding.
+ ///
+ /// This requires the server to support it otherwise it might respond with an
+ /// error.
+ #[must_use]
+ pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
+ self.inner = self.inner.send_compressed(encoding);
+ self
+ }
+ /// Enable decompressing responses.
+ #[must_use]
+ pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
+ self.inner = self.inner.accept_compressed(encoding);
+ self
+ }
+ /// Limits the maximum size of a decoded message.
+ ///
+ /// Default: `4MB`
+ #[must_use]
+ pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
+ self.inner = self.inner.max_decoding_message_size(limit);
+ self
+ }
+ /// Limits the maximum size of an encoded message.
+ ///
+ /// Default: `usize::MAX`
+ #[must_use]
+ pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
+ self.inner = self.inner.max_encoding_message_size(limit);
+ self
+ }
+ pub async fn launch_task(
+ &mut self,
+ request: impl tonic::IntoRequest<super::LaunchTaskParams>,
+ ) -> std::result::Result<
+ tonic::Response<super::LaunchTaskResult>,
+ tonic::Status,
+ > {
+ self.inner
+ .ready()
+ .await
+ .map_err(|e| {
+ tonic::Status::unknown(
+ format!("Service was not ready: {}", e.into()),
+ )
+ })?;
+ let codec = tonic_prost::ProstCodec::default();
+ let path = http::uri::PathAndQuery::from_static(
+ "/ballista.protobuf.ExecutorGrpc/LaunchTask",
+ );
+ let mut req = request.into_request();
+ req.extensions_mut()
+ .insert(GrpcMethod::new("ballista.protobuf.ExecutorGrpc", "LaunchTask"));
+ self.inner.unary(req, path, codec).await
+ }
+ pub async fn launch_multi_task(
+ &mut self,
+ request: impl tonic::IntoRequest<super::LaunchMultiTaskParams>,
+ ) -> std::result::Result<
+ tonic::Response<super::LaunchMultiTaskResult>,
+ tonic::Status,
+ > {
+ self.inner
+ .ready()
+ .await
+ .map_err(|e| {
+ tonic::Status::unknown(
+ format!("Service was not ready: {}", e.into()),
+ )
+ })?;
+ let codec = tonic_prost::ProstCodec::default();
+ let path = http::uri::PathAndQuery::from_static(
+ "/ballista.protobuf.ExecutorGrpc/LaunchMultiTask",
+ );
+ let mut req = request.into_request();
+ req.extensions_mut()
+ .insert(
+ GrpcMethod::new("ballista.protobuf.ExecutorGrpc", "LaunchMultiTask"),
+ );
+ self.inner.unary(req, path, codec).await
+ }
+ pub async fn stop_executor(
+ &mut self,
+ request: impl tonic::IntoRequest<super::StopExecutorParams>,
+ ) -> std::result::Result<
+ tonic::Response<super::StopExecutorResult>,
+ tonic::Status,
+ > {
+ self.inner
+ .ready()
+ .await
+ .map_err(|e| {
+ tonic::Status::unknown(
+ format!("Service was not ready: {}", e.into()),
+ )
+ })?;
+ let codec = tonic_prost::ProstCodec::default();
+ let path = http::uri::PathAndQuery::from_static(
+ "/ballista.protobuf.ExecutorGrpc/StopExecutor",
+ );
+ let mut req = request.into_request();
+ req.extensions_mut()
+ .insert(
+ GrpcMethod::new("ballista.protobuf.ExecutorGrpc", "StopExecutor"),
+ );
+ self.inner.unary(req, path, codec).await
+ }
+ pub async fn cancel_tasks(
+ &mut self,
+ request: impl tonic::IntoRequest<super::CancelTasksParams>,
+ ) -> std::result::Result<
+ tonic::Response<super::CancelTasksResult>,
+ tonic::Status,
+ > {
+ self.inner
+ .ready()
+ .await
+ .map_err(|e| {
+ tonic::Status::unknown(
+ format!("Service was not ready: {}", e.into()),
+ )
+ })?;
+ let codec = tonic_prost::ProstCodec::default();
+ let path = http::uri::PathAndQuery::from_static(
+ "/ballista.protobuf.ExecutorGrpc/CancelTasks",
+ );
+ let mut req = request.into_request();
+ req.extensions_mut()
+ .insert(
+ GrpcMethod::new("ballista.protobuf.ExecutorGrpc", "CancelTasks"),
+ );
+ self.inner.unary(req, path, codec).await
+ }
+ pub async fn remove_job_data(
+ &mut self,
+ request: impl tonic::IntoRequest<super::RemoveJobDataParams>,
+ ) -> std::result::Result<
+ tonic::Response<super::RemoveJobDataResult>,
+ tonic::Status,
+ > {
+ self.inner
+ .ready()
+ .await
+ .map_err(|e| {
+ tonic::Status::unknown(
+ format!("Service was not ready: {}", e.into()),
+ )
+ })?;
+ let codec = tonic_prost::ProstCodec::default();
+ let path = http::uri::PathAndQuery::from_static(
+ "/ballista.protobuf.ExecutorGrpc/RemoveJobData",
+ );
+ let mut req = request.into_request();
+ req.extensions_mut()
+ .insert(
+ GrpcMethod::new("ballista.protobuf.ExecutorGrpc", "RemoveJobData"),
+ );
+ self.inner.unary(req, path, codec).await
+ }
+ }
+}
/// Generated server implementations.
pub mod executor_grpc_server {
#![allow(
@@ -2508,7 +2532,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = LaunchTaskSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -2554,7 +2578,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = LaunchMultiTaskSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -2599,7 +2623,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = StopExecutorSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -2644,7 +2668,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = CancelTasksSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
@@ -2689,7 +2713,7 @@
let inner = self.inner.clone();
let fut = async move {
let method = RemoveJobDataSvc(inner);
- let codec = tonic::codec::ProstCodec::default();
+ let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
diff --git a/ballista/core/src/serde/mod.rs b/ballista/core/src/serde/mod.rs
index c9bd7f2..0890168 100644
--- a/ballista/core/src/serde/mod.rs
+++ b/ballista/core/src/serde/mod.rs
@@ -23,9 +23,8 @@
use arrow_flight::sql::ProstMessageExt;
use datafusion::arrow::datatypes::SchemaRef;
use datafusion::common::{DataFusionError, Result};
-use datafusion::execution::{FunctionRegistry, SessionStateBuilder};
+use datafusion::execution::TaskContext;
use datafusion::physical_plan::{ExecutionPlan, Partitioning};
-use datafusion::prelude::SessionContext;
use datafusion_proto::logical_plan::file_formats::{
ArrowLogicalExtensionCodec, AvroLogicalExtensionCodec, CsvLogicalExtensionCodec,
JsonLogicalExtensionCodec, ParquetLogicalExtensionCodec,
@@ -178,7 +177,7 @@
&self,
buf: &[u8],
inputs: &[datafusion::logical_expr::LogicalPlan],
- ctx: &datafusion::prelude::SessionContext,
+ ctx: &TaskContext,
) -> Result<datafusion::logical_expr::Extension> {
self.default_codec.try_decode(buf, inputs, ctx)
}
@@ -196,7 +195,7 @@
buf: &[u8],
table_ref: &datafusion::sql::TableReference,
schema: datafusion::arrow::datatypes::SchemaRef,
- ctx: &datafusion::prelude::SessionContext,
+ ctx: &TaskContext,
) -> Result<Arc<dyn datafusion::catalog::TableProvider>> {
self.default_codec
.try_decode_table_provider(buf, table_ref, schema, ctx)
@@ -215,7 +214,7 @@
fn try_decode_file_format(
&self,
buf: &[u8],
- ctx: &datafusion::prelude::SessionContext,
+ ctx: &TaskContext,
) -> Result<Arc<dyn datafusion::datasource::file_format::FileFormatFactory>> {
let proto = FileFormatProto::decode(buf)
.map_err(|e| DataFusionError::Internal(e.to_string()))?;
@@ -267,7 +266,7 @@
&self,
buf: &[u8],
inputs: &[Arc<dyn ExecutionPlan>],
- registry: &dyn FunctionRegistry,
+ ctx: &TaskContext,
) -> Result<Arc<dyn ExecutionPlan>, DataFusionError> {
let ballista_plan: protobuf::BallistaPhysicalPlanNode =
protobuf::BallistaPhysicalPlanNode::decode(buf).map_err(|e| {
@@ -282,33 +281,6 @@
"Could not deserialize BallistaPhysicalPlanNode because it's physical_plan_type is none".to_string()
)
})?;
- // FIXME: this is temporary until we get datafusion 51
- // more details at https://github.com/apache/datafusion/issues/17596
- let mut state = SessionStateBuilder::new_with_default_features().build();
-
- for function_name in registry.udfs() {
- if let Ok(function) = registry.udf(&function_name) {
- state.register_udf(function)?;
- }
- }
-
- for function_name in registry.udafs() {
- if let Ok(function) = registry.udaf(&function_name) {
- state.register_udaf(function)?;
- }
- }
-
- for function_name in registry.udafs() {
- if let Ok(function) = registry.udaf(&function_name) {
- state.register_udaf(function)?;
- }
- }
-
- let ctx = SessionContext::new_with_state(state);
-
- //
- //
- //
match ballista_plan {
PhysicalPlanType::ShuffleWriter(shuffle_writer) => {
@@ -316,7 +288,7 @@
let shuffle_output_partitioning = parse_protobuf_hash_partitioning(
shuffle_writer.output_partitioning.as_ref(),
- &ctx, //registry,
+ ctx,
input.schema().as_ref(),
self.default_codec.as_ref(),
)?;
@@ -351,7 +323,7 @@
.collect::<Result<Vec<_>, DataFusionError>>()?;
let partitioning = parse_protobuf_partitioning(
shuffle_reader.partitioning.as_ref(),
- &ctx, //registry,
+ ctx,
schema.as_ref(),
self.default_codec.as_ref(),
)?;
@@ -370,7 +342,7 @@
Arc::new(convert_required!(unresolved_shuffle.schema)?);
let partitioning = parse_protobuf_partitioning(
unresolved_shuffle.partitioning.as_ref(),
- &ctx, //registry,
+ ctx,
schema.as_ref(),
self.default_codec.as_ref(),
)?;
@@ -519,7 +491,6 @@
mod test {
use super::*;
use datafusion::arrow::datatypes::{DataType, Field, Schema};
- use datafusion::execution::registry::MemoryFunctionRegistry;
use datafusion::physical_plan::expressions::col;
use datafusion::physical_plan::Partitioning;
use datafusion::{
@@ -533,7 +504,7 @@
#[tokio::test]
async fn file_format_serialization_roundtrip() {
- let ctx = SessionContext::new();
+ let ctx = SessionContext::new().task_ctx();
let empty = EmptyRelation {
produce_one_row: false,
schema: Arc::new(DFSchema::empty()),
@@ -593,8 +564,8 @@
.try_encode(Arc::new(original_exec.clone()), &mut buf)
.unwrap();
- let registry = MemoryFunctionRegistry::new();
- let decoded_plan = codec.try_decode(&buf, &[], ®istry).unwrap();
+ let ctx = SessionContext::new().task_ctx();
+ let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap();
let decoded_exec = decoded_plan
.as_any()
@@ -626,8 +597,8 @@
.try_encode(Arc::new(original_exec.clone()), &mut buf)
.unwrap();
- let registry = MemoryFunctionRegistry::new();
- let decoded_plan = codec.try_decode(&buf, &[], ®istry).unwrap();
+ let ctx = SessionContext::new().task_ctx();
+ let decoded_plan = codec.try_decode(&buf, &[], &ctx).unwrap();
let decoded_exec = decoded_plan
.as_any()
diff --git a/ballista/core/src/serde/scheduler/from_proto.rs b/ballista/core/src/serde/scheduler/from_proto.rs
index c4e88f4..4072bf0 100644
--- a/ballista/core/src/serde/scheduler/from_proto.rs
+++ b/ballista/core/src/serde/scheduler/from_proto.rs
@@ -18,16 +18,15 @@
use chrono::{TimeZone, Utc};
use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode};
-use datafusion::execution::SessionStateBuilder;
+use datafusion::execution::TaskContext;
use datafusion::logical_expr::{AggregateUDF, ScalarUDF, WindowUDF};
use datafusion::physical_plan::metrics::{
- Count, Gauge, MetricValue, MetricsSet, Time, Timestamp,
+ Count, Gauge, MetricValue, MetricsSet, PruningMetrics, RatioMetrics, Time, Timestamp,
};
use datafusion::physical_plan::{ExecutionPlan, Metric};
-use datafusion::prelude::{SessionConfig, SessionContext};
+use datafusion::prelude::SessionConfig;
use datafusion_proto::logical_plan::AsLogicalPlan;
use datafusion_proto::physical_plan::AsExecutionPlan;
-use itertools::Itertools;
use std::collections::HashMap;
use std::convert::TryInto;
use std::sync::Arc;
@@ -35,6 +34,7 @@
use crate::error::BallistaError;
use crate::extension::SessionConfigHelperExt;
+use crate::serde::protobuf::{NamedPruningMetrics, NamedRatio};
use crate::serde::scheduler::{
Action, BallistaFunctionRegistry, ExecutorData, ExecutorMetadata,
ExecutorSpecification, PartitionId, PartitionLocation, PartitionStats,
@@ -201,6 +201,33 @@
timestamp.set(Utc.timestamp_nanos(value));
Ok(MetricValue::EndTimestamp(timestamp))
}
+ Some(operator_metric::Metric::OutputBytes(value)) => {
+ let count = Count::new();
+ count.add(value as usize);
+ Ok(MetricValue::OutputBytes(count))
+ }
+ Some(operator_metric::Metric::PruningMetrics(NamedPruningMetrics {
+ name,
+ pruned,
+ matched,
+ })) => {
+ let pruning_metrics = PruningMetrics::new();
+ pruning_metrics.add_pruned(pruned as usize);
+ pruning_metrics.add_matched(matched as usize);
+ Ok(MetricValue::PruningMetrics {
+ name: name.into(),
+ pruning_metrics,
+ })
+ }
+ Some(operator_metric::Metric::Ratio(NamedRatio { name, part, total })) => {
+ let ratio_metrics = RatioMetrics::new();
+ ratio_metrics.add_part(part as usize);
+ ratio_metrics.add_total(total as usize);
+ Ok(MetricValue::Ratio {
+ name: name.into(),
+ ratio_metrics,
+ })
+ }
None => Err(BallistaError::General(
"scheduler::from_proto(OperatorMetric) metric is None.".to_owned(),
)),
@@ -303,27 +330,19 @@
window_functions: window_functions.clone(),
});
- // this is temporary fix until we get
- // https://github.com/apache/datafusion/pull/17601
- // merged
- //
- let session_state = SessionStateBuilder::new()
- .with_aggregate_functions(aggregate_functions.values().cloned().collect_vec())
- .with_scalar_functions(scalar_functions.values().cloned().collect_vec())
- .with_window_functions(window_functions.values().cloned().collect_vec())
- .with_config(session_config.clone())
- .with_runtime_env(runtime.clone())
- .build();
- let ctx = SessionContext::new_with_state(session_state);
- //
+ let ctx = TaskContext::new(
+ None,
+ task.session_id.clone(),
+ session_config.clone(),
+ scalar_functions.clone(),
+ aggregate_functions.clone(),
+ window_functions.clone(),
+ runtime.clone(),
+ );
let encoded_plan = task.plan.as_slice();
let plan: Arc<dyn ExecutionPlan> = U::try_decode(encoded_plan).and_then(|proto| {
- proto.try_into_physical_plan(
- &ctx,
- runtime.as_ref(),
- codec.physical_extension_codec(),
- )
+ proto.try_into_physical_plan(&ctx, codec.physical_extension_codec())
})?;
let job_id = task.job_id;
@@ -371,27 +390,19 @@
window_functions: window_functions.clone(),
});
- // this is temporary fix until we get
- // https://github.com/apache/datafusion/pull/17601
- // merged
- //
- let session_state = SessionStateBuilder::new()
- .with_aggregate_functions(aggregate_functions.values().cloned().collect_vec())
- .with_scalar_functions(scalar_functions.values().cloned().collect_vec())
- .with_window_functions(window_functions.values().cloned().collect_vec())
- .with_config(session_config.clone())
- .with_runtime_env(runtime.clone())
- .build();
- let ctx = SessionContext::new_with_state(session_state);
- //
+ let ctx = TaskContext::new(
+ None,
+ uuid::Uuid::new_v4().to_string(),
+ session_config.clone(),
+ scalar_functions.clone(),
+ aggregate_functions.clone(),
+ window_functions.clone(),
+ runtime.clone(),
+ );
let encoded_plan = multi_task.plan.as_slice();
let plan: Arc<dyn ExecutionPlan> = U::try_decode(encoded_plan).and_then(|proto| {
- proto.try_into_physical_plan(
- &ctx,
- runtime.as_ref(),
- codec.physical_extension_codec(),
- )
+ proto.try_into_physical_plan(&ctx, codec.physical_extension_codec())
})?;
let job_id = multi_task.job_id;
diff --git a/ballista/core/src/serde/scheduler/to_proto.rs b/ballista/core/src/serde/scheduler/to_proto.rs
index 5028c51..5abb099 100644
--- a/ballista/core/src/serde/scheduler/to_proto.rs
+++ b/ballista/core/src/serde/scheduler/to_proto.rs
@@ -21,7 +21,7 @@
use crate::error::BallistaError;
-use crate::serde::protobuf::{self};
+use crate::serde::protobuf::{self, NamedPruningMetrics, NamedRatio};
use datafusion_proto::protobuf as datafusion_protobuf;
use crate::serde::scheduler::{
@@ -177,6 +177,31 @@
.unwrap_or(0),
)),
}),
+ MetricValue::OutputBytes(count) => Ok(protobuf::OperatorMetric {
+ metric: Some(operator_metric::Metric::OutputBytes(count.value() as u64)),
+ }),
+ MetricValue::PruningMetrics {
+ name,
+ pruning_metrics,
+ } => Ok(protobuf::OperatorMetric {
+ metric: Some(operator_metric::Metric::PruningMetrics(
+ NamedPruningMetrics {
+ name: name.to_string(),
+ pruned: pruning_metrics.pruned() as u64,
+ matched: pruning_metrics.matched() as u64,
+ },
+ )),
+ }),
+ MetricValue::Ratio {
+ name,
+ ratio_metrics,
+ } => Ok(protobuf::OperatorMetric {
+ metric: Some(operator_metric::Metric::Ratio(NamedRatio {
+ name: name.to_string(),
+ part: ratio_metrics.part() as u64,
+ total: ratio_metrics.total() as u64,
+ })),
+ }),
// at the moment there there is no way to serialize custom metrics
// thus at the moment we can't support it
MetricValue::Custom { .. } => Err(BallistaError::General(String::from(
diff --git a/ballista/executor/src/execution_loop.rs b/ballista/executor/src/execution_loop.rs
index f6428a5..36b58a1 100644
--- a/ballista/executor/src/execution_loop.rs
+++ b/ballista/executor/src/execution_loop.rs
@@ -28,9 +28,7 @@
use ballista_core::serde::scheduler::{ExecutorSpecification, PartitionId};
use ballista_core::serde::BallistaCodec;
use datafusion::execution::context::TaskContext;
-use datafusion::execution::SessionStateBuilder;
use datafusion::physical_plan::ExecutionPlan;
-use datafusion::prelude::SessionContext;
use datafusion_proto::logical_plan::AsLogicalPlan;
use datafusion_proto::physical_plan::AsExecutionPlan;
use futures::FutureExt;
@@ -38,7 +36,6 @@
use std::any::Any;
use std::convert::TryInto;
use std::error::Error;
-use std::ops::Deref;
use std::sync::mpsc::{Receiver, Sender, TryRecvError};
use std::time::{SystemTime, UNIX_EPOCH};
use std::{sync::Arc, time::Duration};
@@ -240,20 +237,6 @@
let runtime = executor.produce_runtime(&session_config)?;
- // this is temporary fix until we get
- // https://github.com/apache/datafusion/pull/17601
- // merged
- //
- let session_state = SessionStateBuilder::new()
- .with_aggregate_functions(task_aggregate_functions.values().cloned().collect())
- .with_scalar_functions(task_scalar_functions.values().cloned().collect())
- .with_window_functions(task_window_functions.values().cloned().collect())
- .with_config(session_config.clone())
- .with_runtime_env(runtime.clone())
- .build();
- let ctx = SessionContext::new_with_state(session_state);
- //
-
let session_id = task.session_id.clone();
let task_context = Arc::new(TaskContext::new(
Some(task_identity.clone()),
@@ -267,11 +250,7 @@
let plan: Arc<dyn ExecutionPlan> =
U::try_decode(task.plan.as_slice()).and_then(|proto| {
- proto.try_into_physical_plan(
- &ctx,
- runtime.deref(),
- codec.physical_extension_codec(),
- )
+ proto.try_into_physical_plan(&task_context, codec.physical_extension_codec())
})?;
let query_stage_exec = executor.execution_engine.create_query_stage_exec(
diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs
index daf4d97..80fa844 100644
--- a/ballista/scheduler/src/planner.rs
+++ b/ballista/scheduler/src/planner.rs
@@ -328,6 +328,7 @@
use ballista_core::execution_plans::{ShuffleWriterExec, UnresolvedShuffleExec};
use ballista_core::serde::BallistaCodec;
use datafusion::arrow::compute::SortOptions;
+ use datafusion::execution::TaskContext;
use datafusion::physical_expr::expressions::Column;
use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode};
use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
@@ -339,11 +340,9 @@
use datafusion::physical_plan::windows::BoundedWindowAggExec;
use datafusion::physical_plan::{displayable, ExecutionPlan};
use datafusion::physical_plan::{InputOrderMode, Partitioning};
- use datafusion::prelude::SessionContext;
use datafusion_proto::physical_plan::AsExecutionPlan;
use datafusion_proto::protobuf::LogicalPlanNode;
use datafusion_proto::protobuf::PhysicalPlanNode;
- use std::ops::Deref;
use std::sync::Arc;
use uuid::Uuid;
@@ -789,7 +788,8 @@
)?;
let partial_hash = stages[0].children()[0].clone();
- let partial_hash_serde = roundtrip_operator(&ctx, partial_hash.clone())?;
+ let partial_hash_serde =
+ roundtrip_operator(&ctx.task_ctx(), partial_hash.clone())?;
let partial_hash = downcast_exec!(partial_hash, AggregateExec);
let partial_hash_serde = downcast_exec!(partial_hash_serde, AggregateExec);
@@ -803,7 +803,7 @@
}
fn roundtrip_operator(
- ctx: &SessionContext,
+ ctx: &TaskContext,
plan: Arc<dyn ExecutionPlan>,
) -> Result<Arc<dyn ExecutionPlan>, BallistaError> {
let codec: BallistaCodec<LogicalPlanNode, PhysicalPlanNode> =
@@ -813,12 +813,8 @@
plan.clone(),
codec.physical_extension_codec(),
)?;
- let runtime = ctx.runtime_env();
- let result_exec_plan: Arc<dyn ExecutionPlan> = (proto).try_into_physical_plan(
- ctx,
- runtime.deref(),
- codec.physical_extension_codec(),
- )?;
+ let result_exec_plan: Arc<dyn ExecutionPlan> =
+ (proto).try_into_physical_plan(ctx, codec.physical_extension_codec())?;
Ok(result_exec_plan)
}
}
diff --git a/ballista/scheduler/src/scheduler_server/grpc.rs b/ballista/scheduler/src/scheduler_server/grpc.rs
index 8613384..58f53cb 100644
--- a/ballista/scheduler/src/scheduler_server/grpc.rs
+++ b/ballista/scheduler/src/scheduler_server/grpc.rs
@@ -369,7 +369,7 @@
Query::LogicalPlan(message) => {
match T::try_decode(message.as_slice()).and_then(|m| {
m.try_into_logical_plan(
- session_ctx.deref(),
+ session_ctx.task_ctx().deref(),
self.state.codec.logical_extension_codec(),
)
}) {
diff --git a/ballista/scheduler/src/state/distributed_explain.rs b/ballista/scheduler/src/state/distributed_explain.rs
index 5aee48f..8d602a4 100644
--- a/ballista/scheduler/src/state/distributed_explain.rs
+++ b/ballista/scheduler/src/state/distributed_explain.rs
@@ -156,7 +156,7 @@
Vec::new(),
out_schema,
UnnestOptions::default(),
- ));
+ )?);
// Final projection: rename columns to (plan_type, plan)
let proj_final = Arc::new(ProjectionExec::try_new(
diff --git a/benchmarks/src/bin/tpch.rs b/benchmarks/src/bin/tpch.rs
index 16a5917..86a4187 100644
--- a/benchmarks/src/bin/tpch.rs
+++ b/benchmarks/src/bin/tpch.rs
@@ -1672,7 +1672,6 @@
use datafusion_proto::logical_plan::AsLogicalPlan;
use datafusion_proto::physical_plan::AsExecutionPlan;
use std::env;
- use std::ops::Deref;
async fn round_trip_logical_plan(n: usize) -> Result<()> {
let config = SessionConfig::new()
@@ -1719,7 +1718,7 @@
)
.unwrap();
let round_trip: LogicalPlan = proto
- .try_into_logical_plan(&ctx, codec.logical_extension_codec())
+ .try_into_logical_plan(&ctx.task_ctx(), codec.logical_extension_codec())
.unwrap();
assert_eq!(
format!("{plan:?}"),
@@ -1777,13 +1776,8 @@
codec.physical_extension_codec(),
)
.unwrap();
- let runtime = ctx.runtime_env();
let round_trip: Arc<dyn ExecutionPlan> = proto
- .try_into_physical_plan(
- &ctx,
- runtime.deref(),
- codec.physical_extension_codec(),
- )
+ .try_into_physical_plan(&ctx.task_ctx(), codec.physical_extension_codec())
.unwrap();
assert_eq!(
format!("{}", displayable(physical_plan.as_ref()).indent(false)),
