This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 19c440584 feat: Upgrade to DataFusion 45 (#1364)
19c440584 is described below
commit 19c440584c161be5b78abc792ecc9ea52f29491e
Author: Andy Grove <[email protected]>
AuthorDate: Fri Feb 7 11:30:56 2025 -0800
feat: Upgrade to DataFusion 45 (#1364)
* bump to DF rev 8d006a2
* fix
* bump to 7f9a8c0
* use 45.0.0-rc1
* use official DF 45 release
---
native/Cargo.lock | 333 +++++++++++++++----------
native/Cargo.toml | 30 +--
native/core/src/execution/operators/filter.rs | 15 +-
native/spark-expr/src/agg_funcs/avg_decimal.rs | 10 +-
native/spark-expr/src/utils.rs | 6 +-
5 files changed, 236 insertions(+), 158 deletions(-)
diff --git a/native/Cargo.lock b/native/Cargo.lock
index ee734a076..d64ca3250 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -25,7 +25,7 @@ checksum =
"e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
dependencies = [
"cfg-if",
"const-random",
- "getrandom",
+ "getrandom 0.2.15",
"once_cell",
"version_check",
"zerocopy",
@@ -123,9 +123,9 @@ checksum =
"7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d2ccdcc8fb14508ca20aaec7076032e5c0b0751b906036d4496786e2f227a37a"
+checksum = "6422e12ac345a0678d7a17e316238e3a40547ae7f92052b77bd86d5e0239f3fc"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -144,9 +144,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a1aad8e27f32e411a0fc0bf5a625a35f0bf9b9f871cf4542abe31f7cef4beea2"
+checksum = "23cf34bb1f48c41d3475927bcc7be498665b8e80b379b88f62a840337f8b8248"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -158,9 +158,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bd6ed90c28c6f73a706c55799b8cc3a094e89257238e5b1d65ca7c70bd3ae23f"
+checksum = "fb4a06d507f54b70a277be22a127c8ffe0cec6cd98c0ad8a48e77779bbda8223"
dependencies = [
"ahash",
"arrow-buffer",
@@ -175,9 +175,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fe4a40bdc1552ea10fbdeae4e5a945d8572c32f66bce457b96c13d9c46b80447"
+checksum = "d69d326d5ad1cb82dcefa9ede3fee8fdca98f9982756b16f9cb142f4aa6edc89"
dependencies = [
"bytes",
"half",
@@ -186,9 +186,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "430c0a21aa7f81bcf0f97c57216d7127795ea755f494d27bae2bd233be43c2cc"
+checksum = "626e65bd42636a84a238bed49d09c8777e3d825bf81f5087a70111c2831d9870"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -207,9 +207,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b4444c8f8c57ac00e6a679ede67d1ae8872c170797dc45b46f75702437a77888"
+checksum = "71c8f959f7a1389b1dbd883cdcd37c3ed12475329c111912f7f69dad8195d8c6"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -223,9 +223,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "09af476cfbe9879937e50b1334c73189de6039186e025b1b1ac84b283b87b20e"
+checksum = "1858e7c7d01c44cf71c21a85534fd1a54501e8d60d1195d0d6fbcc00f4b10754"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -235,9 +235,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "136296e8824333a8a4c4a6e508e4aa65d5678b801246d0408825ae7b2523c628"
+checksum = "a6bb3f727f049884c7603f0364bc9315363f356b59e9f605ea76541847e06a1e"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -249,9 +249,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e222ad0e419ab8276818c5605a5bb1e35ed86fa8c5e550726433cc63b09c3c78"
+checksum = "35de94f165ed8830aede72c35f238763794f0d49c69d30c44d49c9834267ff8c"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -269,9 +269,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "eddf14c5f03b679ec8ceac4dfac43f63cdc4ed54dab3cc120a4ef46af38481eb"
+checksum = "8aa06e5f267dc53efbacb933485c79b6fc1685d3ffbe870a16ce4e696fb429da"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -282,9 +282,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e9acdc58da19f383f4ba381fa0e3583534ae2ceb31269aaf4a03f08ff13e8443"
+checksum = "66f1144bb456a2f9d82677bd3abcea019217e572fc8f07de5a7bac4b2c56eb2c"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -295,18 +295,18 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3a1822a1a952955637e85e8f9d6b0e04dd75d65492b87ec548dd593d3a1f772b"
+checksum = "105f01ec0090259e9a33a9263ec18ff223ab91a0ea9fbc18042f7e38005142f6"
dependencies = [
"bitflags 2.8.0",
]
[[package]]
name = "arrow-select"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5c4172e9a12dfe15303d3926269f9ead471ea93bdd067d113abc65cb6c48e246"
+checksum = "f690752fdbd2dee278b5f1636fefad8f2f7134c85e20fd59c4199e15a39a6807"
dependencies = [
"ahash",
"arrow-array",
@@ -318,9 +318,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "73683040445f4932342781926189901c9521bb1a787c35dbe628a3ce51372d3c"
+checksum = "d0fff9cd745a7039b66c47ecaf5954460f9fa12eed628f65170117ea93e64ee0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -341,13 +341,13 @@ checksum =
"0c24e9d990669fbd16806bff449e4ac644fd9b1fca014760087732fe4102f131"
[[package]]
name = "async-trait"
-version = "0.1.85"
+version = "0.1.86"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056"
+checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -465,9 +465,9 @@ dependencies = [
[[package]]
name = "bumpalo"
-version = "3.16.0"
+version = "3.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
+checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
[[package]]
name = "bytemuck"
@@ -483,9 +483,9 @@ checksum =
"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
-version = "1.9.0"
+version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
+checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9"
[[package]]
name = "cast"
@@ -495,9 +495,9 @@ checksum =
"37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
-version = "1.2.10"
+version = "1.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229"
+checksum = "755717a7de9ec452bf7f3f1a3099085deabd7f2962b861dae91ecd7a365903d2"
dependencies = [
"jobserver",
"libc",
@@ -600,9 +600,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.5.27"
+version = "4.5.28"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796"
+checksum = "3e77c3243bd94243c03672cb5154667347c457ca271254724f9f393aee1c05ff"
dependencies = [
"clap_builder",
]
@@ -659,7 +659,7 @@ version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
"once_cell",
"tiny-keccak",
]
@@ -819,8 +819,9 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "eae420e7a5b0b7f1c39364cc76cbcd0f5fdc416b2514ae3847c2676bbd60702a"
dependencies = [
"arrow",
"arrow-array",
@@ -829,7 +830,6 @@ dependencies = [
"async-trait",
"bytes",
"chrono",
- "dashmap",
"datafusion-catalog",
"datafusion-common",
"datafusion-common-runtime",
@@ -863,16 +863,23 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "6f27987bc22b810939e8dfecc55571e9d50355d6ea8ec1c47af8383a76a6d0e1"
dependencies = [
- "arrow-schema",
+ "arrow",
"async-trait",
+ "dashmap",
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-plan",
+ "datafusion-sql",
+ "futures",
+ "itertools 0.14.0",
+ "log",
"parking_lot",
+ "sqlparser",
]
[[package]]
@@ -963,8 +970,9 @@ dependencies = [
[[package]]
name = "datafusion-common"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e3f6d5b8c9408cc692f7c194b8aa0c0f9b253e065a8d960ad9cdc2a13e697602"
dependencies = [
"ahash",
"arrow",
@@ -988,8 +996,9 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "0d4603c8e8a4baf77660ab7074cc66fc15cc8a18f2ce9dfadb755fc6ee294e48"
dependencies = [
"log",
"tokio",
@@ -997,13 +1006,15 @@ dependencies = [
[[package]]
name = "datafusion-doc"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e5bf4bc68623a5cf231eed601ed6eb41f46a37c4d15d11a0bff24cbc8396cd66"
[[package]]
name = "datafusion-execution"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "88b491c012cdf8e051053426013429a76f74ee3c2db68496c79c323ca1084d27"
dependencies = [
"arrow",
"dashmap",
@@ -1020,8 +1031,9 @@ dependencies = [
[[package]]
name = "datafusion-expr"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e5a181408d4fc5dc22f9252781a8f39f2d0e5d1b33ec9bde242844980a2689c1"
dependencies = [
"arrow",
"chrono",
@@ -1039,18 +1051,21 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "d1129b48e8534d8c03c6543bcdccef0b55c8ac0c1272a15a56c67068b6eb1885"
dependencies = [
"arrow",
"datafusion-common",
"itertools 0.14.0",
+ "paste",
]
[[package]]
name = "datafusion-functions"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "6125874e4856dfb09b59886784fcb74cde5cfc5930b3a80a1a728ef7a010df6b"
dependencies = [
"arrow",
"arrow-buffer",
@@ -1078,8 +1093,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f3add7b1d3888e05e7c95f2b281af900ca69ebdcb21069ba679b33bde8b3b9d6"
dependencies = [
"ahash",
"arrow",
@@ -1100,8 +1116,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate-common"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "6e18baa4cfc3d2f144f74148ed68a1f92337f5072b6dde204a0dbbdf3324989c"
dependencies = [
"ahash",
"arrow",
@@ -1112,8 +1129,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "3ec5ee8cecb0dc370291279673097ddabec03a011f73f30d7f1096457127e03e"
dependencies = [
"arrow",
"arrow-array",
@@ -1135,8 +1153,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-table"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "2c403ddd473bbb0952ba880008428b3c7febf0ed3ce1eec35a205db20efb2a36"
dependencies = [
"arrow",
"async-trait",
@@ -1150,8 +1169,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-window"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "1ab18c2fb835614d06a75f24a9e09136d3a8c12a92d97c95a6af316a1787a9c5"
dependencies = [
"datafusion-common",
"datafusion-doc",
@@ -1166,8 +1186,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-window-common"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "a77b73bc15e7d1967121fdc7a55d819bfb9d6c03766a6c322247dce9094a53a4"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -1175,18 +1196,20 @@ dependencies = [
[[package]]
name = "datafusion-macros"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "09369b8d962291e808977cf94d495fd8b5b38647232d7ef562c27ac0f495b0af"
dependencies = [
"datafusion-expr",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
name = "datafusion-optimizer"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "2403a7e4a84637f3de7d8d4d7a9ccc0cc4be92d89b0161ba3ee5be82f0531c54"
dependencies = [
"arrow",
"chrono",
@@ -1202,8 +1225,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "86ff72ac702b62dbf2650c4e1d715ebd3e4aab14e3885e72e8549e250307347c"
dependencies = [
"ahash",
"arrow",
@@ -1226,11 +1250,13 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "60982b7d684e25579ee29754b4333057ed62e2cc925383c5f0bd8cab7962f435"
dependencies = [
"ahash",
"arrow",
+ "arrow-buffer",
"datafusion-common",
"datafusion-expr-common",
"hashbrown 0.14.5",
@@ -1239,8 +1265,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ac5e85c189d5238a5cf181a624e450c4cd4c66ac77ca551d6f3ff9080bac90bb"
dependencies = [
"arrow",
"arrow-schema",
@@ -1248,7 +1275,6 @@ dependencies = [
"datafusion-execution",
"datafusion-expr",
"datafusion-expr-common",
- "datafusion-functions-aggregate",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
@@ -1260,8 +1286,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c36bf163956d7e2542657c78b3383fdc78f791317ef358a359feffcdb968106f"
dependencies = [
"ahash",
"arrow",
@@ -1291,8 +1318,9 @@ dependencies = [
[[package]]
name = "datafusion-sql"
-version = "44.0.0"
-source =
"git+https://github.com/apache/datafusion?rev=5592834#5592834617f2b8ca99332bdf0be753744af910fd";
+version = "45.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e13caa4daede211ecec53c78b13c503b592794d125f9a3cc3afe992edf9e7f43"
dependencies = [
"arrow",
"arrow-array",
@@ -1351,7 +1379,7 @@ checksum =
"97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -1377,7 +1405,7 @@ checksum =
"3bf679796c0322556351f287a51b49e48f7c4986e727b5dd78c972d30e2e16cc"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -1517,7 +1545,7 @@ checksum =
"162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -1568,7 +1596,19 @@ checksum =
"c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.13.3+wasi-0.2.2",
+ "windows-targets 0.52.6",
]
[[package]]
@@ -1790,7 +1830,7 @@ checksum =
"1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -1912,9 +1952,9 @@ checksum =
"d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
[[package]]
name = "java-locator"
-version = "0.1.8"
+version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6f25f28894af6a5dd349ed5ec46e178654e75f62edb6717ac74007102a57deb5"
+checksum = "09c46c1fe465c59b1474e665e85e1256c3893dd00927b8d55f63b09044c1e64f"
dependencies = [
"glob",
]
@@ -2314,9 +2354,9 @@ dependencies = [
[[package]]
name = "once_cell"
-version = "1.20.2"
+version = "1.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
+checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e"
[[package]]
name = "oorandom"
@@ -2358,9 +2398,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3334c50239d9f4951653d84fa6f636da86f53742e5e5849a30fbe852f3ff4383"
+checksum = "8a01a0efa30bbd601ae85b375c728efdb211ade54390281628a7b16708beb235"
dependencies = [
"ahash",
"arrow-array",
@@ -2384,6 +2424,7 @@ dependencies = [
"object_store",
"paste",
"seq-macro",
+ "simdutf8",
"snap",
"thrift",
"tokio",
@@ -2620,7 +2661,7 @@ dependencies = [
"itertools 0.12.1",
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -2678,7 +2719,7 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
]
[[package]]
@@ -2784,9 +2825,9 @@ checksum =
"f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4"
[[package]]
name = "ryu"
-version = "1.0.18"
+version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd"
[[package]]
name = "same-file"
@@ -2842,14 +2883,14 @@ checksum =
"5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
name = "serde_json"
-version = "1.0.137"
+version = "1.0.138"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b"
+checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949"
dependencies = [
"itoa",
"memchr",
@@ -2893,6 +2934,12 @@ version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
+[[package]]
+name = "simdutf8"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+
[[package]]
name = "siphasher"
version = "1.0.1"
@@ -2932,7 +2979,7 @@ dependencies = [
"heck 0.5.0",
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -2959,7 +3006,7 @@ checksum =
"da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -2996,7 +3043,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -3007,9 +3054,9 @@ checksum =
"13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "symbolic-common"
-version = "12.13.3"
+version = "12.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "13a4dfe4bbeef59c1f32fc7524ae7c95b9e1de5e79a43ce1604e181081d71b0c"
+checksum = "b6189977df1d6ec30c920647919d76f29fb8d8f25e8952e835b0fcda25e8f792"
dependencies = [
"debugid",
"memmap2",
@@ -3019,9 +3066,9 @@ dependencies = [
[[package]]
name = "symbolic-demangle"
-version = "12.13.3"
+version = "12.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "98cf6a95abff97de4d7ff3473f33cacd38f1ddccad5c1feab435d6760300e3b6"
+checksum = "d234917f7986498e7f62061438cee724bafb483fe84cfbe2486f68dce48240d7"
dependencies = [
"cpp_demangle",
"rustc-demangle",
@@ -3041,9 +3088,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.96"
+version = "2.0.98"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80"
+checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1"
dependencies = [
"proc-macro2",
"quote",
@@ -3058,18 +3105,18 @@ checksum =
"c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
name = "tempfile"
-version = "3.15.0"
+version = "3.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704"
+checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91"
dependencies = [
"cfg-if",
"fastrand",
- "getrandom",
+ "getrandom 0.3.1",
"once_cell",
"rustix",
"windows-sys 0.59.0",
@@ -3092,7 +3139,7 @@ checksum =
"4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -3165,7 +3212,7 @@ checksum =
"6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -3187,7 +3234,7 @@ checksum =
"395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -3235,9 +3282,9 @@ checksum =
"42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
[[package]]
name = "unicode-ident"
-version = "1.0.15"
+version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243"
+checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034"
[[package]]
name = "unicode-segmentation"
@@ -3291,11 +3338,11 @@ checksum =
"b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "uuid"
-version = "1.12.1"
+version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b"
+checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0"
dependencies = [
- "getrandom",
+ "getrandom 0.3.1",
]
[[package]]
@@ -3320,6 +3367,15 @@ version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+[[package]]
+name = "wasi"
+version = "0.13.3+wasi-0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2"
+dependencies = [
+ "wit-bindgen-rt",
+]
+
[[package]]
name = "wasm-bindgen"
version = "0.2.100"
@@ -3342,7 +3398,7 @@ dependencies = [
"log",
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
"wasm-bindgen-shared",
]
@@ -3364,7 +3420,7 @@ checksum =
"8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -3589,6 +3645,15 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+[[package]]
+name = "wit-bindgen-rt"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
+dependencies = [
+ "bitflags 2.8.0",
+]
+
[[package]]
name = "write16"
version = "1.0.0"
@@ -3621,7 +3686,7 @@ checksum =
"2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
"synstructure",
]
@@ -3643,7 +3708,7 @@ checksum =
"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
@@ -3663,7 +3728,7 @@ checksum =
"595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
"synstructure",
]
@@ -3686,7 +3751,7 @@ checksum =
"6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.96",
+ "syn 2.0.98",
]
[[package]]
diff --git a/native/Cargo.toml b/native/Cargo.toml
index 29fb269db..6c905554f 100644
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@ -33,21 +33,21 @@ edition = "2021"
rust-version = "1.81"
[workspace.dependencies]
-arrow = { version = "54.0.0", features = ["prettyprint", "ffi", "chrono-tz"] }
-arrow-array = { version = "54.0.0" }
-arrow-buffer = { version = "54.0.0" }
-arrow-data = { version = "54.0.0" }
-arrow-schema = { version = "54.0.0" }
-parquet = { version = "54.0.0", default-features = false, features =
["experimental"] }
-datafusion = { git = "https://github.com/apache/datafusion";, rev = "5592834",
default-features = false, features = ["unicode_expressions",
"crypto_expressions"] }
-datafusion-common = { git = "https://github.com/apache/datafusion";, rev =
"5592834", default-features = false }
-datafusion-functions = { git = "https://github.com/apache/datafusion";, rev =
"5592834", default-features = false, features = ["crypto_expressions"] }
-datafusion-functions-nested = { git = "https://github.com/apache/datafusion";,
rev = "5592834", default-features = false }
-datafusion-expr = { git = "https://github.com/apache/datafusion";, rev =
"5592834", default-features = false }
-datafusion-expr-common = { git = "https://github.com/apache/datafusion";, rev =
"5592834", default-features = false }
-datafusion-execution = { git = "https://github.com/apache/datafusion";, rev =
"5592834", default-features = false }
-datafusion-physical-plan = { git = "https://github.com/apache/datafusion";, rev
= "5592834", default-features = false }
-datafusion-physical-expr = { git = "https://github.com/apache/datafusion";, rev
= "5592834", default-features = false }
+arrow = { version = "54.1.0", features = ["prettyprint", "ffi", "chrono-tz"] }
+arrow-array = { version = "54.1.0" }
+arrow-buffer = { version = "54.1.0" }
+arrow-data = { version = "54.1.0" }
+arrow-schema = { version = "54.1.0" }
+parquet = { version = "54.1.0", default-features = false, features =
["experimental"] }
+datafusion = { version = "45.0.0", default-features = false, features =
["unicode_expressions", "crypto_expressions"] }
+datafusion-common = { version = "45.0.0", default-features = false }
+datafusion-functions = { version = "45.0.0", default-features = false,
features = ["crypto_expressions"] }
+datafusion-functions-nested = { version = "45.0.0", default-features = false }
+datafusion-expr = { version = "45.0.0", default-features = false }
+datafusion-expr-common = { version = "45.0.0", default-features = false }
+datafusion-execution = { version = "45.0.0", default-features = false }
+datafusion-physical-plan = { version = "45.0.0", default-features = false }
+datafusion-physical-expr = { version = "45.0.0", default-features = false }
datafusion-comet-spark-expr = { path = "spark-expr", version = "0.6.0" }
datafusion-comet-proto = { path = "proto", version = "0.6.0" }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
diff --git a/native/core/src/execution/operators/filter.rs
b/native/core/src/execution/operators/filter.rs
index c8ca24077..516217c82 100644
--- a/native/core/src/execution/operators/filter.rs
+++ b/native/core/src/execution/operators/filter.rs
@@ -36,7 +36,9 @@ use datafusion::physical_plan::common::can_project;
use datafusion::physical_plan::execution_plan::CardinalityEffect;
use datafusion_common::cast::as_boolean_array;
use datafusion_common::stats::Precision;
-use datafusion_common::{internal_err, plan_err, project_schema,
DataFusionError, Result};
+use datafusion_common::{
+ internal_err, plan_err, project_schema, DataFusionError, Result,
ScalarValue,
+};
use datafusion_execution::TaskContext;
use datafusion_expr::Operator;
use datafusion_physical_expr::equivalence::ProjectionMapping;
@@ -407,6 +409,16 @@ fn collect_new_statistics(
..
},
)| {
+ let Some(interval) = interval else {
+ // If the interval is `None`, we can say that there are no
rows:
+ return ColumnStatistics {
+ null_count: Precision::Exact(0),
+ max_value: Precision::Exact(ScalarValue::Null),
+ min_value: Precision::Exact(ScalarValue::Null),
+ sum_value: Precision::Exact(ScalarValue::Null),
+ distinct_count: Precision::Exact(0),
+ };
+ };
let (lower, upper) = interval.into_bounds();
let (min_value, max_value) = if lower.eq(&upper) {
(Precision::Exact(lower), Precision::Exact(upper))
@@ -417,6 +429,7 @@ fn collect_new_statistics(
null_count:
input_column_stats[idx].null_count.to_inexact(),
max_value,
min_value,
+ sum_value: Precision::Absent,
distinct_count: distinct_count.to_inexact(),
}
},
diff --git a/native/spark-expr/src/agg_funcs/avg_decimal.rs
b/native/spark-expr/src/agg_funcs/avg_decimal.rs
index 05fc28e58..58b450fa4 100644
--- a/native/spark-expr/src/agg_funcs/avg_decimal.rs
+++ b/native/spark-expr/src/agg_funcs/avg_decimal.rs
@@ -30,7 +30,7 @@ use std::{any::Any, sync::Arc};
use crate::utils::is_valid_decimal_precision;
use arrow_array::ArrowNativeTypeOp;
-use arrow_data::decimal::{MAX_DECIMAL_FOR_EACH_PRECISION,
MIN_DECIMAL_FOR_EACH_PRECISION};
+use arrow_data::decimal::{MAX_DECIMAL128_FOR_EACH_PRECISION,
MIN_DECIMAL128_FOR_EACH_PRECISION};
use datafusion::logical_expr::Volatility::Immutable;
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
use datafusion_expr::type_coercion::aggregates::avg_return_type;
@@ -265,8 +265,8 @@ impl Accumulator for AvgDecimalAccumulator {
fn evaluate(&mut self) -> Result<ScalarValue> {
let scaler =
10_i128.pow(self.target_scale.saturating_sub(self.sum_scale) as u32);
- let target_min = MIN_DECIMAL_FOR_EACH_PRECISION[self.target_precision
as usize - 1];
- let target_max = MAX_DECIMAL_FOR_EACH_PRECISION[self.target_precision
as usize - 1];
+ let target_min =
MIN_DECIMAL128_FOR_EACH_PRECISION[self.target_precision as usize];
+ let target_max =
MAX_DECIMAL128_FOR_EACH_PRECISION[self.target_precision as usize];
let result = self
.sum
@@ -445,8 +445,8 @@ impl GroupsAccumulator for AvgDecimalGroupsAccumulator {
let iter = sums.into_iter().zip(counts);
let scaler =
10_i128.pow(self.target_scale.saturating_sub(self.sum_scale) as u32);
- let target_min = MIN_DECIMAL_FOR_EACH_PRECISION[self.target_precision
as usize - 1];
- let target_max = MAX_DECIMAL_FOR_EACH_PRECISION[self.target_precision
as usize - 1];
+ let target_min =
MIN_DECIMAL128_FOR_EACH_PRECISION[self.target_precision as usize];
+ let target_max =
MAX_DECIMAL128_FOR_EACH_PRECISION[self.target_precision as usize];
for (sum, count) in iter {
if count != 0 {
diff --git a/native/spark-expr/src/utils.rs b/native/spark-expr/src/utils.rs
index ed04941bd..ddee3f643 100644
--- a/native/spark-expr/src/utils.rs
+++ b/native/spark-expr/src/utils.rs
@@ -28,7 +28,7 @@ use arrow::{
temporal_conversions::as_datetime,
};
use arrow_array::types::TimestampMillisecondType;
-use arrow_data::decimal::{MAX_DECIMAL_FOR_EACH_PRECISION,
MIN_DECIMAL_FOR_EACH_PRECISION};
+use arrow_data::decimal::{MAX_DECIMAL128_FOR_EACH_PRECISION,
MIN_DECIMAL128_FOR_EACH_PRECISION};
use chrono::{DateTime, Offset, TimeZone};
/// Preprocesses input arrays to add timezone information from Spark to Arrow
array datatype or
@@ -221,8 +221,8 @@ fn pre_timestamp_cast(array: ArrayRef, timezone: String) ->
Result<ArrayRef, Arr
#[inline]
pub fn is_valid_decimal_precision(value: i128, precision: u8) -> bool {
precision <= DECIMAL128_MAX_PRECISION
- && value >= MIN_DECIMAL_FOR_EACH_PRECISION[precision as usize - 1]
- && value <= MAX_DECIMAL_FOR_EACH_PRECISION[precision as usize - 1]
+ && value >= MIN_DECIMAL128_FOR_EACH_PRECISION[precision as usize]
+ && value <= MAX_DECIMAL128_FOR_EACH_PRECISION[precision as usize]
}
// These are borrowed from hashbrown crate:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
