This is an automated email from the ASF dual-hosted git repository.
timsaucer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-python.git
The following commit(s) were added to refs/heads/main by this push:
new 6c8bf5f Upgrade datafusion (#867)
6c8bf5f is described below
commit 6c8bf5f8aadaf5a3431da272ed6b3afa06efa8ec
Author: Matt Green <[email protected]>
AuthorDate: Tue Sep 17 13:58:11 2024 -0700
Upgrade datafusion (#867)
* update dependencies
* update get_logical_plan signature
* remove row_number() function
row_number was converted to a UDF in datafusion v42
https://github.com/apache/datafusion/pull/12030
This specific functionality needs to be added back in.
* remove unneeded dependency
* fix pyo3 warnings
Implicit defaults for trailing optional arguments have been deprecated
in pyo3 v0.22.0 https://github.com/PyO3/pyo3/pull/4078
* update object_store dependency
* change PyExpr -> PySortExpr
* comment out key.extract::<&PyTuple>() condition statement
* change more instances of PyExpr > PySortExpr
* update function signatures to use _bound versions
* remove clone
* Working through some of the sort requirement changes
* remove unused import
* expr.display_name is deprecated, used format!() + schema_name() instead
* expr.canonical_name() is deprecated, use format!() expr instead
* remove comment
* fix tuple extraction in dataframe.__getitem__()
* remove unneeded import
* Add docstring comments to SortExpr python class
* change extract() to downcast()
Co-authored-by: Michael J Ward <[email protected]>
* deprecate Expr::display_name
Ref: https://github.com/apache/datafusion/pull/11797
* fix lint errors
* update datafusion commit hash
* fix type in cargo file for arrow features
* upgrade to datafusion 42
* cleanup
---------
Co-authored-by: Tim Saucer <[email protected]>
Co-authored-by: Michael J Ward <[email protected]>
Co-authored-by: Michael-J-Ward <[email protected]>
---
Cargo.lock | 784 +++++++++++------------
Cargo.toml | 17 +-
python/datafusion/context.py | 13 +-
python/datafusion/dataframe.py | 8 +-
python/datafusion/expr.py | 83 ++-
python/datafusion/functions.py | 95 +--
python/datafusion/tests/test_expr.py | 34 +-
python/datafusion/tests/test_sql.py | 11 +-
python/datafusion/tests/test_wrapper_coverage.py | 5 +-
src/common/data_type.rs | 8 +-
src/common/schema.rs | 37 +-
src/context.rs | 18 +-
src/dataframe.rs | 14 +-
src/dataset.rs | 2 +-
src/dataset_exec.rs | 4 +-
src/expr.rs | 41 +-
src/expr/sort.rs | 6 +-
src/expr/sort_expr.rs | 21 +-
src/expr/window.rs | 5 +-
src/functions.rs | 87 ++-
src/pyarrow_filter_expression.rs | 2 +-
src/udf.rs | 10 +-
22 files changed, 710 insertions(+), 595 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index d507321..d06073b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4,18 +4,18 @@ version = 3
[[package]]
name = "addr2line"
-version = "0.22.0"
+version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678"
+checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375"
dependencies = [
"gimli",
]
[[package]]
-name = "adler"
-version = "1.0.2"
+name = "adler2"
+version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
[[package]]
name = "adler32"
@@ -84,9 +84,9 @@ dependencies = [
[[package]]
name = "anyhow"
-version = "1.0.86"
+version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
+checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
[[package]]
name = "apache-avro"
@@ -118,21 +118,21 @@ dependencies = [
[[package]]
name = "arrayref"
-version = "0.3.8"
+version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a"
+checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
[[package]]
name = "arrayvec"
-version = "0.7.4"
+version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "05048a8932648b63f21c37d88b552ccc8a65afb6dfe9fc9f30ce79174c2e7a85"
+checksum = "45aef0d9cf9a039bf6cd1acc451b137aca819977b0928dece52bd92811b640ba"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -152,9 +152,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1d8a57966e43bfe9a3277984a14c24ec617ad874e4c0e1d2a1b083a39cfbf22c"
+checksum = "03675e42d1560790f3524800e41403b40d0da1c793fe9528929fde06d8c7649a"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -167,9 +167,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "16f4a9468c882dc66862cef4e1fd8423d47e67972377d85d80e022786427768c"
+checksum = "cd2bf348cf9f02a5975c5962c7fa6dee107a2009a7b41ac5fb1a027e12dc033f"
dependencies = [
"ahash",
"arrow-buffer",
@@ -184,9 +184,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c975484888fc95ec4a632cdc98be39c085b1bb518531b0c80c5d462063e5daa1"
+checksum = "3092e37715f168976012ce52273c3989b5793b0db5f06cbaa246be25e5f0924d"
dependencies = [
"bytes",
"half",
@@ -195,9 +195,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "da26719e76b81d8bc3faad1d4dbdc1bcc10d14704e63dc17fc9f3e7e1e567c8e"
+checksum = "7ce1018bb710d502f9db06af026ed3561552e493e989a79d0d0f5d9cf267a785"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -216,9 +216,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c13c36dc5ddf8c128df19bab27898eea64bf9da2b555ec1cd17a8ff57fba9ec2"
+checksum = "fd178575f45624d045e4ebee714e246a05d9652e41363ee3f57ec18cca97f740"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -235,9 +235,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dd9d6f18c65ef7a2573ab498c374d8ae364b4a4edf67105357491c031f716ca5"
+checksum = "4e4ac0c4ee79150afe067dc4857154b3ee9c1cd52b5f40d59a77306d0ed18d65"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -247,9 +247,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e786e1cdd952205d9a8afc69397b317cfbb6e0095e445c69cda7e8da5c1eeb0f"
+checksum = "bb307482348a1267f91b0912e962cd53440e5de0f7fb24c5f7b10da70b38c94a"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -262,9 +262,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fb22284c5a2a01d73cebfd88a33511a3234ab45d66086b2ca2d1228c3498e445"
+checksum = "d24805ba326758effdd6f2cbdd482fcfab749544f21b134701add25b33f474e6"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -282,9 +282,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "42745f86b1ab99ef96d1c0bcf49180848a64fe2c7a7a0d945bc64fa2b21ba9bc"
+checksum = "644046c479d80ae8ed02a7f1e1399072ea344ca6a7b0e293ab2d5d9ed924aa3b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -297,9 +297,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4cd09a518c602a55bd406bcc291a967b284cfa7a63edfbf8b897ea4748aad23c"
+checksum = "a29791f8eb13b340ce35525b723f5f0df17ecb955599e11f65c2a94ab34e2efb"
dependencies = [
"ahash",
"arrow-array",
@@ -311,18 +311,18 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9e972cd1ff4a4ccd22f86d3e53e835c2ed92e0eea6a3e8eadb72b4f1ac802cf8"
+checksum = "c85320a3a2facf2b2822b57aa9d6d9d55edb8aee0b6b5d3b8df158e503d10858"
dependencies = [
"bitflags 2.6.0",
]
[[package]]
name = "arrow-select"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "600bae05d43483d216fb3494f8c32fdbefd8aa4e1de237e790dbb3d9f44690a3"
+checksum = "9cc7e6b582e23855fd1625ce46e51647aa440c20ea2e71b1d748e0839dd73cba"
dependencies = [
"ahash",
"arrow-array",
@@ -334,9 +334,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f0dc1985b67cb45f6606a248ac2b4a288849f196bab8c657ea5589f47cdd55e6"
+checksum = "0775b6567c66e56ded19b87a954b6b1beffbdd784ef95a3a2b03f59570c1d230"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -375,18 +375,18 @@ checksum =
"3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
name = "async-trait"
-version = "0.1.81"
+version = "0.1.82"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107"
+checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -412,17 +412,17 @@ checksum =
"0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
[[package]]
name = "backtrace"
-version = "0.3.73"
+version = "0.3.74"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a"
+checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a"
dependencies = [
"addr2line",
- "cc",
"cfg-if",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
+ "windows-targets",
]
[[package]]
@@ -460,9 +460,9 @@ dependencies = [
[[package]]
name = "blake3"
-version = "1.5.3"
+version = "1.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e9ec96fe9a81b5e365f9db71fe00edc4fe4ca2cc7dcb7861f0603012a7caa210"
+checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7"
dependencies = [
"arrayref",
"arrayvec",
@@ -542,12 +542,13 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.1.7"
+version = "1.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc"
+checksum = "45bcde016d64c21da4be18b655631e5ab6d3107607e71a73a9f53eb48aae23fb"
dependencies = [
"jobserver",
"libc",
+ "shlex",
]
[[package]]
@@ -566,7 +567,7 @@ dependencies = [
"iana-time-zone",
"num-traits",
"serde",
- "windows-targets 0.52.6",
+ "windows-targets",
]
[[package]]
@@ -593,9 +594,9 @@ dependencies = [
[[package]]
name = "cmake"
-version = "0.1.50"
+version = "0.1.51"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
+checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a"
dependencies = [
"cc",
]
@@ -633,9 +634,9 @@ dependencies = [
[[package]]
name = "constant_time_eq"
-version = "0.3.0"
+version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
+checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
[[package]]
name = "core-foundation"
@@ -649,9 +650,9 @@ dependencies = [
[[package]]
name = "core-foundation-sys"
-version = "0.8.6"
+version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "core2"
@@ -664,9 +665,9 @@ dependencies = [
[[package]]
name = "cpufeatures"
-version = "0.2.12"
+version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
+checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0"
dependencies = [
"libc",
]
@@ -731,9 +732,9 @@ checksum =
"7762d17f1241643615821a8455a0b2c3e803784b058693d990b11f2dce25a0ca"
[[package]]
name = "dashmap"
-version = "6.0.1"
+version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28"
+checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
dependencies = [
"cfg-if",
"crossbeam-utils",
@@ -745,9 +746,9 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e4fd4a99fc70d40ef7e52b243b4a399c3f8d353a40d5ecb200deee05e49c61bb"
+checksum = "ee907b081e45e1d14e1f327e89ef134f91fcebad0bfc2dc229fa9f6044379682"
dependencies = [
"ahash",
"apache-avro",
@@ -769,6 +770,7 @@ dependencies = [
"datafusion-functions",
"datafusion-functions-aggregate",
"datafusion-functions-nested",
+ "datafusion-functions-window",
"datafusion-optimizer",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
@@ -781,7 +783,7 @@ dependencies = [
"half",
"hashbrown",
"indexmap",
- "itertools 0.12.1",
+ "itertools",
"log",
"num-traits",
"num_cpus",
@@ -803,9 +805,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e13b3cfbd84c6003594ae1972314e3df303a27ce8ce755fcea3240c90f4c0529"
+checksum = "6c2b914f6e33c429af7d8696c72a47ed9225d7e2b82c747ebdfa2408ed53579f"
dependencies = [
"arrow-schema",
"async-trait",
@@ -813,13 +815,14 @@ dependencies = [
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-plan",
+ "parking_lot",
]
[[package]]
name = "datafusion-common"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "44fdbc877e3e40dcf88cc8f283d9f5c8851f0a3aa07fee657b1b75ac1ad49b9c"
+checksum = "3a84f8e76330c582a6b8ada0b2c599ca46cfe46b7585e458fc3f4092bc722a18"
dependencies = [
"ahash",
"apache-avro",
@@ -835,24 +838,27 @@ dependencies = [
"num_cpus",
"object_store",
"parquet",
+ "paste",
"pyo3",
"sqlparser",
+ "tokio",
]
[[package]]
name = "datafusion-common-runtime"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8a7496d1f664179f6ce3a5cbef6566056ccaf3ea4aa72cc455f80e62c1dd86b1"
+checksum = "cf08cc30d92720d557df13bd5a5696213bd5ea0f38a866d8d85055d866fba774"
dependencies = [
+ "log",
"tokio",
]
[[package]]
name = "datafusion-execution"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "799e70968c815b611116951e3dd876aef04bf217da31b72eec01ee6a959336a1"
+checksum = "86bc4183d5c45b9f068a6f351678a0d1eb1225181424542bb75db18ec280b822"
dependencies = [
"arrow",
"chrono",
@@ -871,9 +877,9 @@ dependencies = [
[[package]]
name = "datafusion-expr"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1c1841c409d9518c17971d15c9bae62e629eb937e6fb6c68cd32e9186f8b30d2"
+checksum = "202119ce58e4d103e37ae64aab40d4e574c97bdd2bea994bf307b175fcbfa74d"
dependencies = [
"ahash",
"arrow",
@@ -881,6 +887,9 @@ dependencies = [
"arrow-buffer",
"chrono",
"datafusion-common",
+ "datafusion-expr-common",
+ "datafusion-functions-aggregate-common",
+ "datafusion-physical-expr-common",
"paste",
"serde_json",
"sqlparser",
@@ -888,11 +897,22 @@ dependencies = [
"strum_macros 0.26.4",
]
+[[package]]
+name = "datafusion-expr-common"
+version = "42.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f8b181ce8569216abb01ef3294aa16c0a40d7d39350c2ff01ede00f167a535f2"
+dependencies = [
+ "arrow",
+ "datafusion-common",
+ "paste",
+]
+
[[package]]
name = "datafusion-functions"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a8e481cf34d2a444bd8fa09b65945f0ce83dc92df8665b761505b3d9f351bebb"
+checksum = "6e4124b8066444e05a24472f852e94cf56546c0f4d92d00f018f207216902712"
dependencies = [
"arrow",
"arrow-buffer",
@@ -905,7 +925,7 @@ dependencies = [
"datafusion-expr",
"hashbrown",
"hex",
- "itertools 0.12.1",
+ "itertools",
"log",
"md-5",
"rand",
@@ -917,9 +937,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2b4ece19f73c02727e5e8654d79cd5652de371352c1df3c4ac3e419ecd6943fb"
+checksum = "b94acdac235ea21810150a89751617ef2db7e32eba27f54be48a81bde2bfe119"
dependencies = [
"ahash",
"arrow",
@@ -927,17 +947,34 @@ dependencies = [
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
+ "datafusion-functions-aggregate-common",
+ "datafusion-physical-expr",
"datafusion-physical-expr-common",
+ "half",
"log",
"paste",
"sqlparser",
]
+[[package]]
+name = "datafusion-functions-aggregate-common"
+version = "42.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "5c9ea085bbf900bf16e2ca0f56fc56236b2e4f2e1a2cccb67bcd83c5ab4ad0ef"
+dependencies = [
+ "ahash",
+ "arrow",
+ "datafusion-common",
+ "datafusion-expr-common",
+ "datafusion-physical-expr-common",
+ "rand",
+]
+
[[package]]
name = "datafusion-functions-nested"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a1474552cc824e8c9c88177d454db5781d4b66757d4aca75719306b8343a5e8d"
+checksum = "6c882e61665ed60c5ce9b061c1e587aeb8ae5ae4bcb5e5f2465139ab25328e0f"
dependencies = [
"arrow",
"arrow-array",
@@ -949,17 +986,30 @@ dependencies = [
"datafusion-expr",
"datafusion-functions",
"datafusion-functions-aggregate",
- "itertools 0.12.1",
+ "datafusion-physical-expr-common",
+ "itertools",
"log",
"paste",
"rand",
]
+[[package]]
+name = "datafusion-functions-window"
+version = "42.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "98a354ce96df3ca6d025093adac9fd55ca09931c9b6f2630140721a95873fde4"
+dependencies = [
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-physical-expr-common",
+ "log",
+]
+
[[package]]
name = "datafusion-optimizer"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "791ff56f55608bc542d1ea7a68a64bdc86a9413f5a381d06a39fd49c2a3ab906"
+checksum = "baf677c74fb7b5a1899ef52709e4a70fff3ed80bdfb4bbe495909810e83d5f39"
dependencies = [
"arrow",
"async-trait",
@@ -969,7 +1019,7 @@ dependencies = [
"datafusion-physical-expr",
"hashbrown",
"indexmap",
- "itertools 0.12.1",
+ "itertools",
"log",
"paste",
"regex-syntax",
@@ -977,9 +1027,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9a223962b3041304a3e20ed07a21d5de3d88d7e4e71ca192135db6d24e3365a4"
+checksum = "30b077999f6eb6c43d6b25bc66332a3be2f693c382840f008dd763b8540f9530"
dependencies = [
"ahash",
"arrow",
@@ -993,12 +1043,14 @@ dependencies = [
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-functions-aggregate-common",
"datafusion-physical-expr-common",
"half",
"hashbrown",
"hex",
"indexmap",
- "itertools 0.12.1",
+ "itertools",
"log",
"paste",
"petgraph",
@@ -1007,35 +1059,37 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "db5e7d8532a1601cd916881db87a70b0a599900d23f3db2897d389032da53bc6"
+checksum = "dce847f885c2b13bbe29f5c8b7948797131aa470af6e16d2a94f4428b4f4f1bd"
dependencies = [
"ahash",
"arrow",
"datafusion-common",
- "datafusion-expr",
+ "datafusion-expr-common",
"hashbrown",
"rand",
]
[[package]]
name = "datafusion-physical-optimizer"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fdb9c78f308e050f5004671039786a925c3fee83b90004e9fcfd328d7febdcc0"
+checksum = "d13238e3b9fdd62a4c18760bfef714bb990d1e1d3430e9f416aae4b3cfaa71af"
dependencies = [
+ "arrow-schema",
"datafusion-common",
"datafusion-execution",
"datafusion-physical-expr",
"datafusion-physical-plan",
+ "itertools",
]
[[package]]
name = "datafusion-physical-plan"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8d1116949432eb2d30f6362707e2846d942e491052a206f2ddcb42d08aea1ffe"
+checksum = "faba6f55a7eaf0241d07d12c2640de52742646b10f754485d5192bdfe2c9ceae"
dependencies = [
"ahash",
"arrow",
@@ -1050,13 +1104,14 @@ dependencies = [
"datafusion-execution",
"datafusion-expr",
"datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"futures",
"half",
"hashbrown",
"indexmap",
- "itertools 0.12.1",
+ "itertools",
"log",
"once_cell",
"parking_lot",
@@ -1083,7 +1138,7 @@ dependencies = [
"pyo3-build-config",
"rand",
"regex-syntax",
- "syn 2.0.72",
+ "syn",
"tokio",
"url",
"uuid",
@@ -1091,9 +1146,9 @@ dependencies = [
[[package]]
name = "datafusion-sql"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b45d0180711165fe94015d7c4123eb3e1cf5fb60b1506453200b8d1ce666bef0"
+checksum = "dad8d96a9b52e1aa24f9373696a815be828193efce7cb0bbd2140b6bb67d1819"
dependencies = [
"arrow",
"arrow-array",
@@ -1108,15 +1163,15 @@ dependencies = [
[[package]]
name = "datafusion-substrait"
-version = "41.0.0"
+version = "42.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bf0a0055aa98246c79f98f0d03df11f16cb7adc87818d02d4413e3f3cdadbbee"
+checksum = "f92b1b80e98bf5a9921bf118816e0e766d18527e343153321fcccfe4d68c5c45"
dependencies = [
"arrow-buffer",
"async-recursion",
"chrono",
"datafusion",
- "itertools 0.12.1",
+ "itertools",
"object_store",
"pbjson-types",
"prost",
@@ -1135,12 +1190,6 @@ dependencies = [
"subtle",
]
-[[package]]
-name = "doc-comment"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
-
[[package]]
name = "dyn-clone"
version = "1.0.17"
@@ -1171,9 +1220,9 @@ dependencies = [
[[package]]
name = "fastrand"
-version = "2.1.0"
+version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
+checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
[[package]]
name = "fixedbitset"
@@ -1193,9 +1242,9 @@ dependencies = [
[[package]]
name = "flate2"
-version = "1.0.31"
+version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920"
+checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253"
dependencies = [
"crc32fast",
"miniz_oxide",
@@ -1272,7 +1321,7 @@ checksum =
"87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -1328,9 +1377,9 @@ dependencies = [
[[package]]
name = "gimli"
-version = "0.29.0"
+version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"
+checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64"
[[package]]
name = "glob"
@@ -1340,9 +1389,9 @@ checksum =
"d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "h2"
-version = "0.4.5"
+version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab"
+checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205"
dependencies = [
"atomic-waker",
"bytes",
@@ -1470,16 +1519,16 @@ dependencies = [
[[package]]
name = "hyper-rustls"
-version = "0.27.2"
+version = "0.27.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155"
+checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
dependencies = [
"futures-util",
"http",
"hyper",
"hyper-util",
"rustls",
- "rustls-native-certs",
+ "rustls-native-certs 0.8.0",
"rustls-pki-types",
"tokio",
"tokio-rustls",
@@ -1488,9 +1537,9 @@ dependencies = [
[[package]]
name = "hyper-util"
-version = "0.1.7"
+version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9"
+checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba"
dependencies = [
"bytes",
"futures-channel",
@@ -1508,9 +1557,9 @@ dependencies = [
[[package]]
name = "iana-time-zone"
-version = "0.1.60"
+version = "0.1.61"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141"
+checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
dependencies = [
"android_system_properties",
"core-foundation-sys",
@@ -1541,9 +1590,9 @@ dependencies = [
[[package]]
name = "indexmap"
-version = "2.3.0"
+version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0"
+checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5"
dependencies = [
"equivalent",
"hashbrown",
@@ -1575,27 +1624,9 @@ checksum =
"8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
[[package]]
name = "ipnet"
-version = "2.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
-
-[[package]]
-name = "itertools"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
-dependencies = [
- "either",
-]
-
-[[package]]
-name = "itertools"
-version = "0.12.1"
+version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
-dependencies = [
- "either",
-]
+checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4"
[[package]]
name = "itertools"
@@ -1623,9 +1654,9 @@ dependencies = [
[[package]]
name = "js-sys"
-version = "0.3.69"
+version = "0.3.70"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
+checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
dependencies = [
"wasm-bindgen",
]
@@ -1702,9 +1733,9 @@ dependencies = [
[[package]]
name = "libc"
-version = "0.2.155"
+version = "0.2.158"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
[[package]]
name = "libflate"
@@ -1830,18 +1861,18 @@ checksum =
"6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "miniz_oxide"
-version = "0.7.4"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"
+checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
dependencies = [
- "adler",
+ "adler2",
]
[[package]]
name = "mio"
-version = "1.0.1"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4"
+checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
dependencies = [
"hermit-abi",
"libc",
@@ -1941,18 +1972,18 @@ dependencies = [
[[package]]
name = "object"
-version = "0.36.2"
+version = "0.36.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e"
+checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a"
dependencies = [
"memchr",
]
[[package]]
name = "object_store"
-version = "0.10.2"
+version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3"
+checksum = "25a0c4b3a0e31f8b66f71ad8064521efa773910196e2cde791436f13409f3b45"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -1961,7 +1992,7 @@ dependencies = [
"futures",
"humantime",
"hyper",
- "itertools 0.13.0",
+ "itertools",
"md-5",
"parking_lot",
"percent-encoding",
@@ -2020,14 +2051,14 @@ dependencies = [
"libc",
"redox_syscall",
"smallvec",
- "windows-targets 0.52.6",
+ "windows-targets",
]
[[package]]
name = "parquet"
-version = "52.2.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e977b9066b4d3b03555c22bdc442f3fadebd96a39111249113087d0edb2691cd"
+checksum = "f0fbf928021131daaa57d334ca8e3904fe9ae22f73c56244fc7db9b04eedc3d8"
dependencies = [
"ahash",
"arrow-array",
@@ -2076,9 +2107,9 @@ checksum =
"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "pbjson"
-version = "0.6.0"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1030c719b0ec2a2d25a5df729d6cff1acf3cc230bf766f4f97833591f7577b90"
+checksum = "c7e6349fa080353f4a597daffd05cb81572a9c031a6d4fff7e504947496fcc68"
dependencies = [
"base64 0.21.7",
"serde",
@@ -2086,21 +2117,21 @@ dependencies = [
[[package]]
name = "pbjson-build"
-version = "0.6.2"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735"
+checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9"
dependencies = [
- "heck 0.4.1",
- "itertools 0.11.0",
+ "heck 0.5.0",
+ "itertools",
"prost",
"prost-types",
]
[[package]]
name = "pbjson-types"
-version = "0.6.0"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "18f596653ba4ac51bdecbb4ef6773bc7f56042dc13927910de1684ad3d32aa12"
+checksum = "e54e5e7bfb1652f95bc361d76f3c780d8e526b134b85417e774166ee941f0887"
dependencies = [
"bytes",
"chrono",
@@ -2182,7 +2213,7 @@ checksum =
"2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -2220,12 +2251,12 @@ dependencies = [
[[package]]
name = "prettyplease"
-version = "0.2.20"
+version = "0.2.22"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e"
+checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba"
dependencies = [
"proc-macro2",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -2239,9 +2270,9 @@ dependencies = [
[[package]]
name = "prost"
-version = "0.12.6"
+version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29"
+checksum = "3b2ecbe40f08db5c006b5764a2645f7f3f141ce756412ac9e1dd6087e6d32995"
dependencies = [
"bytes",
"prost-derive",
@@ -2249,13 +2280,13 @@ dependencies = [
[[package]]
name = "prost-build"
-version = "0.12.6"
+version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
+checksum = "f8650aabb6c35b860610e9cff5dc1af886c9e25073b7b1712a68972af4281302"
dependencies = [
"bytes",
"heck 0.5.0",
- "itertools 0.12.1",
+ "itertools",
"log",
"multimap",
"once_cell",
@@ -2264,28 +2295,28 @@ dependencies = [
"prost",
"prost-types",
"regex",
- "syn 2.0.72",
+ "syn",
"tempfile",
]
[[package]]
name = "prost-derive"
-version = "0.12.6"
+version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1"
+checksum = "acf0c195eebb4af52c752bec4f52f645da98b6e92077a04110c7f349477ae5ac"
dependencies = [
"anyhow",
- "itertools 0.12.1",
+ "itertools",
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
name = "prost-types"
-version = "0.12.6"
+version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0"
+checksum = "60caa6738c7369b940c3d49246a8d1749323674c65cb13010134f5c9bad5b519"
dependencies = [
"prost",
]
@@ -2301,15 +2332,15 @@ dependencies = [
[[package]]
name = "pyo3"
-version = "0.21.2"
+version = "0.22.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8"
+checksum = "15ee168e30649f7f234c3d49ef5a7a6cbf5134289bc46c29ff3155fa3221c225"
dependencies = [
"cfg-if",
"indoc",
"libc",
"memoffset",
- "parking_lot",
+ "once_cell",
"portable-atomic",
"pyo3-build-config",
"pyo3-ffi",
@@ -2319,9 +2350,9 @@ dependencies = [
[[package]]
name = "pyo3-build-config"
-version = "0.21.2"
+version = "0.22.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50"
+checksum = "e61cef80755fe9e46bb8a0b8f20752ca7676dcc07a5277d8b7768c6172e529b3"
dependencies = [
"once_cell",
"target-lexicon",
@@ -2329,9 +2360,9 @@ dependencies = [
[[package]]
name = "pyo3-ffi"
-version = "0.21.2"
+version = "0.22.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403"
+checksum = "67ce096073ec5405f5ee2b8b31f03a68e02aa10d5d4f565eca04acc41931fa1c"
dependencies = [
"libc",
"pyo3-build-config",
@@ -2339,34 +2370,34 @@ dependencies = [
[[package]]
name = "pyo3-macros"
-version = "0.21.2"
+version = "0.22.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c"
+checksum = "2440c6d12bc8f3ae39f1e775266fa5122fd0c8891ce7520fa6048e683ad3de28"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
name = "pyo3-macros-backend"
-version = "0.21.2"
+version = "0.22.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c"
+checksum = "1be962f0e06da8f8465729ea2cb71a416d2257dff56cbe40a70d3e62a93ae5d1"
dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
"proc-macro2",
"pyo3-build-config",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
name = "quad-rand"
-version = "0.2.1"
+version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88"
+checksum = "b76f1009795ca44bb5aaae8fd3f18953e209259c33d9b059b1f53d58ab7511db"
[[package]]
name = "quick-xml"
@@ -2380,9 +2411,9 @@ dependencies = [
[[package]]
name = "quinn"
-version = "0.11.3"
+version = "0.11.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156"
+checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684"
dependencies = [
"bytes",
"pin-project-lite",
@@ -2398,9 +2429,9 @@ dependencies = [
[[package]]
name = "quinn-proto"
-version = "0.11.6"
+version = "0.11.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd"
+checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6"
dependencies = [
"bytes",
"rand",
@@ -2415,22 +2446,22 @@ dependencies = [
[[package]]
name = "quinn-udp"
-version = "0.5.4"
+version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285"
+checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b"
dependencies = [
"libc",
"once_cell",
"socket2",
"tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
name = "quote"
-version = "1.0.36"
+version = "1.0.37"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
dependencies = [
"proc-macro2",
]
@@ -2467,9 +2498,9 @@ dependencies = [
[[package]]
name = "redox_syscall"
-version = "0.5.3"
+version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4"
+checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853"
dependencies = [
"bitflags 2.6.0",
]
@@ -2521,9 +2552,9 @@ dependencies = [
[[package]]
name = "reqwest"
-version = "0.12.5"
+version = "0.12.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37"
+checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -2545,7 +2576,7 @@ dependencies = [
"pin-project-lite",
"quinn",
"rustls",
- "rustls-native-certs",
+ "rustls-native-certs 0.7.3",
"rustls-pemfile",
"rustls-pki-types",
"serde",
@@ -2561,7 +2592,7 @@ dependencies = [
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
- "winreg",
+ "windows-registry",
]
[[package]]
@@ -2599,18 +2630,18 @@ checksum =
"583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152"
[[package]]
name = "rustc_version"
-version = "0.4.0"
+version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
dependencies = [
"semver",
]
[[package]]
name = "rustix"
-version = "0.38.34"
+version = "0.38.37"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
+checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
dependencies = [
"bitflags 2.6.0",
"errno",
@@ -2621,9 +2652,9 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.23.12"
+version = "0.23.13"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044"
+checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8"
dependencies = [
"once_cell",
"ring",
@@ -2635,9 +2666,22 @@ dependencies = [
[[package]]
name = "rustls-native-certs"
-version = "0.7.1"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5"
+dependencies = [
+ "openssl-probe",
+ "rustls-pemfile",
+ "rustls-pki-types",
+ "schannel",
+ "security-framework",
+]
+
+[[package]]
+name = "rustls-native-certs"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba"
+checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a"
dependencies = [
"openssl-probe",
"rustls-pemfile",
@@ -2664,9 +2708,9 @@ checksum =
"fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"
[[package]]
name = "rustls-webpki"
-version = "0.102.6"
+version = "0.102.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e"
+checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
dependencies = [
"ring",
"rustls-pki-types",
@@ -2696,11 +2740,11 @@ dependencies = [
[[package]]
name = "schannel"
-version = "0.1.23"
+version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"
+checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b"
dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -2724,7 +2768,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -2773,22 +2817,22 @@ checksum =
"a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
[[package]]
name = "serde"
-version = "1.0.204"
+version = "1.0.210"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
+checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
-version = "1.0.204"
+version = "1.0.210"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
+checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -2799,14 +2843,14 @@ checksum =
"18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
name = "serde_json"
-version = "1.0.122"
+version = "1.0.128"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
+checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
dependencies = [
"itoa",
"memchr",
@@ -2816,14 +2860,14 @@ dependencies = [
[[package]]
name = "serde_tokenstream"
-version = "0.2.1"
+version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8790a7c3fe883e443eaa2af6f705952bc5d6e8671a220b9335c8cae92c037e74"
+checksum = "64060d864397305347a78851c51588fd283767e7e7589829e8121d65512340f1"
dependencies = [
"proc-macro2",
"quote",
"serde",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -2862,6 +2906,12 @@ dependencies = [
"digest",
]
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
[[package]]
name = "siphasher"
version = "0.3.11"
@@ -2885,24 +2935,23 @@ checksum =
"3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]]
name = "snafu"
-version = "0.7.5"
+version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6"
+checksum = "2b835cb902660db3415a672d862905e791e54d306c6e8189168c7f3d9ae1c79d"
dependencies = [
- "doc-comment",
"snafu-derive",
]
[[package]]
name = "snafu-derive"
-version = "0.7.5"
+version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf"
+checksum = "38d1e02fca405f6280643174a50c942219f0bbf4dbf7d480f1dd864d6f211ae5"
dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
"proc-macro2",
"quote",
- "syn 1.0.109",
+ "syn",
]
[[package]]
@@ -2929,9 +2978,9 @@ checksum =
"6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
[[package]]
name = "sqlparser"
-version = "0.49.0"
+version = "0.50.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a4a404d0e14905361b918cb8afdb73605e25c1d5029312bd9785142dcb3aa49e"
+checksum = "b2e5b515a2bd5168426033e9efbfd05500114833916f1d5c268f938b4ee130ac"
dependencies = [
"log",
"sqlparser_derive",
@@ -2945,7 +2994,7 @@ checksum =
"01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -2979,7 +3028,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -2992,14 +3041,14 @@ dependencies = [
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.72",
+ "syn",
]
[[package]]
name = "substrait"
-version = "0.36.0"
+version = "0.41.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b1ee6e584c8bf37104b7eb51c25eae07a9321b0e01379bec3b7c462d2f42afbf"
+checksum = "2a3bf05f1d7a3fd7a97790d410f6e859b3a98dcde05e7a3fc00b31b0f60fe7cb"
dependencies = [
"heck 0.5.0",
"pbjson",
@@ -3015,7 +3064,7 @@ dependencies = [
"serde",
"serde_json",
"serde_yaml",
- "syn 2.0.72",
+ "syn",
"typify",
"walkdir",
]
@@ -3028,20 +3077,9 @@ checksum =
"13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "syn"
-version = "1.0.109"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.72"
+version = "2.0.77"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
+checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed"
dependencies = [
"proc-macro2",
"quote",
@@ -3053,6 +3091,9 @@ name = "sync_wrapper"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
+dependencies = [
+ "futures-core",
+]
[[package]]
name = "target-lexicon"
@@ -3062,15 +3103,15 @@ checksum =
"61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
[[package]]
name = "tempfile"
-version = "3.11.0"
+version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b8fcd239983515c23a32fb82099f97d0b11b8c72f654ed659363a95c3dad7a53"
+checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
dependencies = [
"cfg-if",
"fastrand",
"once_cell",
"rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -3090,7 +3131,7 @@ checksum =
"a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -3130,9 +3171,9 @@ checksum =
"1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
-version = "1.39.2"
+version = "1.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1"
+checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998"
dependencies = [
"backtrace",
"bytes",
@@ -3152,7 +3193,7 @@ checksum =
"693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -3168,9 +3209,9 @@ dependencies = [
[[package]]
name = "tokio-util"
-version = "0.7.11"
+version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1"
+checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a"
dependencies = [
"bytes",
"futures-core",
@@ -3196,15 +3237,15 @@ dependencies = [
[[package]]
name = "tower-layer"
-version = "0.3.2"
+version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
[[package]]
name = "tower-service"
-version = "0.3.2"
+version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"
@@ -3225,7 +3266,7 @@ checksum =
"34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -3270,7 +3311,7 @@ checksum =
"f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -3304,7 +3345,7 @@ dependencies = [
"semver",
"serde",
"serde_json",
- "syn 2.0.72",
+ "syn",
"thiserror",
"unicode-ident",
]
@@ -3322,7 +3363,7 @@ dependencies = [
"serde",
"serde_json",
"serde_tokenstream",
- "syn 2.0.72",
+ "syn",
"typify-impl",
]
@@ -3334,24 +3375,24 @@ checksum =
"08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
[[package]]
name = "unicode-ident"
-version = "1.0.12"
+version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
[[package]]
name = "unicode-normalization"
-version = "0.1.23"
+version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
+checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-segmentation"
-version = "1.11.0"
+version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
[[package]]
name = "unicode-width"
@@ -3431,34 +3472,35 @@ checksum =
"9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
-version = "0.2.92"
+version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
+checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
dependencies = [
"cfg-if",
+ "once_cell",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
-version = "0.2.92"
+version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
+checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.42"
+version = "0.4.43"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
+checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed"
dependencies = [
"cfg-if",
"js-sys",
@@ -3468,9 +3510,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.92"
+version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
+checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -3478,22 +3520,22 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.92"
+version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
+checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.92"
+version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
+checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
[[package]]
name = "wasm-streams"
@@ -3510,9 +3552,9 @@ dependencies = [
[[package]]
name = "web-sys"
-version = "0.3.69"
+version = "0.3.70"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
+checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -3533,49 +3575,55 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
dependencies = [
- "windows-targets 0.52.6",
+ "windows-targets",
]
[[package]]
-name = "windows-sys"
-version = "0.48.0"
+name = "windows-registry"
+version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
- "windows-targets 0.48.5",
+ "windows-result",
+ "windows-strings",
+ "windows-targets",
]
[[package]]
-name = "windows-sys"
-version = "0.52.0"
+name = "windows-result"
+version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
- "windows-targets 0.52.6",
+ "windows-result",
+ "windows-targets",
]
[[package]]
name = "windows-sys"
-version = "0.59.0"
+version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
- "windows-targets 0.52.6",
+ "windows-targets",
]
[[package]]
-name = "windows-targets"
-version = "0.48.5"
+name = "windows-sys"
+version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
- "windows_aarch64_gnullvm 0.48.5",
- "windows_aarch64_msvc 0.48.5",
- "windows_i686_gnu 0.48.5",
- "windows_i686_msvc 0.48.5",
- "windows_x86_64_gnu 0.48.5",
- "windows_x86_64_gnullvm 0.48.5",
- "windows_x86_64_msvc 0.48.5",
+ "windows-targets",
]
[[package]]
@@ -3584,46 +3632,28 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
- "windows_aarch64_gnullvm 0.52.6",
- "windows_aarch64_msvc 0.52.6",
- "windows_i686_gnu 0.52.6",
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
"windows_i686_gnullvm",
- "windows_i686_msvc 0.52.6",
- "windows_x86_64_gnu 0.52.6",
- "windows_x86_64_gnullvm 0.52.6",
- "windows_x86_64_msvc 0.52.6",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
]
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
-
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
-
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
-[[package]]
-name = "windows_i686_gnu"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
-
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
@@ -3636,64 +3666,30 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
-[[package]]
-name = "windows_i686_msvc"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
-
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
-
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
-
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
-
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
-[[package]]
-name = "winreg"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5"
-dependencies = [
- "cfg-if",
- "windows-sys 0.48.0",
-]
-
[[package]]
name = "xz2"
version = "0.1.7"
@@ -3721,7 +3717,7 @@ checksum =
"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.72",
+ "syn",
]
[[package]]
@@ -3769,9 +3765,9 @@ dependencies = [
[[package]]
name = "zstd-sys"
-version = "2.0.12+zstd.1.5.6"
+version = "2.0.13+zstd.1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13"
+checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
dependencies = [
"cc",
"pkg-config",
diff --git a/Cargo.toml b/Cargo.toml
index 8635776..11dccc4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -36,24 +36,24 @@ substrait = ["dep:datafusion-substrait"]
[dependencies]
tokio = { version = "1.39", features = ["macros", "rt", "rt-multi-thread",
"sync"] }
rand = "0.8"
-pyo3 = { version = "0.21", features = ["extension-module", "abi3",
"abi3-py38"] }
-arrow = { version = "52", feature = ["pyarrow"] }
-datafusion = { version = "41.0.0", features = ["pyarrow", "avro",
"unicode_expressions"] }
-datafusion-substrait = { version = "41.0.0", optional = true }
-prost = "0.12" # keep in line with `datafusion-substrait`
-prost-types = "0.12" # keep in line with `datafusion-substrait`
+pyo3 = { version = "0.22", features = ["extension-module", "abi3",
"abi3-py38"] }
+arrow = { version = "53", features = ["pyarrow"] }
+datafusion = { version = "42.0.0", features = ["pyarrow", "avro",
"unicode_expressions"] }
+datafusion-substrait = { version = "42.0.0", optional = true }
+prost = "0.13" # keep in line with `datafusion-substrait`
+prost-types = "0.13" # keep in line with `datafusion-substrait`
uuid = { version = "1.9", features = ["v4"] }
mimalloc = { version = "0.1", optional = true, default-features = false,
features = ["local_dynamic_tls"] }
async-trait = "0.1"
futures = "0.3"
-object_store = { version = "0.10.1", features = ["aws", "gcp", "azure"] }
+object_store = { version = "0.11.0", features = ["aws", "gcp", "azure"] }
parking_lot = "0.12"
regex-syntax = "0.8"
syn = "2.0.68"
url = "2"
[build-dependencies]
-pyo3-build-config = "0.21"
+pyo3-build-config = "0.22"
[lib]
name = "datafusion_python"
@@ -62,4 +62,3 @@ crate-type = ["cdylib", "rlib"]
[profile.release]
lto = true
codegen-units = 1
-
diff --git a/python/datafusion/context.py b/python/datafusion/context.py
index 903d4a1..35a40cc 100644
--- a/python/datafusion/context.py
+++ b/python/datafusion/context.py
@@ -28,7 +28,7 @@ from ._internal import LogicalPlan, ExecutionPlan
from datafusion._internal import AggregateUDF
from datafusion.catalog import Catalog, Table
from datafusion.dataframe import DataFrame
-from datafusion.expr import Expr
+from datafusion.expr import Expr, SortExpr, sort_list_to_raw_sort_list
from datafusion.record_batch import RecordBatchStream
from datafusion.udf import ScalarUDF
@@ -466,7 +466,7 @@ class SessionContext:
table_partition_cols: list[tuple[str, str]] | None = None,
file_extension: str = ".parquet",
schema: pyarrow.Schema | None = None,
- file_sort_order: list[list[Expr]] | None = None,
+ file_sort_order: list[list[Expr | SortExpr]] | None = None,
) -> None:
"""Register multiple files as a single table.
@@ -484,15 +484,18 @@ class SessionContext:
"""
if table_partition_cols is None:
table_partition_cols = []
- if file_sort_order is not None:
- file_sort_order = [[x.expr for x in xs] for xs in file_sort_order]
+ file_sort_order_raw = (
+ [sort_list_to_raw_sort_list(f) for f in file_sort_order]
+ if file_sort_order is not None
+ else None
+ )
self.ctx.register_listing_table(
name,
str(path),
table_partition_cols,
file_extension,
schema,
- file_sort_order,
+ file_sort_order_raw,
)
def sql(self, query: str, options: SQLOptions | None = None) -> DataFrame:
diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py
index 56dff22..2328ef8 100644
--- a/python/datafusion/dataframe.py
+++ b/python/datafusion/dataframe.py
@@ -33,7 +33,7 @@ if TYPE_CHECKING:
from typing import Callable
from datafusion._internal import DataFrame as DataFrameInternal
-from datafusion.expr import Expr
+from datafusion.expr import Expr, SortExpr, sort_or_default
from datafusion._internal import (
LogicalPlan,
ExecutionPlan,
@@ -199,7 +199,7 @@ class DataFrame:
aggs = [e.expr for e in aggs]
return DataFrame(self.df.aggregate(group_by, aggs))
- def sort(self, *exprs: Expr) -> DataFrame:
+ def sort(self, *exprs: Expr | SortExpr) -> DataFrame:
"""Sort the DataFrame by the specified sorting expressions.
Note that any expression can be turned into a sort expression by
@@ -211,8 +211,8 @@ class DataFrame:
Returns:
DataFrame after sorting.
"""
- exprs = [expr.expr for expr in exprs]
- return DataFrame(self.df.sort(*exprs))
+ exprs_raw = [sort_or_default(expr) for expr in exprs]
+ return DataFrame(self.df.sort(*exprs_raw))
def limit(self, count: int, offset: int = 0) -> DataFrame:
"""Return a new :py:class:`DataFrame` with a limited number of rows.
diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py
index f6a51ce..fd5e6f0 100644
--- a/python/datafusion/expr.py
+++ b/python/datafusion/expr.py
@@ -22,14 +22,15 @@ See :ref:`Expressions` in the online documentation for more
details.
from __future__ import annotations
-from ._internal import (
- expr as expr_internal,
- LogicalPlan,
- functions as functions_internal,
-)
-from datafusion.common import NullTreatment, RexType, DataTypeMap
from typing import Any, Optional, Type
+
import pyarrow as pa
+from datafusion.common import DataTypeMap, NullTreatment, RexType
+from typing_extensions import deprecated
+
+from ._internal import LogicalPlan
+from ._internal import expr as expr_internal
+from ._internal import functions as functions_internal
# The following are imported from the internal representation. We may choose to
# give these all proper wrappers, or to simply leave as is. These were added
@@ -84,7 +85,6 @@ ScalarSubquery = expr_internal.ScalarSubquery
ScalarVariable = expr_internal.ScalarVariable
SimilarTo = expr_internal.SimilarTo
Sort = expr_internal.Sort
-SortExpr = expr_internal.SortExpr
Subquery = expr_internal.Subquery
SubqueryAlias = expr_internal.SubqueryAlias
TableScan = expr_internal.TableScan
@@ -159,6 +159,27 @@ __all__ = [
]
+def expr_list_to_raw_expr_list(
+ expr_list: Optional[list[Expr]],
+) -> Optional[list[expr_internal.Expr]]:
+ """Helper function to convert an optional list to raw expressions."""
+ return [e.expr for e in expr_list] if expr_list is not None else None
+
+
+def sort_or_default(e: Expr | SortExpr) -> expr_internal.SortExpr:
+ """Helper function to return a default Sort if an Expr is provided."""
+ if isinstance(e, SortExpr):
+ return e.raw_sort
+ return SortExpr(e.expr, True, True).raw_sort
+
+
+def sort_list_to_raw_sort_list(
+ sort_list: Optional[list[Expr | SortExpr]],
+) -> Optional[list[expr_internal.SortExpr]]:
+ """Helper function to return an optional sort list to raw variant."""
+ return [sort_or_default(e) for e in sort_list] if sort_list is not None
else None
+
+
class Expr:
"""Expression object.
@@ -174,12 +195,22 @@ class Expr:
"""Convert this expression into a python object if possible."""
return self.expr.to_variant()
+ @deprecated(
+ "display_name() is deprecated. Use :py:meth:`~Expr.schema_name`
instead"
+ )
def display_name(self) -> str:
"""Returns the name of this expression as it should appear in a schema.
This name will not include any CAST expressions.
"""
- return self.expr.display_name()
+ return self.schema_name()
+
+ def schema_name(self) -> str:
+ """Returns the name of this expression as it should appear in a schema.
+
+ This name will not include any CAST expressions.
+ """
+ return self.expr.schema_name()
def canonical_name(self) -> str:
"""Returns a complete string representation of this expression."""
@@ -355,14 +386,14 @@ class Expr:
"""Assign a name to the expression."""
return Expr(self.expr.alias(name))
- def sort(self, ascending: bool = True, nulls_first: bool = True) -> Expr:
+ def sort(self, ascending: bool = True, nulls_first: bool = True) ->
SortExpr:
"""Creates a sort :py:class:`Expr` from an existing :py:class:`Expr`.
Args:
ascending: If true, sort in ascending order.
nulls_first: Return null values first.
"""
- return Expr(self.expr.sort(ascending=ascending,
nulls_first=nulls_first))
+ return SortExpr(self.expr, ascending=ascending,
nulls_first=nulls_first)
def is_null(self) -> Expr:
"""Returns ``True`` if this expression is null."""
@@ -455,14 +486,14 @@ class Expr:
"""Compute the output column name based on the provided logical
plan."""
return self.expr.column_name(plan)
- def order_by(self, *exprs: Expr) -> ExprFuncBuilder:
+ def order_by(self, *exprs: Expr | SortExpr) -> ExprFuncBuilder:
"""Set the ordering for a window or aggregate function.
This function will create an :py:class:`ExprFuncBuilder` that can be
used to
set parameters for either window or aggregate functions. If used on
any other
type of expression, an error will be generated when ``build()`` is
called.
"""
- return ExprFuncBuilder(self.expr.order_by(list(e.expr for e in exprs)))
+ return ExprFuncBuilder(self.expr.order_by([sort_or_default(e) for e in
exprs]))
def filter(self, filter: Expr) -> ExprFuncBuilder:
"""Filter an aggregate function.
@@ -522,7 +553,9 @@ class ExprFuncBuilder:
Values given in ``exprs`` must be sort expressions. You can convert
any other
expression to a sort expression using `.sort()`.
"""
- return ExprFuncBuilder(self.builder.order_by(list(e.expr for e in
exprs)))
+ return ExprFuncBuilder(
+ self.builder.order_by([sort_or_default(e) for e in exprs])
+ )
def filter(self, filter: Expr) -> ExprFuncBuilder:
"""Filter values during aggregation."""
@@ -659,3 +692,27 @@ class CaseBuilder:
Any non-matching cases will end in a `null` value.
"""
return Expr(self.case_builder.end())
+
+
+class SortExpr:
+ """Used to specify sorting on either a DataFrame or function."""
+
+ def __init__(self, expr: Expr, ascending: bool, nulls_first: bool) -> None:
+ """This constructor should not be called by the end user."""
+ self.raw_sort = expr_internal.SortExpr(expr, ascending, nulls_first)
+
+ def expr(self) -> Expr:
+ """Return the raw expr backing the SortExpr."""
+ return Expr(self.raw_sort.expr())
+
+ def ascending(self) -> bool:
+ """Return ascending property."""
+ return self.raw_sort.ascending()
+
+ def nulls_first(self) -> bool:
+ """Return nulls_first property."""
+ return self.raw_sort.nulls_first()
+
+ def __repr__(self) -> str:
+ """Generate a string representation of this expression."""
+ return self.raw_sort.__repr__()
diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py
index 163ff04..0401afb 100644
--- a/python/datafusion/functions.py
+++ b/python/datafusion/functions.py
@@ -18,8 +18,15 @@
from __future__ import annotations
-from datafusion._internal import functions as f, expr as expr_internal
-from datafusion.expr import CaseBuilder, Expr, WindowFrame
+from datafusion._internal import functions as f
+from datafusion.expr import (
+ CaseBuilder,
+ Expr,
+ WindowFrame,
+ SortExpr,
+ sort_list_to_raw_sort_list,
+ expr_list_to_raw_expr_list,
+)
from datafusion.context import SessionContext
from datafusion.common import NullTreatment
@@ -261,12 +268,6 @@ __all__ = [
]
-def expr_list_to_raw_expr_list(
- expr_list: Optional[list[Expr]],
-) -> Optional[list[expr_internal.Expr]]:
- return [e.expr for e in expr_list] if expr_list is not None else None
-
-
def isnan(expr: Expr) -> Expr:
"""Returns true if a given number is +NaN or -NaN otherwise returns
false."""
return Expr(f.isnan(expr.expr))
@@ -352,9 +353,9 @@ def concat_ws(separator: str, *args: Expr) -> Expr:
return Expr(f.concat_ws(separator, args))
-def order_by(expr: Expr, ascending: bool = True, nulls_first: bool = True) ->
Expr:
+def order_by(expr: Expr, ascending: bool = True, nulls_first: bool = True) ->
SortExpr:
"""Creates a new sort expression."""
- return Expr(f.order_by(expr.expr, ascending, nulls_first))
+ return SortExpr(expr.expr, ascending=ascending, nulls_first=nulls_first)
def alias(expr: Expr, name: str) -> Expr:
@@ -405,7 +406,7 @@ def window(
name: str,
args: list[Expr],
partition_by: list[Expr] | None = None,
- order_by: list[Expr] | None = None,
+ order_by: list[Expr | SortExpr] | None = None,
window_frame: WindowFrame | None = None,
ctx: SessionContext | None = None,
) -> Expr:
@@ -419,9 +420,9 @@ def window(
"""
args = [a.expr for a in args]
partition_by = expr_list_to_raw_expr_list(partition_by)
- order_by = expr_list_to_raw_expr_list(order_by)
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
window_frame = window_frame.window_frame if window_frame is not None else
None
- return Expr(f.window(name, args, partition_by, order_by, window_frame,
ctx))
+ return Expr(f.window(name, args, partition_by, order_by_raw, window_frame,
ctx))
# scalar functions
@@ -1608,7 +1609,7 @@ def array_agg(
expression: Expr,
distinct: bool = False,
filter: Optional[Expr] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
) -> Expr:
"""Aggregate values into an array.
@@ -1625,7 +1626,7 @@ def array_agg(
filter: If provided, only compute against rows for which the filter is
True
order_by: Order the resultant array values
"""
- order_by_raw = expr_list_to_raw_expr_list(order_by)
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
filter_raw = filter.expr if filter is not None else None
return Expr(
@@ -2107,7 +2108,7 @@ def regr_syy(
def first_value(
expression: Expr,
filter: Optional[Expr] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
null_treatment: NullTreatment = NullTreatment.RESPECT_NULLS,
) -> Expr:
"""Returns the first value in a group of values.
@@ -2123,7 +2124,7 @@ def first_value(
order_by: Set the ordering of the expression to evaluate
null_treatment: Assign whether to respect or ignull null values.
"""
- order_by_raw = expr_list_to_raw_expr_list(order_by)
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
filter_raw = filter.expr if filter is not None else None
return Expr(
@@ -2139,7 +2140,7 @@ def first_value(
def last_value(
expression: Expr,
filter: Optional[Expr] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
null_treatment: NullTreatment = NullTreatment.RESPECT_NULLS,
) -> Expr:
"""Returns the last value in a group of values.
@@ -2155,7 +2156,7 @@ def last_value(
order_by: Set the ordering of the expression to evaluate
null_treatment: Assign whether to respect or ignull null values.
"""
- order_by_raw = expr_list_to_raw_expr_list(order_by)
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
filter_raw = filter.expr if filter is not None else None
return Expr(
@@ -2172,7 +2173,7 @@ def nth_value(
expression: Expr,
n: int,
filter: Optional[Expr] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
null_treatment: NullTreatment = NullTreatment.RESPECT_NULLS,
) -> Expr:
"""Returns the n-th value in a group of values.
@@ -2189,7 +2190,7 @@ def nth_value(
order_by: Set the ordering of the expression to evaluate
null_treatment: Assign whether to respect or ignull null values.
"""
- order_by_raw = expr_list_to_raw_expr_list(order_by)
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
filter_raw = filter.expr if filter is not None else None
return Expr(
@@ -2293,7 +2294,7 @@ def lead(
shift_offset: int = 1,
default_value: Optional[Any] = None,
partition_by: Optional[list[Expr]] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
) -> Expr:
"""Create a lead window function.
@@ -2330,7 +2331,7 @@ def lead(
partition_cols = (
[col.expr for col in partition_by] if partition_by is not None else
None
)
- order_cols = [col.expr for col in order_by] if order_by is not None else
None
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
return Expr(
f.lead(
@@ -2338,7 +2339,7 @@ def lead(
shift_offset,
default_value,
partition_by=partition_cols,
- order_by=order_cols,
+ order_by=order_by_raw,
)
)
@@ -2348,7 +2349,7 @@ def lag(
shift_offset: int = 1,
default_value: Optional[Any] = None,
partition_by: Optional[list[Expr]] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
) -> Expr:
"""Create a lag window function.
@@ -2382,7 +2383,7 @@ def lag(
partition_cols = (
[col.expr for col in partition_by] if partition_by is not None else
None
)
- order_cols = [col.expr for col in order_by] if order_by is not None else
None
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
return Expr(
f.lag(
@@ -2390,14 +2391,14 @@ def lag(
shift_offset,
default_value,
partition_by=partition_cols,
- order_by=order_cols,
+ order_by=order_by_raw,
)
)
def row_number(
partition_by: Optional[list[Expr]] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
) -> Expr:
"""Create a row number window function.
@@ -2421,19 +2422,19 @@ def row_number(
partition_cols = (
[col.expr for col in partition_by] if partition_by is not None else
None
)
- order_cols = [col.expr for col in order_by] if order_by is not None else
None
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
return Expr(
f.row_number(
partition_by=partition_cols,
- order_by=order_cols,
+ order_by=order_by_raw,
)
)
def rank(
partition_by: Optional[list[Expr]] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
) -> Expr:
"""Create a rank window function.
@@ -2462,19 +2463,19 @@ def rank(
partition_cols = (
[col.expr for col in partition_by] if partition_by is not None else
None
)
- order_cols = [col.expr for col in order_by] if order_by is not None else
None
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
return Expr(
f.rank(
partition_by=partition_cols,
- order_by=order_cols,
+ order_by=order_by_raw,
)
)
def dense_rank(
partition_by: Optional[list[Expr]] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
) -> Expr:
"""Create a dense_rank window function.
@@ -2498,19 +2499,19 @@ def dense_rank(
partition_cols = (
[col.expr for col in partition_by] if partition_by is not None else
None
)
- order_cols = [col.expr for col in order_by] if order_by is not None else
None
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
return Expr(
f.dense_rank(
partition_by=partition_cols,
- order_by=order_cols,
+ order_by=order_by_raw,
)
)
def percent_rank(
partition_by: Optional[list[Expr]] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
) -> Expr:
"""Create a percent_rank window function.
@@ -2535,19 +2536,19 @@ def percent_rank(
partition_cols = (
[col.expr for col in partition_by] if partition_by is not None else
None
)
- order_cols = [col.expr for col in order_by] if order_by is not None else
None
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
return Expr(
f.percent_rank(
partition_by=partition_cols,
- order_by=order_cols,
+ order_by=order_by_raw,
)
)
def cume_dist(
partition_by: Optional[list[Expr]] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
) -> Expr:
"""Create a cumulative distribution window function.
@@ -2572,12 +2573,12 @@ def cume_dist(
partition_cols = (
[col.expr for col in partition_by] if partition_by is not None else
None
)
- order_cols = [col.expr for col in order_by] if order_by is not None else
None
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
return Expr(
f.cume_dist(
partition_by=partition_cols,
- order_by=order_cols,
+ order_by=order_by_raw,
)
)
@@ -2585,7 +2586,7 @@ def cume_dist(
def ntile(
groups: int,
partition_by: Optional[list[Expr]] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
) -> Expr:
"""Create a n-tile window function.
@@ -2613,13 +2614,13 @@ def ntile(
partition_cols = (
[col.expr for col in partition_by] if partition_by is not None else
None
)
- order_cols = [col.expr for col in order_by] if order_by is not None else
None
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
return Expr(
f.ntile(
Expr.literal(groups).expr,
partition_by=partition_cols,
- order_by=order_cols,
+ order_by=order_by_raw,
)
)
@@ -2628,7 +2629,7 @@ def string_agg(
expression: Expr,
delimiter: str,
filter: Optional[Expr] = None,
- order_by: Optional[list[Expr]] = None,
+ order_by: Optional[list[Expr | SortExpr]] = None,
) -> Expr:
"""Concatenates the input strings.
@@ -2645,7 +2646,7 @@ def string_agg(
filter: If provided, only compute against rows for which the filter is
True
order_by: Set the ordering of the expression to evaluate
"""
- order_by_raw = expr_list_to_raw_expr_list(order_by)
+ order_by_raw = sort_list_to_raw_sort_list(order_by)
filter_raw = filter.expr if filter is not None else None
return Expr(
diff --git a/python/datafusion/tests/test_expr.py
b/python/datafusion/tests/test_expr.py
index 056d2ea..b58177f 100644
--- a/python/datafusion/tests/test_expr.py
+++ b/python/datafusion/tests/test_expr.py
@@ -15,18 +15,21 @@
# specific language governing permissions and limitations
# under the License.
+import pyarrow
+import pytest
from datafusion import SessionContext, col
-from datafusion.expr import Column, Literal, BinaryExpr, AggregateFunction
from datafusion.expr import (
- Projection,
- Filter,
Aggregate,
+ AggregateFunction,
+ BinaryExpr,
+ Column,
+ Filter,
Limit,
+ Literal,
+ Projection,
Sort,
TableScan,
)
-import pyarrow
-import pytest
@pytest.fixture
@@ -192,3 +195,24 @@ def test_expr_getitem() -> None:
assert names == ["Alice", "Bob", "Charlie", None]
assert array_values == [2, 5, None, None]
+
+
+def test_display_name_deprecation():
+ import warnings
+
+ expr = col("foo")
+ with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered
+ warnings.simplefilter("always")
+
+ # should trigger warning
+ name = expr.display_name()
+
+ # Verify some things
+ assert len(w) == 1
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "deprecated" in str(w[-1].message)
+
+ # returns appropriate result
+ assert name == expr.schema_name()
+ assert name == "foo"
diff --git a/python/datafusion/tests/test_sql.py
b/python/datafusion/tests/test_sql.py
index e41d010..cbb2e9f 100644
--- a/python/datafusion/tests/test_sql.py
+++ b/python/datafusion/tests/test_sql.py
@@ -264,14 +264,17 @@ def test_execute(ctx, tmp_path):
# count
result = ctx.sql("SELECT COUNT(a) AS cnt FROM t WHERE a IS NOT
NULL").collect()
+ ctx.sql("SELECT COUNT(a) AS cnt FROM t WHERE a IS NOT NULL").show()
+
+ expected_schema = pa.schema([("cnt", pa.int64(), False)])
+ expected_values = pa.array([7], type=pa.int64())
+ expected = [pa.RecordBatch.from_arrays([expected_values],
schema=expected_schema)]
- expected = pa.array([7], pa.int64())
- expected = [pa.RecordBatch.from_arrays([expected], ["cnt"])]
assert result == expected
# where
- expected = pa.array([2], pa.int64())
- expected = [pa.RecordBatch.from_arrays([expected], ["cnt"])]
+ expected_values = pa.array([2], type=pa.int64())
+ expected = [pa.RecordBatch.from_arrays([expected_values],
schema=expected_schema)]
result = ctx.sql("SELECT COUNT(a) AS cnt FROM t WHERE a > 10").collect()
assert result == expected
diff --git a/python/datafusion/tests/test_wrapper_coverage.py
b/python/datafusion/tests/test_wrapper_coverage.py
index 4a47de2..c53a89c 100644
--- a/python/datafusion/tests/test_wrapper_coverage.py
+++ b/python/datafusion/tests/test_wrapper_coverage.py
@@ -39,7 +39,10 @@ def missing_exports(internal_obj, wrapped_obj) -> None:
internal_attr = getattr(internal_obj, attr)
wrapped_attr = getattr(wrapped_obj, attr)
- assert wrapped_attr is not None if internal_attr is not None else True
+ if internal_attr is not None:
+ if wrapped_attr is None:
+ print("Missing attribute: ", attr)
+ assert False
if attr in ["__self__", "__class__"]:
continue
diff --git a/src/common/data_type.rs b/src/common/data_type.rs
index a29d179..7f9c75b 100644
--- a/src/common/data_type.rs
+++ b/src/common/data_type.rs
@@ -24,7 +24,7 @@ use pyo3::{exceptions::PyValueError, prelude::*};
use crate::errors::py_datafusion_err;
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
-#[pyclass(name = "RexType", module = "datafusion.common")]
+#[pyclass(eq, eq_int, name = "RexType", module = "datafusion.common")]
pub enum RexType {
Alias,
Literal,
@@ -692,7 +692,7 @@ impl From<DataType> for PyDataType {
/// Represents the possible Python types that can be mapped to the SQL types
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
-#[pyclass(name = "PythonType", module = "datafusion.common")]
+#[pyclass(eq, eq_int, name = "PythonType", module = "datafusion.common")]
pub enum PythonType {
Array,
Bool,
@@ -712,7 +712,7 @@ pub enum PythonType {
#[allow(non_camel_case_types)]
#[allow(clippy::upper_case_acronyms)]
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
-#[pyclass(name = "SqlType", module = "datafusion.common")]
+#[pyclass(eq, eq_int, name = "SqlType", module = "datafusion.common")]
pub enum SqlType {
ANY,
ARRAY,
@@ -770,7 +770,7 @@ pub enum SqlType {
#[allow(non_camel_case_types)]
#[allow(clippy::upper_case_acronyms)]
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
-#[pyclass(name = "NullTreatment", module = "datafusion.common")]
+#[pyclass(eq, eq_int, name = "NullTreatment", module = "datafusion.common")]
pub enum NullTreatment {
IGNORE_NULLS,
RESPECT_NULLS,
diff --git a/src/common/schema.rs b/src/common/schema.rs
index 5806c90..66ce925 100644
--- a/src/common/schema.rs
+++ b/src/common/schema.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use std::any::Any;
+use std::{any::Any, borrow::Cow};
use datafusion::arrow::datatypes::SchemaRef;
use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableSource};
@@ -62,6 +62,7 @@ pub struct SqlTable {
#[pymethods]
impl SqlTable {
#[new]
+ #[pyo3(signature = (table_name, columns, row_count, filepaths=None))]
pub fn new(
table_name: String,
columns: Vec<(String, DataTypeMap)>,
@@ -163,39 +164,33 @@ impl TableSource for SqlTableSource {
self.schema.clone()
}
- fn supports_filter_pushdown(
- &self,
- filter: &Expr,
- ) -> datafusion::common::Result<TableProviderFilterPushDown> {
- let filters = split_conjunction(filter);
- if filters.iter().all(|f| is_supported_push_down_expr(f)) {
- // Push down filters to the tablescan operation if all are
supported
- Ok(TableProviderFilterPushDown::Exact)
- } else if filters.iter().any(|f| is_supported_push_down_expr(f)) {
- // Partially apply the filter in the TableScan but retain
- // the Filter operator in the plan as well
- Ok(TableProviderFilterPushDown::Inexact)
- } else {
- Ok(TableProviderFilterPushDown::Unsupported)
- }
- }
-
fn table_type(&self) -> datafusion::logical_expr::TableType {
datafusion::logical_expr::TableType::Base
}
- #[allow(deprecated)]
fn supports_filters_pushdown(
&self,
filters: &[&Expr],
) -> datafusion::common::Result<Vec<TableProviderFilterPushDown>> {
filters
.iter()
- .map(|f| self.supports_filter_pushdown(f))
+ .map(|f| {
+ let filters = split_conjunction(f);
+ if filters.iter().all(|f| is_supported_push_down_expr(f)) {
+ // Push down filters to the tablescan operation if all are
supported
+ Ok(TableProviderFilterPushDown::Exact)
+ } else if filters.iter().any(|f|
is_supported_push_down_expr(f)) {
+ // Partially apply the filter in the TableScan but retain
+ // the Filter operator in the plan as well
+ Ok(TableProviderFilterPushDown::Inexact)
+ } else {
+ Ok(TableProviderFilterPushDown::Unsupported)
+ }
+ })
.collect()
}
- fn get_logical_plan(&self) ->
Option<&datafusion::logical_expr::LogicalPlan> {
+ fn get_logical_plan(&self) ->
Option<Cow<datafusion::logical_expr::LogicalPlan>> {
None
}
}
diff --git a/src/context.rs b/src/context.rs
index 11b9fed..79db2e6 100644
--- a/src/context.rs
+++ b/src/context.rs
@@ -35,7 +35,7 @@ use crate::catalog::{PyCatalog, PyTable};
use crate::dataframe::PyDataFrame;
use crate::dataset::Dataset;
use crate::errors::{py_datafusion_err, DataFusionError};
-use crate::expr::PyExpr;
+use crate::expr::sort_expr::PySortExpr;
use crate::physical_plan::PyExecutionPlan;
use crate::record_batch::PyRecordBatchStream;
use crate::sql::logical::PyLogicalPlan;
@@ -86,7 +86,7 @@ impl PySessionConfig {
let mut config = SessionConfig::new();
if let Some(hash_map) = config_options {
for (k, v) in &hash_map {
- config = config.set(k, ScalarValue::Utf8(Some(v.clone())));
+ config = config.set(k, &ScalarValue::Utf8(Some(v.clone())));
}
}
@@ -294,6 +294,7 @@ impl PySessionContext {
}
/// Register an object store with the given name
+ #[pyo3(signature = (scheme, store, host=None))]
pub fn register_object_store(
&mut self,
scheme: &str,
@@ -332,7 +333,7 @@ impl PySessionContext {
table_partition_cols: Vec<(String, String)>,
file_extension: &str,
schema: Option<PyArrowType<Schema>>,
- file_sort_order: Option<Vec<Vec<PyExpr>>>,
+ file_sort_order: Option<Vec<Vec<PySortExpr>>>,
py: Python,
) -> PyResult<()> {
let options = ListingOptions::new(Arc::new(ParquetFormat::new()))
@@ -374,6 +375,7 @@ impl PySessionContext {
Ok(PyDataFrame::new(df))
}
+ #[pyo3(signature = (query, options=None))]
pub fn sql_with_options(
&mut self,
query: &str,
@@ -390,6 +392,7 @@ impl PySessionContext {
Ok(PyDataFrame::new(df))
}
+ #[pyo3(signature = (partitions, name=None, schema=None))]
pub fn create_dataframe(
&mut self,
partitions: PyArrowType<Vec<Vec<RecordBatch>>>,
@@ -433,6 +436,7 @@ impl PySessionContext {
}
/// Construct datafusion dataframe from Python list
+ #[pyo3(signature = (data, name=None))]
pub fn from_pylist(
&mut self,
data: Bound<'_, PyList>,
@@ -452,6 +456,7 @@ impl PySessionContext {
}
/// Construct datafusion dataframe from Python dictionary
+ #[pyo3(signature = (data, name=None))]
pub fn from_pydict(
&mut self,
data: Bound<'_, PyDict>,
@@ -471,6 +476,7 @@ impl PySessionContext {
}
/// Construct datafusion dataframe from Arrow Table
+ #[pyo3(signature = (data, name=None))]
pub fn from_arrow(
&mut self,
data: Bound<'_, PyAny>,
@@ -506,6 +512,7 @@ impl PySessionContext {
/// Construct datafusion dataframe from pandas
#[allow(clippy::wrong_self_convention)]
+ #[pyo3(signature = (data, name=None))]
pub fn from_pandas(
&mut self,
data: Bound<'_, PyAny>,
@@ -525,6 +532,7 @@ impl PySessionContext {
}
/// Construct datafusion dataframe from polars
+ #[pyo3(signature = (data, name=None))]
pub fn from_polars(
&mut self,
data: Bound<'_, PyAny>,
@@ -581,7 +589,7 @@ impl PySessionContext {
file_extension: &str,
skip_metadata: bool,
schema: Option<PyArrowType<Schema>>,
- file_sort_order: Option<Vec<Vec<PyExpr>>>,
+ file_sort_order: Option<Vec<Vec<PySortExpr>>>,
py: Python,
) -> PyResult<()> {
let mut options = ParquetReadOptions::default()
@@ -882,7 +890,7 @@ impl PySessionContext {
file_extension: &str,
skip_metadata: bool,
schema: Option<PyArrowType<Schema>>,
- file_sort_order: Option<Vec<Vec<PyExpr>>>,
+ file_sort_order: Option<Vec<Vec<PySortExpr>>>,
py: Python,
) -> PyResult<PyDataFrame> {
let mut options = ParquetReadOptions::default()
diff --git a/src/dataframe.rs b/src/dataframe.rs
index f33622c..1f7f2e6 100644
--- a/src/dataframe.rs
+++ b/src/dataframe.rs
@@ -36,16 +36,19 @@ use datafusion::prelude::*;
use pyo3::exceptions::{PyTypeError, PyValueError};
use pyo3::prelude::*;
use pyo3::pybacked::PyBackedStr;
-use pyo3::types::{PyCapsule, PyTuple};
+use pyo3::types::{PyCapsule, PyTuple, PyTupleMethods};
use tokio::task::JoinHandle;
use crate::errors::py_datafusion_err;
-use crate::expr::to_sort_expressions;
+use crate::expr::sort_expr::to_sort_expressions;
use crate::physical_plan::PyExecutionPlan;
use crate::record_batch::PyRecordBatchStream;
use crate::sql::logical::PyLogicalPlan;
use crate::utils::{get_tokio_runtime, wait_for_future};
-use crate::{errors::DataFusionError, expr::PyExpr};
+use crate::{
+ errors::DataFusionError,
+ expr::{sort_expr::PySortExpr, PyExpr},
+};
/// A PyDataFrame is a representation of a logical plan and an API to compose
statements.
/// Use it to build a plan and `.collect()` to execute the plan and collect
the result.
@@ -70,7 +73,7 @@ impl PyDataFrame {
if let Ok(key) = key.extract::<PyBackedStr>() {
// df[col]
self.select_columns(vec![key])
- } else if let Ok(tuple) = key.extract::<&PyTuple>() {
+ } else if let Ok(tuple) = key.downcast::<PyTuple>() {
// df[col1, col2, col3]
let keys = tuple
.iter()
@@ -196,7 +199,7 @@ impl PyDataFrame {
}
#[pyo3(signature = (*exprs))]
- fn sort(&self, exprs: Vec<PyExpr>) -> PyResult<Self> {
+ fn sort(&self, exprs: Vec<PySortExpr>) -> PyResult<Self> {
let exprs = to_sort_expressions(exprs);
let df = self.df.as_ref().clone().sort(exprs)?;
Ok(Self::new(df))
@@ -504,6 +507,7 @@ impl PyDataFrame {
Ok(table)
}
+ #[pyo3(signature = (requested_schema=None))]
fn __arrow_c_stream__<'py>(
&'py mut self,
py: Python<'py>,
diff --git a/src/dataset.rs b/src/dataset.rs
index de7402f..a8fa21e 100644
--- a/src/dataset.rs
+++ b/src/dataset.rs
@@ -39,7 +39,7 @@ use crate::dataset_exec::DatasetExec;
use crate::pyarrow_filter_expression::PyArrowFilterExpression;
// Wraps a pyarrow.dataset.Dataset class and implements a Datafusion
TableProvider around it
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub(crate) struct Dataset {
dataset: PyObject,
}
diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs
index a377e25..2759aa6 100644
--- a/src/dataset_exec.rs
+++ b/src/dataset_exec.rs
@@ -53,7 +53,7 @@ impl Iterator for PyArrowBatchesAdapter {
fn next(&mut self) -> Option<Self::Item> {
Python::with_gil(|py| {
- let mut batches = self.batches.clone().into_bound(py);
+ let mut batches = self.batches.clone_ref(py).into_bound(py);
Some(
batches
.next()?
@@ -65,7 +65,7 @@ impl Iterator for PyArrowBatchesAdapter {
}
// Wraps a pyarrow.dataset.Dataset class and implements a Datafusion
ExecutionPlan around it
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub(crate) struct DatasetExec {
dataset: PyObject,
schema: SchemaRef,
diff --git a/src/expr.rs b/src/expr.rs
index 0e1a193..304d147 100644
--- a/src/expr.rs
+++ b/src/expr.rs
@@ -28,7 +28,7 @@ use datafusion::arrow::pyarrow::PyArrowType;
use datafusion::functions::core::expr_ext::FieldAccessor;
use datafusion::logical_expr::{
col,
- expr::{AggregateFunction, InList, InSubquery, ScalarFunction, Sort,
WindowFunction},
+ expr::{AggregateFunction, InList, InSubquery, ScalarFunction,
WindowFunction},
lit, Between, BinaryExpr, Case, Cast, Expr, Like, Operator, TryCast,
};
use datafusion::scalar::ScalarValue;
@@ -94,6 +94,8 @@ pub mod unnest;
pub mod unnest_expr;
pub mod window;
+use sort_expr::{to_sort_expressions, PySortExpr};
+
/// A PyExpr that can be used on a DataFrame
#[pyclass(name = "Expr", module = "datafusion.expr", subclass)]
#[derive(Debug, Clone)]
@@ -150,7 +152,6 @@ impl PyExpr {
Expr::Case(value) =>
Ok(case::PyCase::from(value.clone()).into_py(py)),
Expr::Cast(value) =>
Ok(cast::PyCast::from(value.clone()).into_py(py)),
Expr::TryCast(value) =>
Ok(cast::PyTryCast::from(value.clone()).into_py(py)),
- Expr::Sort(value) =>
Ok(sort_expr::PySortExpr::from(value.clone()).into_py(py)),
Expr::ScalarFunction(value) =>
Err(py_unsupported_variant_err(format!(
"Converting Expr::ScalarFunction to a Python object is not
implemented: {:?}",
value
@@ -167,9 +168,9 @@ impl PyExpr {
Expr::ScalarSubquery(value) => {
Ok(scalar_subquery::PyScalarSubquery::from(value.clone()).into_py(py))
}
- Expr::Wildcard { qualifier } =>
Err(py_unsupported_variant_err(format!(
- "Converting Expr::Wildcard to a Python object is not
implemented : {:?}",
- qualifier
+ Expr::Wildcard { qualifier, options } =>
Err(py_unsupported_variant_err(format!(
+ "Converting Expr::Wildcard to a Python object is not
implemented : {:?} {:?}",
+ qualifier, options
))),
Expr::GroupingSet(value) => {
Ok(grouping_set::PyGroupingSet::from(value.clone()).into_py(py))
@@ -188,13 +189,13 @@ impl PyExpr {
/// Returns the name of this expression as it should appear in a schema.
This name
/// will not include any CAST expressions.
- fn display_name(&self) -> PyResult<String> {
- Ok(self.expr.display_name()?)
+ fn schema_name(&self) -> PyResult<String> {
+ Ok(format!("{}", self.expr.schema_name()))
}
/// Returns a full and complete string representation of this expression.
fn canonical_name(&self) -> PyResult<String> {
- Ok(self.expr.canonical_name())
+ Ok(format!("{}", self.expr))
}
/// Returns the name of the Expr variant.
@@ -274,7 +275,7 @@ impl PyExpr {
/// Create a sort PyExpr from an existing PyExpr.
#[pyo3(signature = (ascending=true, nulls_first=true))]
- pub fn sort(&self, ascending: bool, nulls_first: bool) -> PyExpr {
+ pub fn sort(&self, ascending: bool, nulls_first: bool) -> PySortExpr {
self.expr.clone().sort(ascending, nulls_first).into()
}
@@ -323,7 +324,6 @@ impl PyExpr {
| Expr::Case { .. }
| Expr::Cast { .. }
| Expr::TryCast { .. }
- | Expr::Sort { .. }
| Expr::ScalarFunction { .. }
| Expr::AggregateFunction { .. }
| Expr::WindowFunction { .. }
@@ -387,7 +387,6 @@ impl PyExpr {
| Expr::Negative(expr)
| Expr::Cast(Cast { expr, .. })
| Expr::TryCast(TryCast { expr, .. })
- | Expr::Sort(Sort { expr, .. })
| Expr::InSubquery(InSubquery { expr, .. }) =>
Ok(vec![PyExpr::from(*expr.clone())]),
// Expr variants containing a collection of Expr(s) for operands
@@ -529,7 +528,7 @@ impl PyExpr {
// Expression Function Builder functions
- pub fn order_by(&self, order_by: Vec<PyExpr>) -> PyExprFuncBuilder {
+ pub fn order_by(&self, order_by: Vec<PySortExpr>) -> PyExprFuncBuilder {
self.expr
.clone()
.order_by(to_sort_expressions(order_by))
@@ -573,20 +572,9 @@ impl From<ExprFuncBuilder> for PyExprFuncBuilder {
}
}
-pub fn to_sort_expressions(order_by: Vec<PyExpr>) -> Vec<Expr> {
- order_by
- .iter()
- .map(|e| e.expr.clone())
- .map(|e| match e {
- Expr::Sort(_) => e,
- _ => e.sort(true, true),
- })
- .collect()
-}
-
#[pymethods]
impl PyExprFuncBuilder {
- pub fn order_by(&self, order_by: Vec<PyExpr>) -> PyExprFuncBuilder {
+ pub fn order_by(&self, order_by: Vec<PySortExpr>) -> PyExprFuncBuilder {
self.builder
.clone()
.order_by(to_sort_expressions(order_by))
@@ -641,11 +629,6 @@ impl PyExpr {
input_plan: &LogicalPlan,
) -> Result<Arc<Field>, DataFusionError> {
match expr {
- Expr::Sort(Sort { expr, .. }) => {
- // DataFusion does not support create_name for sort
expressions (since they never
- // appear in projections) so we just delegate to the contained
expression instead
- Self::expr_to_field(expr, input_plan)
- }
Expr::Wildcard { .. } => {
// Since * could be any of the valid column names just return
the first one
Ok(Arc::new(input_plan.schema().field(0).clone()))
diff --git a/src/expr/sort.rs b/src/expr/sort.rs
index b31ebfe..a1803cc 100644
--- a/src/expr/sort.rs
+++ b/src/expr/sort.rs
@@ -22,7 +22,7 @@ use std::fmt::{self, Display, Formatter};
use crate::common::df_schema::PyDFSchema;
use crate::expr::logical_node::LogicalNode;
-use crate::expr::PyExpr;
+use crate::expr::sort_expr::PySortExpr;
use crate::sql::logical::PyLogicalPlan;
#[pyclass(name = "Sort", module = "datafusion.expr", subclass)]
@@ -63,12 +63,12 @@ impl Display for PySort {
#[pymethods]
impl PySort {
/// Retrieves the sort expressions for this `Sort`
- fn sort_exprs(&self) -> PyResult<Vec<PyExpr>> {
+ fn sort_exprs(&self) -> PyResult<Vec<PySortExpr>> {
Ok(self
.sort
.expr
.iter()
- .map(|e| PyExpr::from(e.clone()))
+ .map(|e| PySortExpr::from(e.clone()))
.collect())
}
diff --git a/src/expr/sort_expr.rs b/src/expr/sort_expr.rs
index 4299d1f..12f74e4 100644
--- a/src/expr/sort_expr.rs
+++ b/src/expr/sort_expr.rs
@@ -51,10 +51,29 @@ impl Display for PySortExpr {
}
}
+pub fn to_sort_expressions(order_by: Vec<PySortExpr>) -> Vec<SortExpr> {
+ order_by.iter().map(|e| e.sort.clone()).collect()
+}
+
+pub fn py_sort_expr_list(expr: &[SortExpr]) -> PyResult<Vec<PySortExpr>> {
+ Ok(expr.iter().map(|e| PySortExpr::from(e.clone())).collect())
+}
+
#[pymethods]
impl PySortExpr {
+ #[new]
+ fn new(expr: PyExpr, asc: bool, nulls_first: bool) -> Self {
+ Self {
+ sort: SortExpr {
+ expr: expr.into(),
+ asc,
+ nulls_first,
+ },
+ }
+ }
+
fn expr(&self) -> PyResult<PyExpr> {
- Ok((*self.sort.expr).clone().into())
+ Ok(self.sort.expr.clone().into())
}
fn ascending(&self) -> PyResult<bool> {
diff --git a/src/expr/window.rs b/src/expr/window.rs
index f17a6dd..950db12 100644
--- a/src/expr/window.rs
+++ b/src/expr/window.rs
@@ -24,6 +24,7 @@ use std::fmt::{self, Display, Formatter};
use crate::common::df_schema::PyDFSchema;
use crate::errors::py_type_err;
use crate::expr::logical_node::LogicalNode;
+use crate::expr::sort_expr::{py_sort_expr_list, PySortExpr};
use crate::expr::PyExpr;
use crate::sql::logical::PyLogicalPlan;
@@ -114,9 +115,9 @@ impl PyWindow {
}
/// Returns order by columns in a window function expression
- pub fn get_sort_exprs(&self, expr: PyExpr) -> PyResult<Vec<PyExpr>> {
+ pub fn get_sort_exprs(&self, expr: PyExpr) -> PyResult<Vec<PySortExpr>> {
match expr.expr.unalias() {
- Expr::WindowFunction(WindowFunction { order_by, .. }) =>
py_expr_list(&order_by),
+ Expr::WindowFunction(WindowFunction { order_by, .. }) =>
py_sort_expr_list(&order_by),
other => Err(not_window_function_err(other)),
}
}
diff --git a/src/functions.rs b/src/functions.rs
index b9ca630..32f6519 100644
--- a/src/functions.rs
+++ b/src/functions.rs
@@ -25,7 +25,8 @@ use crate::common::data_type::NullTreatment;
use crate::context::PySessionContext;
use crate::errors::DataFusionError;
use crate::expr::conditional_expr::PyCaseBuilder;
-use crate::expr::to_sort_expressions;
+use crate::expr::sort_expr::to_sort_expressions;
+use crate::expr::sort_expr::PySortExpr;
use crate::expr::window::PyWindowFrame;
use crate::expr::PyExpr;
use datafusion::common::{Column, ScalarValue, TableReference};
@@ -35,7 +36,7 @@ use datafusion::functions_aggregate;
use datafusion::logical_expr::expr::Alias;
use datafusion::logical_expr::sqlparser::ast::NullTreatment as DFNullTreatment;
use datafusion::logical_expr::{
- expr::{find_df_window_func, Sort, WindowFunction},
+ expr::{find_df_window_func, WindowFunction},
lit, Expr, WindowFunctionDefinition,
};
@@ -43,7 +44,7 @@ fn add_builder_fns_to_aggregate(
agg_fn: Expr,
distinct: Option<bool>,
filter: Option<PyExpr>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
null_treatment: Option<NullTreatment>,
) -> PyResult<PyExpr> {
// Since ExprFuncBuilder::new() is private, we can guarantee initializing
@@ -96,6 +97,7 @@ fn array_cat(exprs: Vec<PyExpr>) -> PyExpr {
}
#[pyfunction]
+#[pyo3(signature = (array, element, index=None))]
fn array_position(array: PyExpr, element: PyExpr, index: Option<i64>) ->
PyExpr {
let index = ScalarValue::Int64(index);
let index = Expr::Literal(index);
@@ -104,6 +106,7 @@ fn array_position(array: PyExpr, element: PyExpr, index:
Option<i64>) -> PyExpr
}
#[pyfunction]
+#[pyo3(signature = (array, begin, end, stride=None))]
fn array_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride:
Option<PyExpr>) -> PyExpr {
datafusion::functions_nested::expr_fn::array_slice(
array.into(),
@@ -142,16 +145,19 @@ fn concat_ws(sep: String, args: Vec<PyExpr>) ->
PyResult<PyExpr> {
}
#[pyfunction]
+#[pyo3(signature = (values, regex, flags=None))]
fn regexp_like(values: PyExpr, regex: PyExpr, flags: Option<PyExpr>) ->
PyResult<PyExpr> {
Ok(functions::expr_fn::regexp_like(values.expr, regex.expr, flags.map(|x|
x.expr)).into())
}
#[pyfunction]
+#[pyo3(signature = (values, regex, flags=None))]
fn regexp_match(values: PyExpr, regex: PyExpr, flags: Option<PyExpr>) ->
PyResult<PyExpr> {
Ok(functions::expr_fn::regexp_match(values.expr, regex.expr, flags.map(|x|
x.expr)).into())
}
#[pyfunction]
+#[pyo3(signature = (string, pattern, replacement, flags=None))]
/// Replaces substring(s) matching a POSIX regular expression.
fn regexp_replace(
string: PyExpr,
@@ -169,14 +175,12 @@ fn regexp_replace(
}
/// Creates a new Sort Expr
#[pyfunction]
-fn order_by(expr: PyExpr, asc: bool, nulls_first: bool) -> PyResult<PyExpr> {
- Ok(PyExpr {
- expr: datafusion::logical_expr::Expr::Sort(Sort {
- expr: Box::new(expr.expr),
- asc,
- nulls_first,
- }),
- })
+fn order_by(expr: PyExpr, asc: bool, nulls_first: bool) ->
PyResult<PySortExpr> {
+ Ok(PySortExpr::from(datafusion::logical_expr::expr::Sort {
+ expr: expr.expr,
+ asc,
+ nulls_first,
+ }))
}
/// Creates a new Alias Expr
@@ -283,11 +287,12 @@ fn find_window_fn(name: &str, ctx:
Option<PySessionContext>) -> PyResult<WindowF
/// Creates a new Window function expression
#[pyfunction]
+#[pyo3(signature = (name, args, partition_by=None, order_by=None,
window_frame=None, ctx=None))]
fn window(
name: &str,
args: Vec<PyExpr>,
partition_by: Option<Vec<PyExpr>>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
window_frame: Option<PyWindowFrame>,
ctx: Option<PySessionContext>,
) -> PyResult<PyExpr> {
@@ -309,11 +314,7 @@ fn window(
order_by: order_by
.unwrap_or_default()
.into_iter()
- .map(|x| x.expr)
- .map(|e| match e {
- Expr::Sort(_) => e,
- _ => e.sort(true, true),
- })
+ .map(|x| x.into())
.collect::<Vec<_>>(),
window_frame,
null_treatment: None,
@@ -331,11 +332,12 @@ macro_rules! aggregate_function {
};
($NAME: ident, $($arg:ident)*) => {
#[pyfunction]
+ #[pyo3(signature = ($($arg),*, distinct=None, filter=None,
order_by=None, null_treatment=None))]
fn $NAME(
$($arg: PyExpr),*,
distinct: Option<bool>,
filter: Option<PyExpr>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
null_treatment: Option<NullTreatment>
) -> PyResult<PyExpr> {
let agg_fn = functions_aggregate::expr_fn::$NAME($($arg.into()),*);
@@ -351,11 +353,12 @@ macro_rules! aggregate_function_vec_args {
};
($NAME: ident, $($arg:ident)*) => {
#[pyfunction]
+ #[pyo3(signature = ($($arg),*, distinct=None, filter=None,
order_by=None, null_treatment=None))]
fn $NAME(
$($arg: PyExpr),*,
distinct: Option<bool>,
filter: Option<PyExpr>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
null_treatment: Option<NullTreatment>
) -> PyResult<PyExpr> {
let agg_fn =
functions_aggregate::expr_fn::$NAME(vec![$($arg.into()),*]);
@@ -624,6 +627,7 @@ aggregate_function!(approx_median);
// aggregate_function!(grouping);
#[pyfunction]
+#[pyo3(signature = (expression, percentile, num_centroids=None, filter=None))]
pub fn approx_percentile_cont(
expression: PyExpr,
percentile: f64,
@@ -642,6 +646,7 @@ pub fn approx_percentile_cont(
}
#[pyfunction]
+#[pyo3(signature = (expression, weight, percentile, filter=None))]
pub fn approx_percentile_cont_with_weight(
expression: PyExpr,
weight: PyExpr,
@@ -662,11 +667,12 @@ aggregate_function_vec_args!(last_value);
// We handle first_value explicitly because the signature expects an order_by
// https://github.com/apache/datafusion/issues/12376
#[pyfunction]
+#[pyo3(signature = (expr, distinct=None, filter=None, order_by=None,
null_treatment=None))]
pub fn first_value(
expr: PyExpr,
distinct: Option<bool>,
filter: Option<PyExpr>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
null_treatment: Option<NullTreatment>,
) -> PyResult<PyExpr> {
// If we initialize the UDAF with order_by directly, then it gets
over-written by the builder
@@ -677,26 +683,28 @@ pub fn first_value(
// nth_value requires a non-expr argument
#[pyfunction]
+#[pyo3(signature = (expr, n, distinct=None, filter=None, order_by=None,
null_treatment=None))]
pub fn nth_value(
expr: PyExpr,
n: i64,
distinct: Option<bool>,
filter: Option<PyExpr>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
null_treatment: Option<NullTreatment>,
) -> PyResult<PyExpr> {
- let agg_fn =
datafusion::functions_aggregate::nth_value::nth_value(vec![expr.expr, lit(n)]);
+ let agg_fn =
datafusion::functions_aggregate::nth_value::nth_value(expr.expr, n, vec![]);
add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by,
null_treatment)
}
// string_agg requires a non-expr argument
#[pyfunction]
+#[pyo3(signature = (expr, delimiter, distinct=None, filter=None,
order_by=None, null_treatment=None))]
pub fn string_agg(
expr: PyExpr,
delimiter: String,
distinct: Option<bool>,
filter: Option<PyExpr>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
null_treatment: Option<NullTreatment>,
) -> PyResult<PyExpr> {
let agg_fn =
datafusion::functions_aggregate::string_agg::string_agg(expr.expr,
lit(delimiter));
@@ -706,7 +714,7 @@ pub fn string_agg(
fn add_builder_fns_to_window(
window_fn: Expr,
partition_by: Option<Vec<PyExpr>>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
) -> PyResult<PyExpr> {
// Since ExprFuncBuilder::new() is private, set an empty partition and then
// override later if appropriate.
@@ -730,12 +738,13 @@ fn add_builder_fns_to_window(
}
#[pyfunction]
+#[pyo3(signature = (arg, shift_offset, default_value=None, partition_by=None,
order_by=None))]
pub fn lead(
arg: PyExpr,
shift_offset: i64,
default_value: Option<ScalarValue>,
partition_by: Option<Vec<PyExpr>>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
) -> PyResult<PyExpr> {
let window_fn = window_function::lead(arg.expr, Some(shift_offset),
default_value);
@@ -743,12 +752,13 @@ pub fn lead(
}
#[pyfunction]
+#[pyo3(signature = (arg, shift_offset, default_value=None, partition_by=None,
order_by=None))]
pub fn lag(
arg: PyExpr,
shift_offset: i64,
default_value: Option<ScalarValue>,
partition_by: Option<Vec<PyExpr>>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
) -> PyResult<PyExpr> {
let window_fn = window_function::lag(arg.expr, Some(shift_offset),
default_value);
@@ -756,26 +766,32 @@ pub fn lag(
}
#[pyfunction]
+#[pyo3(signature = (partition_by=None, order_by=None))]
pub fn row_number(
partition_by: Option<Vec<PyExpr>>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
) -> PyResult<PyExpr> {
- let window_fn = window_function::row_number();
+ let window_fn = datafusion::functions_window::expr_fn::row_number();
add_builder_fns_to_window(window_fn, partition_by, order_by)
}
#[pyfunction]
-pub fn rank(partition_by: Option<Vec<PyExpr>>, order_by: Option<Vec<PyExpr>>)
-> PyResult<PyExpr> {
+#[pyo3(signature = (partition_by=None, order_by=None))]
+pub fn rank(
+ partition_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
+) -> PyResult<PyExpr> {
let window_fn = window_function::rank();
add_builder_fns_to_window(window_fn, partition_by, order_by)
}
#[pyfunction]
+#[pyo3(signature = (partition_by=None, order_by=None))]
pub fn dense_rank(
partition_by: Option<Vec<PyExpr>>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
) -> PyResult<PyExpr> {
let window_fn = window_function::dense_rank();
@@ -783,9 +799,10 @@ pub fn dense_rank(
}
#[pyfunction]
+#[pyo3(signature = (partition_by=None, order_by=None))]
pub fn percent_rank(
partition_by: Option<Vec<PyExpr>>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
) -> PyResult<PyExpr> {
let window_fn = window_function::percent_rank();
@@ -793,9 +810,10 @@ pub fn percent_rank(
}
#[pyfunction]
+#[pyo3(signature = (partition_by=None, order_by=None))]
pub fn cume_dist(
partition_by: Option<Vec<PyExpr>>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
) -> PyResult<PyExpr> {
let window_fn = window_function::cume_dist();
@@ -803,10 +821,11 @@ pub fn cume_dist(
}
#[pyfunction]
+#[pyo3(signature = (arg, partition_by=None, order_by=None))]
pub fn ntile(
arg: PyExpr,
partition_by: Option<Vec<PyExpr>>,
- order_by: Option<Vec<PyExpr>>,
+ order_by: Option<Vec<PySortExpr>>,
) -> PyResult<PyExpr> {
let window_fn = window_function::ntile(arg.into());
@@ -1002,8 +1021,8 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) ->
PyResult<()> {
// Window Functions
m.add_wrapped(wrap_pyfunction!(lead))?;
m.add_wrapped(wrap_pyfunction!(lag))?;
- m.add_wrapped(wrap_pyfunction!(row_number))?;
m.add_wrapped(wrap_pyfunction!(rank))?;
+ m.add_wrapped(wrap_pyfunction!(row_number))?;
m.add_wrapped(wrap_pyfunction!(dense_rank))?;
m.add_wrapped(wrap_pyfunction!(percent_rank))?;
m.add_wrapped(wrap_pyfunction!(cume_dist))?;
diff --git a/src/pyarrow_filter_expression.rs b/src/pyarrow_filter_expression.rs
index 6e2a45e..0f97ea4 100644
--- a/src/pyarrow_filter_expression.rs
+++ b/src/pyarrow_filter_expression.rs
@@ -27,7 +27,7 @@ use datafusion::logical_expr::{expr::InList, Between,
BinaryExpr, Expr, Operator
use crate::errors::DataFusionError;
-#[derive(Debug, Clone)]
+#[derive(Debug)]
#[repr(transparent)]
pub(crate) struct PyArrowFilterExpression(PyObject);
diff --git a/src/udf.rs b/src/udf.rs
index 8bd9021..7d5db2f 100644
--- a/src/udf.rs
+++ b/src/udf.rs
@@ -21,7 +21,8 @@ use pyo3::{prelude::*, types::PyTuple};
use datafusion::arrow::array::{make_array, Array, ArrayData, ArrayRef};
use datafusion::arrow::datatypes::DataType;
-use datafusion::arrow::pyarrow::{FromPyArrow, PyArrowType, ToPyArrow};
+use datafusion::arrow::pyarrow::FromPyArrow;
+use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow};
use datafusion::error::DataFusionError;
use datafusion::logical_expr::create_udf;
use datafusion::logical_expr::function::ScalarFunctionImplementation;
@@ -43,16 +44,15 @@ fn to_rust_function(func: PyObject) ->
ScalarFunctionImplementation {
.iter()
.map(|arg| arg.into_data().to_pyarrow(py).unwrap())
.collect::<Vec<_>>();
- let py_args = PyTuple::new(py, py_args);
+ let py_args = PyTuple::new_bound(py, py_args);
// 2. call function
let value = func
- .as_ref(py)
- .call(py_args, None)
+ .call_bound(py, py_args, None)
.map_err(|e|
DataFusionError::Execution(format!("{e:?}")))?;
// 3. cast to arrow::array::Array
- let array_data = ArrayData::from_pyarrow(value).unwrap();
+ let array_data =
ArrayData::from_pyarrow_bound(value.bind(py)).unwrap();
Ok(make_array(array_data))
})
},
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
