This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new a06073913b Update to `arrow`, `parquet` to `57.1.0` (#18820)
a06073913b is described below
commit a06073913ba8e419beaefc50e9736d5a7b5dbc3a
Author: Andrew Lamb <[email protected]>
AuthorDate: Mon Dec 1 07:00:26 2025 -0500
Update to `arrow`, `parquet` to `57.1.0` (#18820)
## Which issue does this PR close?
- related to https://github.com/apache/arrow-rs/issues/8464
## Rationale for this change
Get latest and greatest code from arrow
## What changes are included in this PR?
1. Update to Arrow 57.1.0
2. Update for API changes (comments inline)
## Are these changes tested?
Yes, by CI
## Are there any user-facing changes?
No
---------
Co-authored-by: Raz Luvaton <[email protected]>
---
Cargo.lock | 132 ++++++++-------------
Cargo.toml | 14 +--
datafusion-cli/src/main.rs | 12 +-
datafusion/common/src/error.rs | 1 -
datafusion/core/src/physical_planner.rs | 2 +-
.../core/tests/dataframe/dataframe_functions.rs | 8 +-
datafusion/core/tests/parquet/filter_pushdown.rs | 4 +-
datafusion/core/tests/sql/select.rs | 8 +-
datafusion/optimizer/src/analyzer/type_coercion.rs | 2 +-
datafusion/sqllogictest/test_files/array.slt | 53 ++++-----
.../sqllogictest/test_files/arrow_typeof.slt | 10 +-
datafusion/sqllogictest/test_files/coalesce.slt | 6 +-
datafusion/sqllogictest/test_files/map.slt | 4 +-
datafusion/sqllogictest/test_files/struct.slt | 38 +++---
datafusion/sqllogictest/test_files/window.slt | 4 +-
15 files changed, 134 insertions(+), 164 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index b25e60eb6f..6415fa65c0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -234,9 +234,9 @@ checksum =
"7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae"
+checksum = "cb372a7cbcac02a35d3fb7b3fc1f969ec078e871f9bb899bf00a2e1809bec8a3"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -257,9 +257,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491"
+checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -271,9 +271,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31"
+checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002"
dependencies = [
"ahash 0.8.12",
"arrow-buffer",
@@ -290,9 +290,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "76885a2697a7edf6b59577f568b456afc94ce0e2edc15b784ce3685b6c3c5c27"
+checksum = "a2819d893750cb3380ab31ebdc8c68874dd4429f90fd09180f3c93538bd21626"
dependencies = [
"bytes",
"half",
@@ -302,13 +302,14 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168"
+checksum = "e3d131abb183f80c450d4591dc784f8d7750c50c6e2bc3fcaad148afc8361271"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
+ "arrow-ord",
"arrow-schema",
"arrow-select",
"atoi",
@@ -323,9 +324,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c"
+checksum = "2275877a0e5e7e7c76954669366c2aa1a829e340ab1f612e647507860906fb6b"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -338,9 +339,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b"
+checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -351,9 +352,9 @@ dependencies = [
[[package]]
name = "arrow-flight"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f70bb56412a007b0cfc116d15f24dda6adeed9611a213852a004cda20085a3b9"
+checksum = "8b5f57c3d39d1b1b7c1376a772ea86a131e7da310aed54ebea9363124bb885e3"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -379,9 +380,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27"
+checksum = "3d09446e8076c4b3f235603d9ea7c5494e73d441b01cd61fb33d7254c11964b3"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -395,9 +396,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8"
+checksum = "371ffd66fa77f71d7628c63f209c9ca5341081051aa32f9c8020feb0def787c0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -419,9 +420,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b"
+checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -432,9 +433,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2"
+checksum = "169676f317157dc079cc5def6354d16db63d8861d61046d2f3883268ced6f99f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -445,9 +446,9 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5"
+checksum = "d27609cd7dd45f006abae27995c2729ef6f4b9361cde1ddd019dc31a5aa017e0"
dependencies = [
"bitflags 2.9.4",
"serde",
@@ -457,9 +458,9 @@ dependencies = [
[[package]]
name = "arrow-select"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47"
+checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010"
dependencies = [
"ahash 0.8.12",
"arrow-array",
@@ -471,9 +472,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2"
+checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -2813,7 +2814,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
- "windows-sys 0.61.0",
+ "windows-sys 0.60.2",
]
[[package]]
@@ -2957,7 +2958,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
- "windows-sys 0.61.0",
+ "windows-sys 0.52.0",
]
[[package]]
@@ -3003,7 +3004,7 @@ checksum =
"0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78"
dependencies = [
"cfg-if",
"rustix",
- "windows-sys 0.59.0",
+ "windows-sys 0.52.0",
]
[[package]]
@@ -3597,7 +3598,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
- "windows-core 0.62.0",
+ "windows-core",
]
[[package]]
@@ -4080,9 +4081,9 @@ checksum =
"112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "lz4_flex"
-version = "0.11.5"
+version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a"
+checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e"
dependencies = [
"twox-hash",
]
@@ -4439,9 +4440,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "57.0.0"
+version = "57.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7a0f31027ef1af7549f7cec603a9a21dce706d3f8d7c2060a68f43c1773be95a"
+checksum = "be3e4f6d320dd92bfa7d612e265d7d08bba0a240bab86af3425e1d255a511d89"
dependencies = [
"ahash 0.8.12",
"arrow-array",
@@ -4835,7 +4836,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1"
dependencies = [
"heck 0.5.0",
- "itertools 0.14.0",
+ "itertools 0.13.0",
"log",
"multimap",
"once_cell",
@@ -4855,7 +4856,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"
dependencies = [
"anyhow",
- "itertools 0.14.0",
+ "itertools 0.13.0",
"proc-macro2",
"quote",
"syn 2.0.111",
@@ -5395,7 +5396,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
- "windows-sys 0.61.0",
+ "windows-sys 0.52.0",
]
[[package]]
@@ -5931,7 +5932,7 @@ dependencies = [
"cfg-if",
"libc",
"psm",
- "windows-sys 0.59.0",
+ "windows-sys 0.52.0",
]
[[package]]
@@ -6149,7 +6150,7 @@ dependencies = [
"getrandom 0.3.4",
"once_cell",
"rustix",
- "windows-sys 0.61.0",
+ "windows-sys 0.52.0",
]
[[package]]
@@ -7046,7 +7047,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
- "windows-sys 0.61.0",
+ "windows-sys 0.52.0",
]
[[package]]
@@ -7062,7 +7063,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
dependencies = [
"windows-collections",
- "windows-core 0.61.2",
+ "windows-core",
"windows-future",
"windows-link 0.1.3",
"windows-numerics",
@@ -7074,7 +7075,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"
dependencies = [
- "windows-core 0.61.2",
+ "windows-core",
]
[[package]]
@@ -7086,21 +7087,8 @@ dependencies = [
"windows-implement",
"windows-interface",
"windows-link 0.1.3",
- "windows-result 0.3.4",
- "windows-strings 0.4.2",
-]
-
-[[package]]
-name = "windows-core"
-version = "0.62.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "57fe7168f7de578d2d8a05b07fd61870d2e73b4020e9f49aa00da8471723497c"
-dependencies = [
- "windows-implement",
- "windows-interface",
- "windows-link 0.2.0",
- "windows-result 0.4.0",
- "windows-strings 0.5.0",
+ "windows-result",
+ "windows-strings",
]
[[package]]
@@ -7109,7 +7097,7 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
dependencies = [
- "windows-core 0.61.2",
+ "windows-core",
"windows-link 0.1.3",
"windows-threading",
]
@@ -7154,7 +7142,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"
dependencies = [
- "windows-core 0.61.2",
+ "windows-core",
"windows-link 0.1.3",
]
@@ -7167,15 +7155,6 @@ dependencies = [
"windows-link 0.1.3",
]
-[[package]]
-name = "windows-result"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f"
-dependencies = [
- "windows-link 0.2.0",
-]
-
[[package]]
name = "windows-strings"
version = "0.4.2"
@@ -7185,15 +7164,6 @@ dependencies = [
"windows-link 0.1.3",
]
-[[package]]
-name = "windows-strings"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda"
-dependencies = [
- "windows-link 0.2.0",
-]
-
[[package]]
name = "windows-sys"
version = "0.52.0"
diff --git a/Cargo.toml b/Cargo.toml
index 846fd297e0..dd6543db98 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -91,19 +91,19 @@ ahash = { version = "0.8", default-features = false,
features = [
"runtime-rng",
] }
apache-avro = { version = "0.21", default-features = false }
-arrow = { version = "57.0.0", features = [
+arrow = { version = "57.1.0", features = [
"prettyprint",
"chrono-tz",
] }
-arrow-buffer = { version = "57.0.0", default-features = false }
-arrow-flight = { version = "57.0.0", features = [
+arrow-buffer = { version = "57.1.0", default-features = false }
+arrow-flight = { version = "57.1.0", features = [
"flight-sql-experimental",
] }
-arrow-ipc = { version = "57.0.0", default-features = false, features = [
+arrow-ipc = { version = "57.1.0", default-features = false, features = [
"lz4",
] }
-arrow-ord = { version = "57.0.0", default-features = false }
-arrow-schema = { version = "57.0.0", default-features = false }
+arrow-ord = { version = "57.1.0", default-features = false }
+arrow-schema = { version = "57.1.0", default-features = false }
async-trait = "0.1.89"
bigdecimal = "0.4.8"
bytes = "1.11"
@@ -166,7 +166,7 @@ log = "^0.4"
num-traits = { version = "0.2" }
object_store = { version = "0.12.4", default-features = false }
parking_lot = "0.12"
-parquet = { version = "57.0.0", default-features = false, features = [
+parquet = { version = "57.1.0", default-features = false, features = [
"arrow",
"async",
"object_store",
diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs
index 09fa8ef15a..de666fced7 100644
--- a/datafusion-cli/src/main.rs
+++ b/datafusion-cli/src/main.rs
@@ -592,9 +592,9 @@ mod tests {
+-----------------------------------+-----------------+---------------------+------+------------------+
| filename | file_size_bytes |
metadata_size_bytes | hits | extra |
+-----------------------------------+-----------------+---------------------+------+------------------+
- | alltypes_plain.parquet | 1851 | 6957
| 2 | page_index=false |
- | alltypes_tiny_pages.parquet | 454233 | 267014
| 2 | page_index=true |
- | lz4_raw_compressed_larger.parquet | 380836 | 996
| 2 | page_index=false |
+ | alltypes_plain.parquet | 1851 | 8882
| 2 | page_index=false |
+ | alltypes_tiny_pages.parquet | 454233 | 269266
| 2 | page_index=true |
+ | lz4_raw_compressed_larger.parquet | 380836 | 1347
| 2 | page_index=false |
+-----------------------------------+-----------------+---------------------+------+------------------+
");
@@ -623,9 +623,9 @@ mod tests {
+-----------------------------------+-----------------+---------------------+------+------------------+
| filename | file_size_bytes |
metadata_size_bytes | hits | extra |
+-----------------------------------+-----------------+---------------------+------+------------------+
- | alltypes_plain.parquet | 1851 | 6957
| 5 | page_index=false |
- | alltypes_tiny_pages.parquet | 454233 | 267014
| 2 | page_index=true |
- | lz4_raw_compressed_larger.parquet | 380836 | 996
| 3 | page_index=false |
+ | alltypes_plain.parquet | 1851 | 8882
| 5 | page_index=false |
+ | alltypes_tiny_pages.parquet | 454233 | 269266
| 2 | page_index=true |
+ | lz4_raw_compressed_larger.parquet | 380836 | 1347
| 3 | page_index=false |
+-----------------------------------+-----------------+---------------------+------+------------------+
");
diff --git a/datafusion/common/src/error.rs b/datafusion/common/src/error.rs
index c58c33568c..cf951b17ac 100644
--- a/datafusion/common/src/error.rs
+++ b/datafusion/common/src/error.rs
@@ -1233,7 +1233,6 @@ mod test {
// To pass the test the environment variable RUST_BACKTRACE should be set
to 1 to enforce backtrace
#[cfg(feature = "backtrace")]
#[test]
- #[expect(clippy::unnecessary_literal_unwrap)]
fn test_enabled_backtrace() {
match std::env::var("RUST_BACKTRACE") {
Ok(val) if val == "1" => {}
diff --git a/datafusion/core/src/physical_planner.rs
b/datafusion/core/src/physical_planner.rs
index 6f0b201185..aea530d67f 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -3109,7 +3109,7 @@ mod tests {
assert_contains!(
&e,
- r#"Error during planning: Can not find compatible types to compare
Boolean with [Struct("foo": Boolean), Utf8]"#
+ r#"Error during planning: Can not find compatible types to compare
Boolean with [Struct("foo": non-null Boolean), Utf8]"#
);
Ok(())
diff --git a/datafusion/core/tests/dataframe/dataframe_functions.rs
b/datafusion/core/tests/dataframe/dataframe_functions.rs
index 265862ff9a..56cdd78d70 100644
--- a/datafusion/core/tests/dataframe/dataframe_functions.rs
+++ b/datafusion/core/tests/dataframe/dataframe_functions.rs
@@ -313,10 +313,10 @@ async fn test_fn_arrow_typeof() -> Result<()> {
+----------------------+
| arrow_typeof(test.l) |
+----------------------+
- | List(nullable Int32) |
- | List(nullable Int32) |
- | List(nullable Int32) |
- | List(nullable Int32) |
+ | List(Int32) |
+ | List(Int32) |
+ | List(Int32) |
+ | List(Int32) |
+----------------------+
");
diff --git a/datafusion/core/tests/parquet/filter_pushdown.rs
b/datafusion/core/tests/parquet/filter_pushdown.rs
index 966f251613..d371f86253 100644
--- a/datafusion/core/tests/parquet/filter_pushdown.rs
+++ b/datafusion/core/tests/parquet/filter_pushdown.rs
@@ -638,7 +638,9 @@ async fn predicate_cache_pushdown_default() ->
datafusion_common::Result<()> {
// The cache is on by default, and used when filter pushdown is enabled
PredicateCacheTest {
expected_inner_records: 8,
- expected_records: 4,
+ // reads more than necessary from the cache as then another bitmap is
applied
+ // See https://github.com/apache/datafusion/pull/18820 for setting and
workaround
+ expected_records: 7,
}
.run(&ctx)
.await
diff --git a/datafusion/core/tests/sql/select.rs
b/datafusion/core/tests/sql/select.rs
index 84899137e5..5a51451461 100644
--- a/datafusion/core/tests/sql/select.rs
+++ b/datafusion/core/tests/sql/select.rs
@@ -222,10 +222,10 @@ async fn test_parameter_invalid_types() -> Result<()> {
.await;
assert_snapshot!(results.unwrap_err().strip_backtrace(),
@r"
- type_coercion
- caused by
- Error during planning: Cannot infer common argument type for
comparison operation List(nullable Int32) = Int32
- ");
+ type_coercion
+ caused by
+ Error during planning: Cannot infer common argument type for comparison
operation List(Int32) = Int32
+ ");
Ok(())
}
diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs
b/datafusion/optimizer/src/analyzer/type_coercion.rs
index 4fb0f8553b..a557d3356d 100644
--- a/datafusion/optimizer/src/analyzer/type_coercion.rs
+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs
@@ -2465,7 +2465,7 @@ mod test {
assert_analyzed_plan_eq!(
plan,
@r#"
- Projection: a = CAST(CAST(a AS Map("key_value": Struct("key": Utf8,
"value": nullable Float64), unsorted)) AS Map("entries": Struct("key": Utf8,
"value": nullable Float64), unsorted))
+ Projection: a = CAST(CAST(a AS Map("key_value": non-null Struct("key":
non-null Utf8, "value": Float64), unsorted)) AS Map("entries": non-null
Struct("key": non-null Utf8, "value": Float64), unsorted))
EmptyRelation: rows=0
"#
)
diff --git a/datafusion/sqllogictest/test_files/array.slt
b/datafusion/sqllogictest/test_files/array.slt
index 77197721e1..15b50bef11 100644
--- a/datafusion/sqllogictest/test_files/array.slt
+++ b/datafusion/sqllogictest/test_files/array.slt
@@ -710,13 +710,13 @@ select
query TTT
select arrow_typeof(column1), arrow_typeof(column2), arrow_typeof(column3)
from arrays;
----
-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)
-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)
-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)
-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)
-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)
-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)
-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)
+List(List(Int64)) List(Float64) List(Utf8)
+List(List(Int64)) List(Float64) List(Utf8)
+List(List(Int64)) List(Float64) List(Utf8)
+List(List(Int64)) List(Float64) List(Utf8)
+List(List(Int64)) List(Float64) List(Utf8)
+List(List(Int64)) List(Float64) List(Utf8)
+List(List(Int64)) List(Float64) List(Utf8)
# arrays table
query ???
@@ -1182,7 +1182,7 @@ select make_array(make_array(1),
arrow_cast(make_array(-1), 'LargeList(Int8)'))
query T
select arrow_typeof(make_array(make_array(1), arrow_cast(make_array(-1),
'LargeList(Int8)')));
----
-List(nullable LargeList(nullable Int64))
+List(LargeList(Int64))
query ???
@@ -1978,11 +1978,11 @@ select array_slice(arrow_cast(make_array(1, 2, 3, 4,
5), 'LargeList(Int64)'), 0,
----
[1, 2, 3, 4, 5] [h, e, l, l, o]
-# TODO: Enable once arrow_cast supports ListView types.
+# TODO make error message nicer:
https://github.com/apache/datafusion/issues/19004
# Expected output (once supported):
# ----
# [1, 2, 3, 4, 5] [h, e, l, l, o]
-query error DataFusion error: Execution error: Unsupported type
'ListView\(Int64\)'. Must be a supported arrow type name such as 'Int32' or
'Timestamp\(ns\)'. Error unknown token: ListView
+query error Failed to coerce arguments to satisfy a call to 'array_slice'
function:
select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'ListView(Int64)'),
0, 6),
array_slice(arrow_cast(make_array('h', 'e', 'l', 'l', 'o'),
'ListView(Utf8)'), 0, 5);
@@ -2025,11 +2025,11 @@ select array_slice(arrow_cast(make_array(1, 2, 3, 4,
5), 'LargeList(Int64)'), 2,
----
[2, 3, 4, 5] [l, l, o]
-# TODO: Enable once arrow_cast supports LargeListView types.
+# TODO: Enable once array_slice supports LargeListView types.
# Expected output (once supported):
# ----
# [2, 3, 4, 5] [l, l, o]
-query error DataFusion error: Execution error: Unsupported type
'LargeListView\(Int64\)'. Must be a supported arrow type name such as 'Int32'
or 'Timestamp\(ns\)'. Error unknown token: LargeListView
+query error Failed to coerce arguments to satisfy a call to 'array_slice'
function:
select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5),
'LargeListView(Int64)'), 2, 6),
array_slice(arrow_cast(make_array('h', 'e', 'l', 'l', 'o'),
'LargeListView(Utf8)'), 3, 7);
@@ -3321,7 +3321,7 @@ select
array_concat([arrow_cast('1', 'Utf8'), arrow_cast('2', 'Utf8')],
[arrow_cast('3', 'Utf8View')]),
arrow_typeof(array_concat([arrow_cast('1', 'Utf8'), arrow_cast('2',
'Utf8')], [arrow_cast('3', 'Utf8View')]));
----
-[1, 2, 3] List(nullable Utf8View)
+[1, 2, 3] List(Utf8View)
# array_concat error
query error DataFusion error: Error during planning: Execution error: Function
'array_concat' user-defined coercion failed with "Error during planning:
array_concat does not support type Int64"
@@ -4614,7 +4614,7 @@ NULL [baz] baz
query T
SELECT arrow_typeof(make_array(arrow_cast('a', 'Utf8View'), 'b', 'c', 'd'));
----
-List(nullable Utf8View)
+List(Utf8View)
# expect a,b,c,d. make_array forces all types to be of a common type (see
above)
query T
@@ -7708,8 +7708,8 @@ CREATE EXTERNAL TABLE fixed_size_list_array STORED AS
PARQUET LOCATION '../core/
query T
select arrow_typeof(f0) from fixed_size_list_array;
----
-FixedSizeList(2 x nullable Int64)
-FixedSizeList(2 x nullable Int64)
+FixedSizeList(2 x Int64)
+FixedSizeList(2 x Int64)
query ?
select * from fixed_size_list_array;
@@ -7738,8 +7738,8 @@ select make_array(arrow_cast(f0, 'List(Int64)')) from
fixed_size_list_array
query T
select arrow_typeof(make_array(arrow_cast(f0, 'List(Int64)'))) from
fixed_size_list_array
----
-List(nullable List(nullable Int64))
-List(nullable List(nullable Int64))
+List(List(Int64))
+List(List(Int64))
query ?
select make_array(f0) from fixed_size_list_array
@@ -7750,8 +7750,8 @@ select make_array(f0) from fixed_size_list_array
query T
select arrow_typeof(make_array(f0)) from fixed_size_list_array
----
-List(nullable FixedSizeList(2 x nullable Int64))
-List(nullable FixedSizeList(2 x nullable Int64))
+List(FixedSizeList(2 x Int64))
+List(FixedSizeList(2 x Int64))
query ?
select array_concat(column1, [7]) from arrays_values_v2;
@@ -7798,7 +7798,7 @@ select flatten(arrow_cast(make_array([1], [2, 3], [null],
make_array(4, null, 5)
arrow_typeof(flatten(arrow_cast(make_array([1], [2, 3], [null],
make_array(4, null, 5)), 'FixedSizeList(4, LargeList(Int64))'))),
arrow_typeof(flatten(arrow_cast(make_array([[1.1], [2.2]], [[3.3],
[4.4]]), 'List(LargeList(FixedSizeList(1, Float64)))')));
----
-[1, 2, 3, NULL, 4, NULL, 5] [[1.1], [2.2], [3.3], [4.4]] LargeList(nullable
Int64) LargeList(nullable FixedSizeList(1 x nullable Float64))
+[1, 2, 3, NULL, 4, NULL, 5] [[1.1], [2.2], [3.3], [4.4]] LargeList(Int64)
LargeList(FixedSizeList(1 x Float64))
# flatten with column values
query ????
@@ -8338,19 +8338,19 @@ select * from test_create_array_table;
query T
select arrow_typeof(a) from test_create_array_table;
----
-List(nullable Int32)
+List(Int32)
query T
select arrow_typeof(c) from test_create_array_table;
----
-List(nullable List(nullable Int32))
+List(List(Int32))
# Test casting to array types
# issue: https://github.com/apache/datafusion/issues/9440
query ??T
select [1,2,3]::int[], [['1']]::int[][], arrow_typeof([]::text[]);
----
-[1, 2, 3] [[1]] List(nullable Utf8View)
+[1, 2, 3] [[1]] List(Utf8View)
# test empty arrays return length
# issue: https://github.com/apache/datafusion/pull/12459
@@ -8370,8 +8370,8 @@ create table fixed_size_col_table (a int[3]) as values
([1,2,3]), ([4,5,6]);
query T
select arrow_typeof(a) from fixed_size_col_table;
----
-FixedSizeList(3 x nullable Int32)
-FixedSizeList(3 x nullable Int32)
+FixedSizeList(3 x Int32)
+FixedSizeList(3 x Int32)
query ? rowsort
SELECT DISTINCT a FROM fixed_size_col_table
@@ -8407,7 +8407,6 @@ select array_contains(a, b) from array_has order by 1
nulls last;
true
NULL
-# TODO: Enable once arrow_cast supports ListView types.
# Expected output (once supported):
# ----
# [5, 4, 3, 2, 1]
diff --git a/datafusion/sqllogictest/test_files/arrow_typeof.slt
b/datafusion/sqllogictest/test_files/arrow_typeof.slt
index 5ba62be687..c213f2abf7 100644
--- a/datafusion/sqllogictest/test_files/arrow_typeof.slt
+++ b/datafusion/sqllogictest/test_files/arrow_typeof.slt
@@ -357,12 +357,12 @@ select arrow_cast(make_array(1, 2, 3), 'List(Int64)');
query T
select arrow_typeof(arrow_cast(make_array(1, 2, 3), 'List(Int64)'));
----
-List(nullable Int64)
+List(Int64)
query T
select arrow_typeof(arrow_cast(arrow_cast(make_array([1, 2, 3]),
'LargeList(LargeList(Int64))'), 'List(List(Int64))'));
----
-List(nullable List(nullable Int64))
+List(List(Int64))
## LargeList
@@ -380,12 +380,12 @@ select arrow_cast(make_array(1, 2, 3),
'LargeList(Int64)');
query T
select arrow_typeof(arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'));
----
-LargeList(nullable Int64)
+LargeList(Int64)
query T
select arrow_typeof(arrow_cast(make_array([1, 2, 3]),
'LargeList(LargeList(Int64))'));
----
-LargeList(nullable LargeList(nullable Int64))
+LargeList(LargeList(Int64))
## FixedSizeList
@@ -417,7 +417,7 @@ select arrow_cast(make_array(1, 2, 3), 'FixedSizeList(3,
Int64)');
query T
select arrow_typeof(arrow_cast(arrow_cast(make_array(1, 2, 3),
'LargeList(Int64)'), 'FixedSizeList(3, Int64)'));
----
-FixedSizeList(3 x nullable Int64)
+FixedSizeList(3 x Int64)
query ?
select arrow_cast([1, 2, 3], 'FixedSizeList(3, Int64)');
diff --git a/datafusion/sqllogictest/test_files/coalesce.slt
b/datafusion/sqllogictest/test_files/coalesce.slt
index e34a601851..9e5b71b871 100644
--- a/datafusion/sqllogictest/test_files/coalesce.slt
+++ b/datafusion/sqllogictest/test_files/coalesce.slt
@@ -199,14 +199,14 @@ select
coalesce(array[1, 2], array[3, 4]),
arrow_typeof(coalesce(array[1, 2], array[3, 4]));
----
-[1, 2] List(nullable Int64)
+[1, 2] List(Int64)
query ?T
select
coalesce(null, array[3, 4]),
arrow_typeof(coalesce(array[1, 2], array[3, 4]));
----
-[3, 4] List(nullable Int64)
+[3, 4] List(Int64)
# coalesce with array
query ?T
@@ -214,7 +214,7 @@ select
coalesce(array[1, 2], array[arrow_cast(3, 'Int32'), arrow_cast(4, 'Int32')]),
arrow_typeof(coalesce(array[1, 2], array[arrow_cast(3, 'Int32'),
arrow_cast(4, 'Int32')]));
----
-[1, 2] List(nullable Int64)
+[1, 2] List(Int64)
# test dict(int32, utf8)
statement ok
diff --git a/datafusion/sqllogictest/test_files/map.slt
b/datafusion/sqllogictest/test_files/map.slt
index 21fa72ad86..7ea54464d3 100644
--- a/datafusion/sqllogictest/test_files/map.slt
+++ b/datafusion/sqllogictest/test_files/map.slt
@@ -43,8 +43,8 @@ LOCATION '../core/tests/data/parquet_map.parquet';
query TTT
describe data;
----
-ints Map("entries": Struct("key": Utf8, "value": Int64), unsorted) NO
-strings Map("entries": Struct("key": Utf8, "value": Utf8), unsorted) NO
+ints Map("entries": non-null Struct("key": non-null Utf8, "value": non-null
Int64), unsorted) NO
+strings Map("entries": non-null Struct("key": non-null Utf8, "value": non-null
Utf8), unsorted) NO
timestamp Utf8View NO
query ??T
diff --git a/datafusion/sqllogictest/test_files/struct.slt
b/datafusion/sqllogictest/test_files/struct.slt
index dce5fe036b..0989dd382d 100644
--- a/datafusion/sqllogictest/test_files/struct.slt
+++ b/datafusion/sqllogictest/test_files/struct.slt
@@ -53,9 +53,9 @@ select * from struct_values;
query TT
select arrow_typeof(s1), arrow_typeof(s2) from struct_values;
----
-Struct("c0": nullable Int32) Struct("a": nullable Int32, "b": nullable
Utf8View)
-Struct("c0": nullable Int32) Struct("a": nullable Int32, "b": nullable
Utf8View)
-Struct("c0": nullable Int32) Struct("a": nullable Int32, "b": nullable
Utf8View)
+Struct("c0": Int32) Struct("a": Int32, "b": Utf8View)
+Struct("c0": Int32) Struct("a": Int32, "b": Utf8View)
+Struct("c0": Int32) Struct("a": Int32, "b": Utf8View)
# struct[i]
@@ -229,12 +229,12 @@ select named_struct('field_a', 1, 'field_b', 2);
query T
select arrow_typeof(named_struct('first', 1, 'second', 2, 'third', 3));
----
-Struct("first": nullable Int64, "second": nullable Int64, "third": nullable
Int64)
+Struct("first": Int64, "second": Int64, "third": Int64)
query T
select arrow_typeof({'first': 1, 'second': 2, 'third': 3});
----
-Struct("first": nullable Int64, "second": nullable Int64, "third": nullable
Int64)
+Struct("first": Int64, "second": Int64, "third": Int64)
# test nested struct literal
query ?
@@ -413,7 +413,7 @@ create table t(a struct<r varchar, c int>, b struct<r
varchar, c float>) as valu
query T
select arrow_typeof([a, b]) from t;
----
-List(nullable Struct("r": nullable Utf8View, "c": nullable Float32))
+List(Struct("r": Utf8View, "c": Float32))
query ?
select [a, b] from t;
@@ -464,12 +464,12 @@ select * from t;
query T
select arrow_typeof(c1) from t;
----
-Struct("r": nullable Utf8View, "b": nullable Int32)
+Struct("r": Utf8View, "b": Int32)
query T
select arrow_typeof(c2) from t;
----
-Struct("r": nullable Utf8View, "b": nullable Float32)
+Struct("r": Utf8View, "b": Float32)
statement ok
drop table t;
@@ -486,8 +486,8 @@ select * from t;
query T
select arrow_typeof(column1) from t;
----
-Struct("r": nullable Utf8, "c": nullable Float64)
-Struct("r": nullable Utf8, "c": nullable Float64)
+Struct("r": Utf8, "c": Float64)
+Struct("r": Utf8, "c": Float64)
statement ok
drop table t;
@@ -519,9 +519,9 @@ select coalesce(s1) from t;
query T
select arrow_typeof(coalesce(s1, s2)) from t;
----
-Struct("a": nullable Float32, "b": nullable Utf8View)
-Struct("a": nullable Float32, "b": nullable Utf8View)
-Struct("a": nullable Float32, "b": nullable Utf8View)
+Struct("a": Float32, "b": Utf8View)
+Struct("a": Float32, "b": Utf8View)
+Struct("a": Float32, "b": Utf8View)
statement ok
drop table t;
@@ -546,9 +546,9 @@ select coalesce(s1, s2) from t;
query T
select arrow_typeof(coalesce(s1, s2)) from t;
----
-Struct("a": nullable Float32, "b": nullable Utf8View)
-Struct("a": nullable Float32, "b": nullable Utf8View)
-Struct("a": nullable Float32, "b": nullable Utf8View)
+Struct("a": Float32, "b": Utf8View)
+Struct("a": Float32, "b": Utf8View)
+Struct("a": Float32, "b": Utf8View)
statement ok
drop table t;
@@ -583,7 +583,7 @@ create table t(a struct(r varchar, c int), b struct(r
varchar, c float)) as valu
query T
select arrow_typeof([a, b]) from t;
----
-List(nullable Struct("r": nullable Utf8View, "c": nullable Float32))
+List(Struct("r": Utf8View, "c": Float32))
statement ok
drop table t;
@@ -606,13 +606,13 @@ create table t(a struct(r varchar, c int, g float), b
struct(r varchar, c float,
query T
select arrow_typeof(a) from t;
----
-Struct("r": nullable Utf8View, "c": nullable Int32, "g": nullable Float32)
+Struct("r": Utf8View, "c": Int32, "g": Float32)
# type of each column should not coerced but perserve as it is
query T
select arrow_typeof(b) from t;
----
-Struct("r": nullable Utf8View, "c": nullable Float32, "g": nullable Int32)
+Struct("r": Utf8View, "c": Float32, "g": Int32)
statement ok
drop table t;
diff --git a/datafusion/sqllogictest/test_files/window.slt
b/datafusion/sqllogictest/test_files/window.slt
index acb2f07e01..818c861695 100644
--- a/datafusion/sqllogictest/test_files/window.slt
+++ b/datafusion/sqllogictest/test_files/window.slt
@@ -5906,7 +5906,7 @@ LIMIT 5
----
DataFusion error: type_coercion
caused by
-Error during planning: Cannot infer common argument type for comparison
operation Int64 >= List(nullable Null)
+Error during planning: Cannot infer common argument type for comparison
operation Int64 >= List(Null)
@@ -5934,7 +5934,7 @@ logical_plan
physical_plan
01)ProjectionExec: expr=[c1@2 as c1, c2@3 as c2, sum(test.c2) FILTER (WHERE
test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST]
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as sum1, sum(test.c2) FILTER
(WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER
BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED
PRECEDING AND CURRENT ROW@5 as sum2, count(test.c2) FILTER (WHERE test.c2 >=
Int64(2)) ORDER BY [test. [...]
02)--GlobalLimitExec: skip=0, fetch=5
-03)----BoundedWindowAggExec: wdw=[sum(test.c2) FILTER (WHERE test.c2 >=
Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS
BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(test.c2) FILTER
(WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS
LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 },
frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, sum(test.c2) FILTER
(WHERE test.c2 >= Int64(2) AND test.c2 < [...]
+03)----BoundedWindowAggExec: wdw=[sum(test.c2) FILTER (WHERE test.c2 >=
Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS
BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(test.c2) FILTER
(WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS
LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 },
frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, sum(test.c2) FILTER
(WHERE test.c2 >= Int64(2) AND test.c2 < [...]
04)------SortPreservingMergeExec: [c1@2 ASC NULLS LAST, c2@3 ASC NULLS LAST],
fetch=5
05)--------SortExec: TopK(fetch=5), expr=[c1@2 ASC NULLS LAST, c2@3 ASC NULLS
LAST], preserve_partitioning=[true]
06)----------DataSourceExec: file_groups={4 groups:
[[WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-0.csv],
[WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-1.csv],
[WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-2.csv],
[WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-3.csv]]},
projection=[c2@1 >= 2 as __common_expr_1, c2@1 >= 2 AND c2@1 < 4 AND c1@0 > 0
as __common_expr_2, c1, c2], file_type=csv, has_hea [...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
