alamb commented on code in PR #25335:
URL: https://github.com/apache/datafusion/pull/25335#discussion_r4018348354
##########
Cargo.toml:
##########
@@ -179,9 +179,9 @@ log = "^0.4"
loom = "0.7"
memchr = "2.8.1"
num-traits = { version = "0.2" }
-object_store = { version = "0.13.2", default-features = false }
+object_store = { version = "0.14.2", default-features = false }
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10375
##########
datafusion-cli/src/main.rs:
##########
@@ -662,9 +662,9 @@ mod tests {
+-----------------------------------+-----------------+---------------------+------+------------------+
| filename | file_size_bytes |
metadata_size_bytes | hits | extra |
+-----------------------------------+-----------------+---------------------+------+------------------+
- | alltypes_plain.parquet | 1851 | 8794
| 1 | page_index=false |
- | alltypes_tiny_pages.parquet | 454233 | 268970
| 2 | page_index=true |
- | lz4_raw_compressed_larger.parquet | 380836 | 1331
| 1 | page_index=false |
+ | alltypes_plain.parquet | 1851 | 8938
| 1 | page_index=false |
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/9619 (adds `nan_count` to
`Statistics`, 16 bytes per column chunk)
- due to https://github.com/apache/arrow-rs/pull/10842 (`ParquetMetaData`
holds an `Arc<dyn PageIndexProvider>` instead of separate column/offset index
vecs, 32 bytes smaller)
- due to https://github.com/apache/arrow-rs/pull/10149 (row group ordinals
are `i32` rather than `i16`)
##########
Cargo.toml:
##########
@@ -90,31 +90,31 @@ version = "55.1.0"
#
# See for more details: https://github.com/rust-lang/cargo/issues/11329
apache-avro = { version = "0.21", default-features = false }
-arrow = { version = "59.2.0", features = [
+arrow = { version = "60.0.0", features = [
"prettyprint",
"chrono-tz",
] }
-arrow-avro = { version = "59.2.0", default-features = false, features = [
+arrow-avro = { version = "60.0.0", default-features = false, features = [
"deflate",
"snappy",
"zstd",
"bzip2",
"xz",
] }
-arrow-buffer = { version = "59.2.0", default-features = false }
-arrow-data = { version = "59.2.0", default-features = false }
-arrow-flight = { version = "59.2.0", features = [
- "flight-sql-experimental",
+arrow-buffer = { version = "60.0.0", default-features = false }
+arrow-data = { version = "60.0.0", default-features = false }
+arrow-flight = { version = "60.0.0", features = [
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/7546
- due to https://github.com/apache/arrow-rs/pull/10335
##########
datafusion/catalog-listing/src/helpers.rs:
##########
@@ -30,7 +30,7 @@ use datafusion_expr::{BinaryExpr, Operator, lit, utils};
use arrow::{
array::AsArray,
- datatypes::{DataType, Field},
+ datatypes::{DataType, Field, Metadata},
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10075
##########
datafusion/common/src/utils/memory.rs:
##########
@@ -330,7 +330,7 @@ impl RecordBatchMemoryCounter {
array: &GenericByteViewArray<T>,
) {
self.count_buffer_memory_size(array.views().inner());
- for buffer in array.data_buffers() {
+ for buffer in array.data_buffers().iter() {
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10708
##########
datafusion/datasource-arrow/src/file_format.rs:
##########
@@ -26,7 +26,7 @@ use std::sync::Arc;
use arrow::datatypes::{Schema, SchemaRef};
use arrow::error::ArrowError;
-use arrow::ipc::convert::fb_to_schema;
+use arrow::ipc::convert::try_fb_to_schema;
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10647
##########
datafusion/datasource-parquet/src/statistics_order_tests.rs:
##########
@@ -35,6 +35,7 @@ use parquet::arrow::ArrowWriter;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::basic::{ColumnOrder, LogicalType, SortOrder, Type as
PhysicalType};
use parquet::data_type::{ByteArray, FixedLenByteArray};
+use parquet::file::metadata::page_index::{PageIndex, PageIndexBuilder};
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10842
- due to https://github.com/apache/arrow-rs/pull/9619
##########
datafusion/datasource-parquet/src/statistics_order_tests.rs:
##########
@@ -177,8 +187,10 @@ impl TestFile {
bytes[new_end..new_end +
4].copy_from_slice(&metadata_len.to_le_bytes());
} else {
// Change the first union member from field 1 (TYPEORDER) to
- // an unrecognized field 2. The numeric column stays known.
- bytes[start + 2] = 0x2c;
+ // an unrecognized field 4 (fields 2 and 3 are
+ // IEEE754TotalOrder and INT96TimestampOrder as of arrow 60).
+ // The numeric column stays known.
+ bytes[start + 2] = 0x4c;
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/9619
- due to https://github.com/apache/arrow-rs/pull/10106
##########
datafusion/physical-plan/src/spill/spill_manager.rs:
##########
@@ -232,35 +231,14 @@ impl GetSlicedSize for RecordBatch {
let mut total = 0;
for array in self.columns() {
let data = array.to_data();
+ // Since https://github.com/apache/arrow-rs/issues/8230 this also
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10519
##########
datafusion/sqllogictest/test_files/datetime/arith_interval_double.slt:
##########
@@ -35,7 +35,7 @@ query error Invalid interval arithmetic operation:
Interval\(MonthDayNano\) \* I
SELECT interval '1 day' * 21
-query error Invalid interval arithmetic operation: Interval\(MonthDayNano\) \*
Interval\(MonthDayNano\)
+query error DataFusion error: Arrow error: Invalid argument error: Invalid
interval arithmetic operation: Interval\(MonthDayNano\) \* Float64
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10336
- due to https://github.com/apache/arrow-rs/pull/10409
##########
datafusion/common/src/utils/mod.rs:
##########
@@ -1246,13 +1246,11 @@ fn sliced_list_values<O: OffsetSizeTrait>(list:
&GenericListArray<O>) -> ArrayRe
let values = list.values();
let offsets = list.offsets();
- if let (Some(first), Some(last)) = (offsets.first(), offsets.last()) {
- let first = first.as_usize();
- let last = last.as_usize();
+ let first = offsets.first().as_usize();
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10759
##########
datafusion/sqllogictest/test_files/explain_analyze.slt:
##########
@@ -159,7 +159,7 @@ SELECT a+1, pow(a,2)
FROM generate_series(1, 100) as t1(a);
----
Plan with Metrics
-01)ProjectionExec: expr=[a@0 + 1 as t1.a + Int64(1), power(CAST(a@0 AS
Float64), 2) as pow(t1.a,Int64(2))], metrics=[output_rows=100,
elapsed_compute=<slt:ignore>, output_bytes=1632.0 B, output_batches=1,
expr_0_eval_time=<slt:ignore>, expr_1_eval_time=<slt:ignore>]
+01)ProjectionExec: expr=[a@0 + 1 as t1.a + Int64(1), power(CAST(a@0 AS
Float64), 2) as pow(t1.a,Int64(2))], metrics=[output_rows=100,
elapsed_compute=<slt:ignore>, output_bytes=1600.0 B, output_batches=1,
expr_0_eval_time=<slt:ignore>, expr_1_eval_time=<slt:ignore>]
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10783
##########
datafusion/datasource-parquet/src/metadata.rs:
##########
@@ -80,7 +80,12 @@ pub(crate) fn has_untrusted_min_max_order(
parquet_column_index: usize,
) -> bool {
let column = parquet_schema.column(parquet_column_index);
- if column.sort_order() == SortOrder::UNDEFINED {
+ // As of arrow 60, INT96 columns report `SortOrder::INT96_TIMESTAMP`
+ // rather than `UNDEFINED`; keep treating their min/max as untrusted.
+ if matches!(
+ column.sort_order(),
+ SortOrder::UNDEFINED | SortOrder::INT96_TIMESTAMP
Review Comment:
we may want to revisit this one
##########
datafusion-cli/src/main.rs:
##########
@@ -862,14 +862,14 @@ mod tests {
])?
.sort(vec![col("filename").sort(true, false)])?;
let rbs = df.collect().await?;
- assert_snapshot!(batches_to_string(&rbs),@r"
+ assert_snapshot!(batches_to_string(&rbs),@r#"
+---------------------+-----------+-----------------+------+------+
| metadata_size_bytes | filename | file_size_bytes | etag | hits |
+---------------------+-----------+-----------------+------+------+
- | 212 | 0.parquet | 3642 | 0 | 2 |
- | 212 | 1.parquet | 3642 | 1 | 2 |
+ | 216 | 0.parquet | 3620 | "0" | 2 |
Review Comment:
- etag changes due to
https://github.com/apache/arrow-rs-object-store/pull/770
- the size got bigger due to new ordering info
https://github.com/apache/arrow-rs/pull/9619
##########
datafusion/common/src/nested_struct.rs:
##########
@@ -1821,8 +1821,8 @@ mod tests {
Arc::new(non_null_field(
"entries",
struct_type(vec![
- non_null_field("keys", DataType::Utf8),
- field("values", DataType::Int32),
+ non_null_field("key", DataType::Utf8),
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10517
##########
datafusion/core/src/datasource/file_format/csv.rs:
##########
@@ -136,6 +136,7 @@ mod tests {
},
range: Default::default(),
attributes: Attributes::default(),
+ extensions: Default::default(),
Review Comment:
- due to https://github.com/apache/arrow-rs-object-store/pull/743
##########
datafusion/datasource-parquet/src/metadata.rs:
##########
@@ -80,7 +80,12 @@ pub(crate) fn has_untrusted_min_max_order(
parquet_column_index: usize,
) -> bool {
let column = parquet_schema.column(parquet_column_index);
- if column.sort_order() == SortOrder::UNDEFINED {
+ // As of arrow 60, INT96 columns report `SortOrder::INT96_TIMESTAMP`
+ // rather than `UNDEFINED`; keep treating their min/max as untrusted.
+ if matches!(
+ column.sort_order(),
+ SortOrder::UNDEFINED | SortOrder::INT96_TIMESTAMP
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10106
##########
datafusion/core/src/datasource/file_format/parquet.rs:
##########
@@ -165,10 +165,8 @@ mod tests {
};
use parquet::arrow::ParquetRecordBatchStreamBuilder;
use parquet::arrow::arrow_reader::ArrowReaderOptions;
- use parquet::file::metadata::{
- KeyValue, PageIndexPolicy, ParquetColumnIndex, ParquetMetaData,
- ParquetOffsetIndex,
- };
+ use parquet::file::metadata::page_index::PageIndexProvider;
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10719
- due to https://github.com/apache/arrow-rs/pull/10842
##########
datafusion/functions-aggregate/benches/array_agg.rs:
##########
@@ -65,8 +64,7 @@ pub fn create_list_array<T>(
zero_length_lists_probability: f32,
) -> ListArray
where
- T: ArrowPrimitiveType,
- StandardUniform: Distribution<T::Native>,
+ T: ArrowPrimitiveType<Native = i64>,
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10566
##########
datafusion/physical-expr/benches/case_when.rs:
##########
@@ -28,7 +27,13 @@ use itertools::Itertools;
use rand::distr::Alphanumeric;
use rand::distr::uniform::SampleUniform;
use rand::rngs::StdRng;
-use rand::{Rng, RngCore};
+use rand::{Rng, RngCore, SeedableRng};
+
+/// Returns a fixed-seed RNG using this crate's `rand` version (arrow's
+/// `test_util::seedable_rng` returns its own `rand` version's `StdRng`)
+fn seedable_rng() -> StdRng {
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10566
##########
datafusion/functions/src/core/arrow_field.rs:
##########
@@ -125,8 +126,18 @@ impl ScalarUDFImpl for ArrowFieldFunc {
// Build the metadata map array (same pattern as arrow_metadata.rs)
let metadata = field.metadata();
- let mut map_builder =
- MapBuilder::new(None, StringBuilder::new(), StringBuilder::new());
+ // Match the field names declared in `return_type` (the arrow-rs
+ // default changed to `key`/`value` in arrow 60)
+ let map_field_names = MapFieldNames {
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10517
##########
datafusion/sqllogictest/test_files/push_down_filter_parquet.slt:
##########
@@ -268,7 +268,7 @@ EXPLAIN ANALYZE SELECT * FROM topk_single_col ORDER BY b
DESC LIMIT 1;
----
Plan with Metrics
01)SortExec: TopK(fetch=1), expr=[b@1 DESC], preserve_partitioning=[false],
filter=[b@1 IS NULL OR b@1 > bd], metrics=[output_rows=1, output_batches=1,
row_replacements=1]
-02)--DataSourceExec: file_groups={1 group:
[[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]},
projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 IS NULL
OR b@1 > bd ], sort_order_for_reorder=[b@1 DESC], reverse_row_groups=true,
dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@0 > 0 OR
b_null_count@0 != row_count@2 AND b_max@1 > bd, required_guarantees=[],
metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1
total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched,
row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0
total → 0 matched, page_index_rows_pruned=0 total → 0 matched,
limit_pruned_row_groups=0 total → 0 matched, batches_split=0,
file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1,
num_predicate_creation_errors=0, predicate_evaluation_errors=0,
pushdown_rows_matched=4, pushdown_rows_pruned=0,
predicate_cache_inner_records=4, predicate_cache_records=4,
scan_efficiency_ratio=21.62% (222/1.03 K)]
+02)--DataSourceExec: file_groups={1 group:
[[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]},
projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 IS NULL
OR b@1 > bd ], sort_order_for_reorder=[b@1 DESC], reverse_row_groups=true,
dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@0 > 0 OR
b_null_count@0 != row_count@2 AND b_max@1 > bd, required_guarantees=[],
metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1
total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched,
row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0
total → 0 matched, page_index_rows_pruned=0 total → 0 matched,
limit_pruned_row_groups=0 total → 0 matched, batches_split=0,
file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1,
num_predicate_creation_errors=0, predicate_evaluation_errors=0,
pushdown_rows_matched=4, pushdown_rows_pruned=0,
predicate_cache_inner_records=4, predicate_cache_records=4,
scan_efficiency_ratio=21.94% (222/1.01 K)]
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/9619
##########
datafusion/sqllogictest/test_files/scalar.slt:
##########
@@ -1128,7 +1128,7 @@ select arrow_typeof(round(cast(500 as decimal(10,-2)),
-3)),
round(cast(400 as decimal(10,-2)), -3),
round(cast(-500 as decimal(10,-2)), -3);
----
-Decimal128(10, -3) 1000 0000 -1000
+Decimal128(10, -3) 1000 0 -1000
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10869
##########
datafusion/common/src/scalar/mod.rs:
##########
@@ -11109,7 +11109,7 @@ mod tests {
Box::new(ScalarValue::Float32(None)),
);
let err = scalar.eq_array(&run_array, 0).unwrap_err();
- let expected = "Internal error: could not cast array of type
RunEndEncoded(\"run_ends\": non-null Int16, \"values\": Float32) to
arrow_array::array::run_array::RunArray<arrow_array::types::Int32Type>";
+ let expected = "Internal error: could not cast array of type
RunEndEncoded(non-null Int16, Float32) to
arrow_array::array::run_array::RunArray<arrow_array::types::Int32Type>";
Review Comment:
- due to https://github.com/apache/arrow-rs/pull/10840
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
