This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 36f29053a7 Minor: consolidate some parquet_testing tests into
arrow_reader (#10584)
36f29053a7 is described below
commit 36f29053a72044505919e35f3b7c8095a8b31e21
Author: Andrew Lamb <[email protected]>
AuthorDate: Fri Aug 7 10:40:52 2026 -0400
Minor: consolidate some parquet_testing tests into arrow_reader (#10584)
# Which issue does this PR close?
- part of #10540
# Rationale for this change
The arrow_reader tests are over 4000 lines in one module. I would like
to try and break it up a bit so the tests are easier to understand and
navigate
I think tests that read example fixtures from parquet_testing is a
natural thing to consolidate (so among other things we can be sure we
are testing all the parquet_testing files)
# What changes are included in this PR?
1. Add parquet_testing.rs to the `arrow_reader` test
2. Move some tests
# Are these changes tested?
Only tests
# Are there any user-facing changes?
No
---
parquet/src/arrow/arrow_reader/mod.rs | 167 +---------------------
parquet/tests/arrow_reader/mod.rs | 1 +
parquet/tests/arrow_reader/parquet_testing.rs | 192 ++++++++++++++++++++++++++
3 files changed, 194 insertions(+), 166 deletions(-)
diff --git a/parquet/src/arrow/arrow_reader/mod.rs
b/parquet/src/arrow/arrow_reader/mod.rs
index 8d85dc088d..7ab219086f 100644
--- a/parquet/src/arrow/arrow_reader/mod.rs
+++ b/parquet/src/arrow/arrow_reader/mod.rs
@@ -1678,7 +1678,7 @@ pub(crate) mod tests {
virtual_type::{RowGroupIndex, RowNumber},
};
use crate::arrow::{ArrowWriter, ProjectionMask};
- use crate::basic::{ConvertedType, Encoding, LogicalType, Repetition, Type
as PhysicalType};
+ use crate::basic::{ConvertedType, Encoding, Repetition, Type as
PhysicalType};
use crate::column::reader::decoder::REPETITION_LEVELS_BATCH_SIZE;
use crate::data_type::{
BoolType, ByteArray, ByteArrayType, DataType, DoubleType,
FixedLenByteArray,
@@ -2417,106 +2417,6 @@ pub(crate) mod tests {
})
}
- #[test]
- fn test_int96_from_spark_file_with_provided_schema() {
- // int96_from_spark.parquet was written based on Spark's microsecond
timestamps which trade
- // range for resolution compared to a nanosecond timestamp. We must
provide a schema with
- // microsecond resolution for the Parquet reader to interpret these
values correctly.
- use arrow_schema::DataType::Timestamp;
- let test_data = arrow::util::test_util::parquet_test_data();
- let path = format!("{test_data}/int96_from_spark.parquet");
- let file = File::open(path).unwrap();
-
- let supplied_schema = Arc::new(Schema::new(vec![Field::new(
- "a",
- Timestamp(TimeUnit::Microsecond, None),
- true,
- )]));
- let options =
ArrowReaderOptions::new().with_schema(supplied_schema.clone());
-
- let mut record_reader =
- ParquetRecordBatchReaderBuilder::try_new_with_options(file,
options)
- .unwrap()
- .build()
- .unwrap();
-
- let batch = record_reader.next().unwrap().unwrap();
- assert_eq!(batch.num_columns(), 1);
- let column = batch.column(0);
- assert_eq!(column.data_type(), &Timestamp(TimeUnit::Microsecond,
None));
-
- let expected = Arc::new(Int64Array::from(vec![
- Some(1704141296123456),
- Some(1704070800000000),
- Some(253402225200000000),
- Some(1735599600000000),
- None,
- Some(9089380393200000000),
- ]));
-
- // arrow-rs relies on the chrono library to convert between timestamps
and strings, so
- // instead compare as Int64. The underlying type should be a
PrimitiveArray of Int64
- // anyway, so this should be a zero-copy non-modifying cast.
-
- let binding = arrow_cast::cast(batch.column(0),
&arrow_schema::DataType::Int64).unwrap();
- let casted_timestamps = binding.as_primitive::<types::Int64Type>();
-
- assert_eq!(casted_timestamps.len(), expected.len());
-
- casted_timestamps
- .iter()
- .zip(expected.iter())
- .for_each(|(lhs, rhs)| {
- assert_eq!(lhs, rhs);
- });
- }
-
- #[test]
- fn test_int96_from_spark_file_without_provided_schema() {
- // int96_from_spark.parquet was written based on Spark's microsecond
timestamps which trade
- // range for resolution compared to a nanosecond timestamp. Without a
provided schema, some
- // values when read as nanosecond resolution overflow and result in
garbage values.
- use arrow_schema::DataType::Timestamp;
- let test_data = arrow::util::test_util::parquet_test_data();
- let path = format!("{test_data}/int96_from_spark.parquet");
- let file = File::open(path).unwrap();
-
- let mut record_reader = ParquetRecordBatchReaderBuilder::try_new(file)
- .unwrap()
- .build()
- .unwrap();
-
- let batch = record_reader.next().unwrap().unwrap();
- assert_eq!(batch.num_columns(), 1);
- let column = batch.column(0);
- assert_eq!(column.data_type(), &Timestamp(TimeUnit::Nanosecond, None));
-
- let expected = Arc::new(Int64Array::from(vec![
- Some(1704141296123456000), // Reads as nanosecond fine (note 3
extra 0s)
- Some(1704070800000000000), // Reads as nanosecond fine (note 3
extra 0s)
- Some(-4852191831933722624), // Cannot be represented with nanos
timestamp (year 9999)
- Some(1735599600000000000), // Reads as nanosecond fine (note 3
extra 0s)
- None,
- Some(-4864435138808946688), // Cannot be represented with nanos
timestamp (year 290000)
- ]));
-
- // arrow-rs relies on the chrono library to convert between timestamps
and strings, so
- // instead compare as Int64. The underlying type should be a
PrimitiveArray of Int64
- // anyway, so this should be a zero-copy non-modifying cast.
-
- let binding = arrow_cast::cast(batch.column(0),
&arrow_schema::DataType::Int64).unwrap();
- let casted_timestamps = binding.as_primitive::<types::Int64Type>();
-
- assert_eq!(casted_timestamps.len(), expected.len());
-
- casted_timestamps
- .iter()
- .zip(expected.iter())
- .for_each(|(lhs, rhs)| {
- assert_eq!(lhs, rhs);
- });
- }
-
struct RandUtf8Gen {}
impl RandGen<ByteArrayType> for RandUtf8Gen {
@@ -5408,71 +5308,6 @@ pub(crate) mod tests {
assert_eq!(r0arr, &a);
}
- #[test]
- fn test_map_no_value() {
- // File schema:
- // message schema {
- // required group my_map (MAP) {
- // repeated group key_value {
- // required int32 key;
- // optional int32 value;
- // }
- // }
- // required group my_map_no_v (MAP) {
- // repeated group key_value {
- // required int32 key;
- // }
- // }
- // required group my_list (LIST) {
- // repeated group list {
- // required int32 element;
- // }
- // }
- // }
- let testdata = arrow::util::test_util::parquet_test_data();
- let path = format!("{testdata}/map_no_value.parquet");
- let file = File::open(path).unwrap();
-
- let mut reader = ParquetRecordBatchReaderBuilder::try_new(file)
- .unwrap()
- .build()
- .unwrap();
- let out = reader.next().unwrap().unwrap();
- assert_eq!(out.num_rows(), 3);
- assert_eq!(out.num_columns(), 3);
- // my_map_no_v and my_list columns should now be equivalent
- let c0 = out.column(1).as_list::<i32>();
- let c1 = out.column(2).as_list::<i32>();
- assert_eq!(c0.len(), c1.len());
- c0.iter().zip(c1.iter()).for_each(|(l, r)| assert_eq!(l, r));
- }
-
- #[test]
- fn test_read_unknown_logical_type() {
- let testdata = arrow::util::test_util::parquet_test_data();
- let path = format!("{testdata}/unknown-logical-type.parquet");
- let test_file = File::open(path).unwrap();
-
- let builder = ParquetRecordBatchReaderBuilder::try_new(test_file)
- .expect("Error creating reader builder");
-
- let schema = builder.metadata().file_metadata().schema_descr();
- assert_eq!(
- schema.column(0).logical_type_ref(),
- Some(&LogicalType::String)
- );
- assert_eq!(
- schema.column(1).logical_type_ref(),
- Some(&LogicalType::_Unknown { field_id: 2555 })
- );
- assert_eq!(schema.column(1).physical_type(), PhysicalType::BYTE_ARRAY);
-
- let mut reader = builder.build().unwrap();
- let out = reader.next().unwrap().unwrap();
- assert_eq!(out.num_rows(), 3);
- assert_eq!(out.num_columns(), 2);
- }
-
#[test]
fn test_read_row_numbers() {
let file = write_parquet_from_iter(vec![(
diff --git a/parquet/tests/arrow_reader/mod.rs
b/parquet/tests/arrow_reader/mod.rs
index 404bebc05d..c4c15d77a2 100644
--- a/parquet/tests/arrow_reader/mod.rs
+++ b/parquet/tests/arrow_reader/mod.rs
@@ -46,6 +46,7 @@ mod int96_stats_roundtrip;
mod invalid_utf8;
mod io;
mod large_string_overflow;
+mod parquet_testing;
#[cfg(feature = "async")]
mod predicate_cache;
mod row_filter;
diff --git a/parquet/tests/arrow_reader/parquet_testing.rs
b/parquet/tests/arrow_reader/parquet_testing.rs
new file mode 100644
index 0000000000..daa3ff79d2
--- /dev/null
+++ b/parquet/tests/arrow_reader/parquet_testing.rs
@@ -0,0 +1,192 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Tests with interoperability files in [parquet-testing]
+//!
+//! [parquet-testing]: https://github.com/apache/parquet-testing
+
+use arrow_array::cast::AsArray;
+use arrow_array::{Array, Int64Array, types};
+use arrow_schema::{Field, Schema, TimeUnit};
+use parquet::arrow::arrow_reader::{ArrowReaderOptions,
ParquetRecordBatchReaderBuilder};
+use parquet::basic::{LogicalType, Type as PhysicalType};
+use std::fs::File;
+use std::sync::Arc;
+
+#[test]
+fn test_int96_from_spark_file_with_provided_schema() {
+ // int96_from_spark.parquet was written based on Spark's microsecond
timestamps which trade
+ // range for resolution compared to a nanosecond timestamp. We must
provide a schema with
+ // microsecond resolution for the Parquet reader to interpret these values
correctly.
+ use arrow_schema::DataType::Timestamp;
+ let test_data = arrow::util::test_util::parquet_test_data();
+ let path = format!("{test_data}/int96_from_spark.parquet");
+ let file = File::open(path).unwrap();
+
+ let supplied_schema = Arc::new(Schema::new(vec![Field::new(
+ "a",
+ Timestamp(TimeUnit::Microsecond, None),
+ true,
+ )]));
+ let options =
ArrowReaderOptions::new().with_schema(supplied_schema.clone());
+
+ let mut record_reader =
ParquetRecordBatchReaderBuilder::try_new_with_options(file, options)
+ .unwrap()
+ .build()
+ .unwrap();
+
+ let batch = record_reader.next().unwrap().unwrap();
+ assert_eq!(batch.num_columns(), 1);
+ let column = batch.column(0);
+ assert_eq!(column.data_type(), &Timestamp(TimeUnit::Microsecond, None));
+
+ let expected = Arc::new(Int64Array::from(vec![
+ Some(1704141296123456),
+ Some(1704070800000000),
+ Some(253402225200000000),
+ Some(1735599600000000),
+ None,
+ Some(9089380393200000000),
+ ]));
+
+ // arrow-rs relies on the chrono library to convert between timestamps and
strings, so
+ // instead compare as Int64. The underlying type should be a
PrimitiveArray of Int64
+ // anyway, so this should be a zero-copy non-modifying cast.
+
+ let binding = arrow_cast::cast(batch.column(0),
&arrow_schema::DataType::Int64).unwrap();
+ let casted_timestamps = binding.as_primitive::<types::Int64Type>();
+
+ assert_eq!(casted_timestamps.len(), expected.len());
+
+ casted_timestamps
+ .iter()
+ .zip(expected.iter())
+ .for_each(|(lhs, rhs)| {
+ assert_eq!(lhs, rhs);
+ });
+}
+
+#[test]
+fn test_int96_from_spark_file_without_provided_schema() {
+ // int96_from_spark.parquet was written based on Spark's microsecond
timestamps which trade
+ // range for resolution compared to a nanosecond timestamp. Without a
provided schema, some
+ // values when read as nanosecond resolution overflow and result in
garbage values.
+ use arrow_schema::DataType::Timestamp;
+ let test_data = arrow::util::test_util::parquet_test_data();
+ let path = format!("{test_data}/int96_from_spark.parquet");
+ let file = File::open(path).unwrap();
+
+ let mut record_reader = ParquetRecordBatchReaderBuilder::try_new(file)
+ .unwrap()
+ .build()
+ .unwrap();
+
+ let batch = record_reader.next().unwrap().unwrap();
+ assert_eq!(batch.num_columns(), 1);
+ let column = batch.column(0);
+ assert_eq!(column.data_type(), &Timestamp(TimeUnit::Nanosecond, None));
+
+ let expected = Arc::new(Int64Array::from(vec![
+ Some(1704141296123456000), // Reads as nanosecond fine (note 3 extra
0s)
+ Some(1704070800000000000), // Reads as nanosecond fine (note 3 extra
0s)
+ Some(-4852191831933722624), // Cannot be represented with nanos
timestamp (year 9999)
+ Some(1735599600000000000), // Reads as nanosecond fine (note 3 extra
0s)
+ None,
+ Some(-4864435138808946688), // Cannot be represented with nanos
timestamp (year 290000)
+ ]));
+
+ // arrow-rs relies on the chrono library to convert between timestamps and
strings, so
+ // instead compare as Int64. The underlying type should be a
PrimitiveArray of Int64
+ // anyway, so this should be a zero-copy non-modifying cast.
+
+ let binding = arrow_cast::cast(batch.column(0),
&arrow_schema::DataType::Int64).unwrap();
+ let casted_timestamps = binding.as_primitive::<types::Int64Type>();
+
+ assert_eq!(casted_timestamps.len(), expected.len());
+
+ casted_timestamps
+ .iter()
+ .zip(expected.iter())
+ .for_each(|(lhs, rhs)| {
+ assert_eq!(lhs, rhs);
+ });
+}
+
+#[test]
+fn test_map_no_value() {
+ // File schema:
+ // message schema {
+ // required group my_map (MAP) {
+ // repeated group key_value {
+ // required int32 key;
+ // optional int32 value;
+ // }
+ // }
+ // required group my_map_no_v (MAP) {
+ // repeated group key_value {
+ // required int32 key;
+ // }
+ // }
+ // required group my_list (LIST) {
+ // repeated group list {
+ // required int32 element;
+ // }
+ // }
+ // }
+ let testdata = arrow::util::test_util::parquet_test_data();
+ let path = format!("{testdata}/map_no_value.parquet");
+ let file = File::open(path).unwrap();
+
+ let mut reader = ParquetRecordBatchReaderBuilder::try_new(file)
+ .unwrap()
+ .build()
+ .unwrap();
+ let out = reader.next().unwrap().unwrap();
+ assert_eq!(out.num_rows(), 3);
+ assert_eq!(out.num_columns(), 3);
+ // my_map_no_v and my_list columns should now be equivalent
+ let c0 = out.column(1).as_list::<i32>();
+ let c1 = out.column(2).as_list::<i32>();
+ assert_eq!(c0.len(), c1.len());
+ c0.iter().zip(c1.iter()).for_each(|(l, r)| assert_eq!(l, r));
+}
+
+#[test]
+fn test_read_unknown_logical_type() {
+ let testdata = arrow::util::test_util::parquet_test_data();
+ let path = format!("{testdata}/unknown-logical-type.parquet");
+ let test_file = File::open(path).unwrap();
+
+ let builder =
+ ParquetRecordBatchReaderBuilder::try_new(test_file).expect("Error
creating reader builder");
+
+ let schema = builder.metadata().file_metadata().schema_descr();
+ assert_eq!(
+ schema.column(0).logical_type_ref(),
+ Some(&LogicalType::String)
+ );
+ assert_eq!(
+ schema.column(1).logical_type_ref(),
+ Some(&LogicalType::_Unknown { field_id: 2555 })
+ );
+ assert_eq!(schema.column(1).physical_type(), PhysicalType::BYTE_ARRAY);
+
+ let mut reader = builder.build().unwrap();
+ let out = reader.next().unwrap().unwrap();
+ assert_eq!(out.num_rows(), 3);
+ assert_eq!(out.num_columns(), 2);
+}
