kosiew commented on code in PR #26668:
URL: https://github.com/apache/datafusion/pull/26668#discussion_r4549213421
##########
datafusion/datasource-parquet/src/nested_schema_pruning.rs:
##########
@@ -602,17 +624,48 @@ mod tests {
/// must fall back instead of predicting a type the reader may not emit.
#[test]
fn type_for_leaf_subset_rejects_unsupported_partial_wrapper() {
- let physical = DataType::FixedSizeList(
+ let physical = DataType::RunEndEncoded(
+ Arc::new(Field::new("run_ends", DataType::Int32, false)),
Arc::new(Field::new(
- "item",
+ "values",
struct_of(vec![int64("x"), utf8("pad")]),
true,
)),
- 2,
);
assert!(type_for_leaf_subset(&physical, &[0]).is_none());
}
+ /// The read plan keeps only [`clip_for_cast`]'s leaf offsets and derives
+ /// the emitted type from them, so [`type_for_leaf_subset`] must reproduce
+ /// the clipped type for every wrapper the clip descends through.
+ #[test]
+ fn type_for_leaf_subset_matches_clip_through_wrappers() {
+ let item = |fields| Arc::new(Field::new("item", struct_of(fields),
true));
+ let wrappers: [fn(FieldRef) -> DataType; 6] = [
+ DataType::List,
+ DataType::LargeList,
+ DataType::ListView,
+ DataType::LargeListView,
+ |item| DataType::FixedSizeList(item, 2),
+ |item| {
+ DataType::Dictionary(
+ Box::new(DataType::Int32),
+ Box::new(item.data_type().clone()),
+ )
+ },
+ ];
+ for wrap in wrappers {
+ let physical = wrap(item(vec![int64("x"), utf8("pad")]));
+ let target = wrap(item(vec![int64("x")]));
Review Comment:
Could we optionally extend this test with physical `Struct<skip:
Wrapper<Struct<a,b>>, keep: Wrapper<Struct<pad,x>>>` and target `Struct<keep:
Wrapper<Struct<x>>>`, asserting offsets `[3]` and an independently constructed
expected type? This would cover cursor advancement past a dropped wrapper and
selection of a non-leading child. Existing generic coverage is helpful, so this
is not a blocker.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
