HippoBaro commented on code in PR #11278:
URL: https://github.com/apache/arrow-rs/pull/11278#discussion_r4174805519
##########
parquet/src/arrow/schema/mod.rs:
##########
@@ -313,12 +313,132 @@ pub fn encode_arrow_schema(schema: &Schema) -> String {
BASE64_STANDARD.encode(&len_prefix_schema)
}
-fn flatten_ree_field(field: &Field) -> Field {
+/// Reject map keys that are nullable after resolving their encoded layout.
+/// This validates the writer's logical contract, not a batch's physical
layout.
+pub(crate) fn validate_map_key_type(data_type: &DataType) -> Result<()> {
+ match data_type {
+ DataType::Map(entries, _) => {
+ let DataType::Struct(fields) = entries.data_type() else {
+ return Err(arrow_err!(
+ "DataType::Map should contain a struct field child"
+ ));
+ };
+ if fields.len() != 2 {
+ return Err(arrow_err!("DataType::Map entries must contain two
fields"));
+ }
+ let key = &fields[0];
+ let mut key_type = key.data_type();
+ let mut nullable = key.is_nullable();
+ loop {
+ match key_type {
+ DataType::Dictionary(_, value) => key_type = value,
+ DataType::RunEndEncoded(_, value) => {
+ nullable |= value.is_nullable();
+ key_type = value.data_type();
+ }
+ _ => break,
+ }
+ }
+ if nullable {
+ return Err(arrow_err!(
+ "Map key field '{}' must be non-nullable, including its
encoded values",
+ key.name()
+ ));
+ }
+ validate_map_key_type(entries.data_type())?;
+ }
+ DataType::Struct(fields) => {
+ for field in fields {
+ validate_map_key_type(field.data_type())?;
+ }
+ }
+ DataType::List(field)
+ | DataType::LargeList(field)
+ | DataType::FixedSizeList(field, _)
+ | DataType::ListView(field)
+ | DataType::LargeListView(field)
+ | DataType::RunEndEncoded(_, field) =>
validate_map_key_type(field.data_type())?,
+ DataType::Dictionary(_, value) => validate_map_key_type(value)?,
+ _ => {}
+ }
+ Ok(())
+}
+
+/// Normalize `field` for Arrow schema metadata.
+///
+/// This removes wrappers the Parquet reader does not reconstruct: REE at any
+/// nesting depth and dictionaries around nested values. Scalar dictionaries,
+/// including `Utf8View` and `BinaryView`, are retained in the metadata.
+fn normalize_metadata_field(field: &Field) -> Field {
Review Comment:
Yep!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
