alamb commented on code in PR #8166: URL: https://github.com/apache/arrow-rs/pull/8166#discussion_r2323365784
########## parquet-variant-compute/src/variant_get/output/row_builder.rs: ########## @@ -0,0 +1,211 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow::array::ArrayRef; +use arrow::datatypes; +use arrow::datatypes::ArrowPrimitiveType; +use arrow::error::{ArrowError, Result}; +use parquet_variant::{Variant, VariantPath}; + +use crate::VariantArrayBuilder; + +use std::sync::Arc; + +pub(crate) fn make_shredding_row_builder<'a>( + //metadata: &BinaryViewArray, Review Comment: could be removed ```suggestion ``` ########## parquet-variant-compute/src/variant_get/mod.rs: ########## @@ -500,4 +668,1013 @@ mod test { VariantArray::try_new(Arc::new(struct_array)).expect("should create variant array"), ) } + /// This test manually constructs a shredded variant array representing objects + /// like {"x": 1, "y": "foo"} and {"x": 42} and tests extracting the "x" field + /// as VariantArray using variant_get. + #[test] + fn test_shredded_object_field_access() { + let array = shredded_object_with_x_field_variant_array(); + + // Test: Extract the "x" field as VariantArray first + let options = GetOptions::new_with_path(VariantPath::from("x")); + let result = variant_get(&array, options).unwrap(); + + let result_variant: &VariantArray = result.as_any().downcast_ref().unwrap(); + assert_eq!(result_variant.len(), 2); + + // Row 0: expect x=1 + assert_eq!(result_variant.value(0), Variant::Int32(1)); + // Row 1: expect x=42 + assert_eq!(result_variant.value(1), Variant::Int32(42)); + } + + /// Test extracting shredded object field with type conversion + #[test] + fn test_shredded_object_field_as_int32() { + let array = shredded_object_with_x_field_variant_array(); + + // Test: Extract the "x" field as Int32Array (type conversion) + let field = Field::new("x", DataType::Int32, false); + let options = GetOptions::new_with_path(VariantPath::from("x")) + .with_as_type(Some(FieldRef::from(field))); + let result = variant_get(&array, options).unwrap(); + + // Should get Int32Array + let expected: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), Some(42)])); + assert_eq!(&result, &expected); + } + + /// Helper function to create a shredded variant array representing objects + /// + /// This creates an array that represents: + /// Row 0: {"x": 1, "y": "foo"} (x is shredded, y is in value field) + /// Row 1: {"x": 42} (x is shredded, perfect shredding) + /// + /// The physical layout follows the shredding spec where: + /// - metadata: contains object metadata + /// - typed_value: StructArray with field "x" (ShreddedVariantFieldArray) + /// - value: contains fallback for unshredded fields like {"y": "foo"} + /// - The "x" field has typed_value=Int32Array and value=NULL (perfect shredding) + fn shredded_object_with_x_field_variant_array() -> ArrayRef { + // Create the base metadata for objects + let (metadata, y_field_value) = { + let mut builder = parquet_variant::VariantBuilder::new(); + let mut obj = builder.new_object(); + obj.insert("x", Variant::Int32(42)); + obj.insert("y", Variant::from("foo")); + obj.finish().unwrap(); + builder.finish() + }; + + // Create metadata array (same for both rows) + let metadata_array = BinaryViewArray::from_iter_values(std::iter::repeat_n(&metadata, 2)); + + // Create the main value field per the 3-step shredding spec: + // Step 2: If field not in shredding schema, check value field + // Row 0: {"y": "foo"} (y is not shredded, stays in value for step 2) + // Row 1: {} (empty object - no unshredded fields) + let empty_object_value = { + let mut builder = parquet_variant::VariantBuilder::new(); + let obj = builder.new_object(); + obj.finish().unwrap(); + let (_, value) = builder.finish(); + value + }; + + let value_array = BinaryViewArray::from(vec![ + Some(y_field_value.as_slice()), // Row 0 has {"y": "foo"} + Some(empty_object_value.as_slice()), // Row 1 has {} + ]); + + // Create the "x" field as a ShreddedVariantFieldArray + // This represents the shredded Int32 values for the "x" field + let x_field_typed_value = Int32Array::from(vec![Some(1), Some(42)]); + + // For perfect shredding of the x field, no "value" column, only typed_value + let x_field_struct = crate::variant_array::StructArrayBuilder::new() + .with_field("typed_value", Arc::new(x_field_typed_value)) + .build(); + + // Wrap the x field struct in a ShreddedVariantFieldArray + let x_field_shredded = ShreddedVariantFieldArray::try_new(Arc::new(x_field_struct)) + .expect("should create ShreddedVariantFieldArray"); + + // Create the main typed_value as a struct containing the "x" field + let typed_value_fields = Fields::from(vec![ + Field::new("x", x_field_shredded.data_type().clone(), true) + ]); + let typed_value_struct = StructArray::try_new( + typed_value_fields, + vec![Arc::new(x_field_shredded)], + None, // No nulls - both rows have the object structure + ).unwrap(); + + // Create the main VariantArray + let main_struct = crate::variant_array::StructArrayBuilder::new() + .with_field("metadata", Arc::new(metadata_array)) + .with_field("value", Arc::new(value_array)) + .with_field("typed_value", Arc::new(typed_value_struct)) + .build(); + + Arc::new( + VariantArray::try_new(Arc::new(main_struct)).expect("should create variant array"), + ) + } + + /// Simple test to check if nested paths are supported by current implementation + #[test] + fn test_simple_nested_path_support() { Review Comment: yes! ########## parquet-variant-compute/src/variant_get/mod.rs: ########## @@ -388,43 +525,720 @@ mod test { VariantArray::try_new(Arc::new(struct_array)).expect("should create variant array"), ) } + /// This test manually constructs a shredded variant array representing objects + /// like {"x": 1, "y": "foo"} and {"x": 42} and tests extracting the "x" field Review Comment: This is a great set of ideas, but perhaps one we can do as a follow on PR -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
