This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 88d2bd40 fix(reader): map Avro local-timestamp logical types instead
of panicking (#813)
88d2bd40 is described below
commit 88d2bd4076dab0d05c2c67ce79f5933b3639b944
Author: Y Ethan Guo <[email protected]>
AuthorDate: Wed Oct 28 14:00:35 2026 -0700
fix(reader): map Avro local-timestamp logical types instead of panicking
(#813)
avro_to_arrow::schema had todo!() for local-timestamp-millis, -micros
and -nanos. Converting a table schema with such a column panicked, and
when the conversion runs behind an FFI boundary the panic aborts the
host process, even when the query did not select the column. Map them
to tz-less Arrow timestamps, the type arrow-avro decodes them to and
a parquet base file written with isAdjustedToUTC=false reads back as.
---
crates/core/src/avro_to_arrow/schema.rs | 37 ++++++++++++++++++++++++++++++---
1 file changed, 34 insertions(+), 3 deletions(-)
diff --git a/crates/core/src/avro_to_arrow/schema.rs
b/crates/core/src/avro_to_arrow/schema.rs
index 3a63a9f9..4556208e 100644
--- a/crates/core/src/avro_to_arrow/schema.rs
+++ b/crates/core/src/avro_to_arrow/schema.rs
@@ -169,9 +169,11 @@ fn schema_to_field_with_props(
AvroSchema::TimestampNanos => {
DataType::Timestamp(TimeUnit::Nanosecond,
Some(UTC_TIMEZONE.into()))
}
- AvroSchema::LocalTimestampMillis => todo!(),
- AvroSchema::LocalTimestampMicros => todo!(),
- AvroSchema::LocalTimestampNanos => todo!(),
+ // Timezone-naive: no zone, matching arrow-avro's decode of these
types and
+ // a parquet base file written with isAdjustedToUTC=false.
+ AvroSchema::LocalTimestampMillis =>
DataType::Timestamp(TimeUnit::Millisecond, None),
+ AvroSchema::LocalTimestampMicros =>
DataType::Timestamp(TimeUnit::Microsecond, None),
+ AvroSchema::LocalTimestampNanos =>
DataType::Timestamp(TimeUnit::Nanosecond, None),
AvroSchema::Duration => DataType::Duration(TimeUnit::Millisecond),
};
@@ -316,6 +318,35 @@ mod tests {
use super::*;
use apache_avro::Schema as AvroSchema;
+ /// The timezone-naive `local-timestamp-*` types map to a tz-less Arrow
timestamp,
+ /// while the UTC `timestamp-*` types keep their zone.
+ #[test]
+ fn test_local_timestamps_convert_to_tz_less_timestamps() {
+ let avro = AvroSchema::parse_str(
+ r#"{"type":"record","name":"r","fields":[
+
{"name":"l_ms","type":{"type":"long","logicalType":"local-timestamp-millis"}},
+
{"name":"l_us","type":{"type":"long","logicalType":"local-timestamp-micros"}},
+
{"name":"l_ns","type":{"type":"long","logicalType":"local-timestamp-nanos"}},
+
{"name":"u_us","type":{"type":"long","logicalType":"timestamp-micros"}},
+
{"name":"opt","type":["null",{"type":"long","logicalType":"local-timestamp-micros"}]}]}"#,
+ )
+ .unwrap();
+
+ let schema = to_arrow_schema(&avro).unwrap();
+ let types: Vec<&DataType> = schema.fields().iter().map(|f|
f.data_type()).collect();
+ assert_eq!(
+ types,
+ vec![
+ &DataType::Timestamp(TimeUnit::Millisecond, None),
+ &DataType::Timestamp(TimeUnit::Microsecond, None),
+ &DataType::Timestamp(TimeUnit::Nanosecond, None),
+ &DataType::Timestamp(TimeUnit::Microsecond,
Some(UTC_TIMEZONE.into())),
+ &DataType::Timestamp(TimeUnit::Microsecond, None),
+ ]
+ );
+ assert!(schema.field_with_name("opt").unwrap().is_nullable());
+ }
+
/// An Avro map becomes an Arrow `Map`, not a dictionary. A dictionary key
/// must be an integer, so the `Dictionary(Utf8, V)` this used to produce
was
/// not a valid Arrow type and could not reconcile against the `Map` a
