This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-2933-b154603a710d730d2f43f8af4db257cc137237ef in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
commit ed42c8b9e71e54eedefc99f622308bf57a84b9fa Author: Xuanyi Li <[email protected]> AuthorDate: Tue Oct 13 22:14:21 2026 +0000 feat: recognize geometry and geography types [1/3] (#2933) feat: recognize geometry and geography types Co-authored-by: Danny C Jones <[email protected]> Co-authored-by: Shawn Chang <[email protected]> --- crates/catalog/glue/src/schema.rs | 31 ++++- crates/catalog/hms/src/schema.rs | 26 +++- crates/iceberg/public-api.txt | 108 +++++++++++++++ crates/iceberg/src/arrow/schema.rs | 25 ++++ crates/iceberg/src/arrow/value.rs | 29 ++++ crates/iceberg/src/avro/schema.rs | 24 ++++ crates/iceberg/src/spec/datatypes.rs | 107 ++++++++++++++- crates/iceberg/src/spec/geospatial.rs | 215 ++++++++++++++++++++++++++++++ crates/iceberg/src/spec/mod.rs | 2 + crates/iceberg/src/spec/schema/mod.rs | 18 ++- crates/iceberg/src/spec/sort.rs | 51 ++++++- crates/iceberg/src/spec/transform.rs | 10 +- crates/iceberg/src/spec/values/datum.rs | 12 ++ crates/iceberg/src/spec/values/literal.rs | 16 ++- crates/iceberg/src/spec/values/tests.rs | 59 ++++++++ crates/iceberg/src/transform/bucket.rs | 5 +- crates/iceberg/src/transform/identity.rs | 5 +- crates/iceberg/src/transform/truncate.rs | 7 +- 18 files changed, 735 insertions(+), 15 deletions(-) diff --git a/crates/catalog/glue/src/schema.rs b/crates/catalog/glue/src/schema.rs index 6116fc224..95c1d8bab 100644 --- a/crates/catalog/glue/src/schema.rs +++ b/crates/catalog/glue/src/schema.rs @@ -148,6 +148,10 @@ impl SchemaVisitor for GlueSchemaBuilder { } fn primitive(&mut self, p: &PrimitiveType) -> Result<Self::T> { + // Glue column types are informational for Iceberg tables; readers use Iceberg metadata as + // the source of truth. Keep the existing Glue-compatible spellings and use the Iceberg + // display representation for geospatial types. A non-failing fallback for all other types + // is tracked in https://github.com/apache/iceberg-rust/issues/3375. let glue_type = match p { PrimitiveType::Unknown => { return Err(Error::new( @@ -173,6 +177,7 @@ impl SchemaVisitor for GlueSchemaBuilder { "string".to_string() } PrimitiveType::Binary | PrimitiveType::Fixed(_) => "binary".to_string(), + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => p.to_string(), PrimitiveType::Decimal { precision, scale } => { format!("decimal({precision},{scale})") } @@ -189,7 +194,9 @@ impl SchemaVisitor for GlueSchemaBuilder { #[cfg(test)] mod tests { use iceberg::TableCreation; - use iceberg::spec::{FormatVersion, Schema, TableMetadataBuilder}; + use iceberg::spec::{ + FormatVersion, GeometryType, NestedField, PrimitiveType, Schema, TableMetadataBuilder, Type, + }; use super::*; @@ -560,4 +567,26 @@ mod tests { assert_eq!(result, expected); Ok(()) } + + #[test] + fn test_schema_with_geospatial_type() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "geom", + Type::Primitive(PrimitiveType::Geometry(GeometryType::default())), + ) + .into(), + ]) + .build() + .unwrap(); + let metadata = create_metadata_with_format_version(schema, FormatVersion::V3).unwrap(); + + let result = GlueSchemaBuilder::from_iceberg(&metadata).unwrap().build(); + + assert_eq!(result, vec![ + create_column("geom", "geometry(OGC:CRS84)", "1", false,).unwrap() + ]); + } } diff --git a/crates/catalog/hms/src/schema.rs b/crates/catalog/hms/src/schema.rs index f04ee11c1..5909454bf 100644 --- a/crates/catalog/hms/src/schema.rs +++ b/crates/catalog/hms/src/schema.rs @@ -124,6 +124,12 @@ impl SchemaVisitor for HiveSchemaBuilder { "string".to_string() } PrimitiveType::Binary | PrimitiveType::Fixed(_) => "binary".to_string(), + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Conversion from {p:?} is not supported"), + )); + } PrimitiveType::Decimal { precision, scale } => { format!("decimal({precision},{scale})") } @@ -142,7 +148,7 @@ impl SchemaVisitor for HiveSchemaBuilder { #[cfg(test)] mod tests { use iceberg::Result; - use iceberg::spec::Schema; + use iceberg::spec::{GeometryType, NestedField, PrimitiveType, Schema, Type}; use super::*; @@ -486,4 +492,22 @@ mod tests { Ok(()) } + + #[test] + fn test_schema_with_geospatial_type_is_unsupported() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "geom", + Type::Primitive(PrimitiveType::Geometry(GeometryType::default())), + ) + .into(), + ]) + .build() + .unwrap(); + + let err = HiveSchemaBuilder::from_iceberg(&schema).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + } } diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 466614757..7df9c6e3e 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1467,6 +1467,71 @@ pub fn iceberg::sensitive::SensitiveString::fmt(&self, f: &mut core::fmt::Format impl core::marker::StructuralPartialEq for iceberg::sensitive::SensitiveString pub mod iceberg::spec pub use iceberg::spec::ByteBuf +pub mod iceberg::spec::geospatial +pub enum iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Andoyer +pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Karney +pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Spherical +pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Thomas +pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Vincenty +impl core::clone::Clone for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::clone(&self) -> iceberg::spec::geospatial::EdgeInterpolationAlgorithm +impl core::cmp::Eq for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +impl core::cmp::PartialEq for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::eq(&self, other: &iceberg::spec::geospatial::EdgeInterpolationAlgorithm) -> bool +impl core::default::Default for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::default() -> iceberg::spec::geospatial::EdgeInterpolationAlgorithm +impl core::fmt::Debug for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::Copy for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +impl serde_core::ser::Serialize for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::geospatial::GeographyType +impl iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::algorithm(&self) -> iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::GeographyType::crs(&self) -> &str +pub fn iceberg::spec::geospatial::GeographyType::new(crs: core::option::Option<alloc::string::String>, algorithm: iceberg::spec::geospatial::EdgeInterpolationAlgorithm) -> iceberg::Result<Self> +impl core::clone::Clone for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::clone(&self) -> iceberg::spec::geospatial::GeographyType +impl core::cmp::Eq for iceberg::spec::geospatial::GeographyType +impl core::cmp::PartialEq for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::eq(&self, other: &iceberg::spec::geospatial::GeographyType) -> bool +impl core::default::Default for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::default() -> Self +impl core::fmt::Debug for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::GeographyType +impl serde_core::ser::Serialize for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::geospatial::GeometryType +impl iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::crs(&self) -> &str +pub fn iceberg::spec::geospatial::GeometryType::new(crs: core::option::Option<alloc::string::String>) -> iceberg::Result<Self> +impl core::clone::Clone for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::clone(&self) -> iceberg::spec::geospatial::GeometryType +impl core::cmp::Eq for iceberg::spec::geospatial::GeometryType +impl core::cmp::PartialEq for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::eq(&self, other: &iceberg::spec::geospatial::GeometryType) -> bool +impl core::default::Default for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::default() -> iceberg::spec::geospatial::GeometryType +impl core::fmt::Debug for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::GeometryType +impl serde_core::ser::Serialize for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de> pub enum iceberg::spec::DataContentType pub iceberg::spec::DataContentType::Data = 0 pub iceberg::spec::DataContentType::EqualityDeletes = 2 @@ -1712,6 +1777,8 @@ pub iceberg::spec::PrimitiveType::Decimal::scale: u32 pub iceberg::spec::PrimitiveType::Double pub iceberg::spec::PrimitiveType::Fixed(u64) pub iceberg::spec::PrimitiveType::Float +pub iceberg::spec::PrimitiveType::Geography(iceberg::spec::geospatial::GeographyType) +pub iceberg::spec::PrimitiveType::Geometry(iceberg::spec::geospatial::GeometryType) pub iceberg::spec::PrimitiveType::Int pub iceberg::spec::PrimitiveType::Long pub iceberg::spec::PrimitiveType::String @@ -2089,6 +2156,47 @@ impl serde_core::ser::Serialize for iceberg::spec::FieldSummary pub fn iceberg::spec::FieldSummary::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::FieldSummary pub fn iceberg::spec::FieldSummary::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::GeographyType +impl iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::algorithm(&self) -> iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::GeographyType::crs(&self) -> &str +pub fn iceberg::spec::geospatial::GeographyType::new(crs: core::option::Option<alloc::string::String>, algorithm: iceberg::spec::geospatial::EdgeInterpolationAlgorithm) -> iceberg::Result<Self> +impl core::clone::Clone for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::clone(&self) -> iceberg::spec::geospatial::GeographyType +impl core::cmp::Eq for iceberg::spec::geospatial::GeographyType +impl core::cmp::PartialEq for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::eq(&self, other: &iceberg::spec::geospatial::GeographyType) -> bool +impl core::default::Default for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::default() -> Self +impl core::fmt::Debug for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::GeographyType +impl serde_core::ser::Serialize for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::GeometryType +impl iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::crs(&self) -> &str +pub fn iceberg::spec::geospatial::GeometryType::new(crs: core::option::Option<alloc::string::String>) -> iceberg::Result<Self> +impl core::clone::Clone for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::clone(&self) -> iceberg::spec::geospatial::GeometryType +impl core::cmp::Eq for iceberg::spec::geospatial::GeometryType +impl core::cmp::PartialEq for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::eq(&self, other: &iceberg::spec::geospatial::GeometryType) -> bool +impl core::default::Default for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::default() -> iceberg::spec::geospatial::GeometryType +impl core::fmt::Debug for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::GeometryType +impl serde_core::ser::Serialize for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg::spec::ListType pub iceberg::spec::ListType::element_field: iceberg::spec::NestedFieldRef impl iceberg::spec::ListType diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index 0c673c6bc..5e545c0c6 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -734,6 +734,10 @@ impl SchemaVisitor for ToArrowSchemaConverter { .unwrap_or(DataType::LargeBinary), )), PrimitiveType::Binary => Ok(ArrowSchemaOrFieldOrType::Type(DataType::LargeBinary)), + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Converting {p} to an Arrow schema is not supported yet"), + )), } } @@ -1203,6 +1207,9 @@ pub(crate) fn primitive_type_to_arrow_type_with_ree(primitive_type: &PrimitiveTy PrimitiveType::Uuid => make_ree(DataType::Binary), PrimitiveType::Fixed(_) => make_ree(DataType::Binary), PrimitiveType::Binary => make_ree(DataType::Binary), + // Match the LargeBinary storage used by schema_to_arrow_schema in the read layer. + // Its 64-bit offsets cover the array's total WKB bytes, not only one geometry's size. + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => make_ree(DataType::LargeBinary), PrimitiveType::Decimal { precision, scale } => { make_ree(DataType::Decimal128(*precision as u8, *scale as i8)) } @@ -2207,6 +2214,24 @@ mod tests { ); } + #[test] + fn test_geospatial_arrow_schema_conversion_is_unsupported() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional( + 1, + "geom", + Type::Primitive(PrimitiveType::Geometry(Default::default())), + ) + .into(), + ]) + .build() + .unwrap(); + + let err = schema_to_arrow_schema(&schema).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + } + #[test] fn test_type_conversion() { // test primitive type diff --git a/crates/iceberg/src/arrow/value.rs b/crates/iceberg/src/arrow/value.rs index 7cbebf68c..b3bfb3f8e 100644 --- a/crates/iceberg/src/arrow/value.rs +++ b/crates/iceberg/src/arrow/value.rs @@ -346,6 +346,10 @@ impl SchemaWithPartnerVisitor<ArrayRef> for ArrowArrayToIcebergStructConverter { Err(invalid_data!("The partner is not a binary array")) } } + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Converting {p} Arrow array to an Iceberg literal is not supported yet"), + )), } } @@ -1106,6 +1110,31 @@ mod test { ); } + #[test] + fn test_arrow_geospatial_to_literal_is_unsupported() { + let values = Arc::new(LargeBinaryArray::from_vec(vec![b"wkb".as_slice()])) as ArrayRef; + let struct_array = Arc::new(StructArray::from(vec![( + Arc::new( + Field::new("geom", DataType::LargeBinary, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ), + values, + )])) as ArrayRef; + let ty = StructType::new(vec![ + NestedField::required( + 1, + "geom", + Type::Primitive(PrimitiveType::Geometry(Default::default())), + ) + .into(), + ]); + + let err = arrow_struct_to_literal(&struct_array, &ty).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + } + #[test] fn test_find_field_by_id() { // Create Arrow arrays for the nested structure diff --git a/crates/iceberg/src/avro/schema.rs b/crates/iceberg/src/avro/schema.rs index 47254317e..8bdd67c50 100644 --- a/crates/iceberg/src/avro/schema.rs +++ b/crates/iceberg/src/avro/schema.rs @@ -276,6 +276,12 @@ impl SchemaVisitor for SchemaToAvroSchema { PrimitiveType::Uuid => AvroSchema::Uuid(UuidSchema::String), PrimitiveType::Fixed(len) => self.define_once(avro_fixed_schema((*len) as usize)?), PrimitiveType::Binary => AvroSchema::Bytes, + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Converting {p} to an Avro schema is not supported yet"), + )); + } PrimitiveType::Decimal { precision, scale } => { self.define_once(avro_decimal_schema(*precision as usize, *scale as usize)?) } @@ -1404,4 +1410,22 @@ mod tests { let err = schema_to_avro_schema("t", &schema).unwrap_err(); assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); } + + #[test] + fn test_geospatial_to_avro_schema_is_unsupported() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional( + 1, + "geom", + Type::Primitive(PrimitiveType::Geometry(Default::default())), + ) + .into(), + ]) + .build() + .unwrap(); + + let err = schema_to_avro_schema("t", &schema).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + } } diff --git a/crates/iceberg/src/spec/datatypes.rs b/crates/iceberg/src/spec/datatypes.rs index bfee52023..a5ea6ee87 100644 --- a/crates/iceberg/src/spec/datatypes.rs +++ b/crates/iceberg/src/spec/datatypes.rs @@ -28,6 +28,7 @@ use serde::de::{Error, IntoDeserializer}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value as JsonValue; +use super::geospatial::{GeographyType, GeometryType, parse_geography, parse_geometry}; use super::values::Literal; use crate::ensure_data_valid; use crate::error::Result; @@ -133,7 +134,7 @@ impl Type { /// Minimum [`FormatVersion`] required to support this type, **without** taking /// nested field types into account. /// - /// `Unknown` / `TimestampNs` / `TimestamptzNs` / `Variant` require + /// `Unknown` / `TimestampNs` / `TimestamptzNs` / `Geometry` / `Geography` / `Variant` require /// [`FormatVersion::V3`]; every other type is valid from [`FormatVersion::V1`]. Mirrors Java's /// `Schema.MIN_FORMAT_VERSIONS` (a shallow lookup keyed by type id), so it /// intentionally does not recurse: callers needing the floor for a whole schema @@ -143,7 +144,11 @@ impl Type { pub(crate) fn min_format_version(&self) -> FormatVersion { match self { Type::Primitive( - PrimitiveType::Unknown | PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs, + PrimitiveType::Unknown + | PrimitiveType::TimestampNs + | PrimitiveType::TimestamptzNs + | PrimitiveType::Geometry(_) + | PrimitiveType::Geography(_), ) | Type::Variant(_) => FormatVersion::V3, _ => FormatVersion::V1, @@ -274,6 +279,10 @@ pub enum PrimitiveType { Binary, /// Default / null column type used when a more specific type is not known. Unknown, + /// Geometry values encoded as well-known binary. + Geometry(GeometryType), + /// Geography values encoded as well-known binary. + Geography(GeographyType), } impl PrimitiveType { @@ -297,6 +306,8 @@ impl PrimitiveType { | (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(_)) | (PrimitiveType::Fixed(_), PrimitiveLiteral::Binary(_)) | (PrimitiveType::Binary, PrimitiveLiteral::Binary(_)) + | (PrimitiveType::Geometry(_), PrimitiveLiteral::Binary(_)) + | (PrimitiveType::Geography(_), PrimitiveLiteral::Binary(_)) ) } } @@ -325,6 +336,16 @@ impl<'de> Deserialize<'de> for PrimitiveType { deserialize_decimal(s.into_deserializer()) } else if s.starts_with("fixed") { deserialize_fixed(s.into_deserializer()) + } else if s + .get(..8) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("geometry")) + { + parse_geometry(&s).map_err(D::Error::custom) + } else if s + .get(..9) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("geography")) + { + parse_geography(&s).map_err(D::Error::custom) } else { PrimitiveType::deserialize(s.into_deserializer()) } @@ -339,6 +360,9 @@ impl Serialize for PrimitiveType { serialize_decimal(precision, scale, serializer) } PrimitiveType::Fixed(l) => serialize_fixed(l, serializer), + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => { + serializer.serialize_str(&self.to_string()) + } _ => PrimitiveType::serialize(self, serializer), } } @@ -435,6 +459,13 @@ impl fmt::Display for PrimitiveType { PrimitiveType::Uuid => write!(f, "uuid"), PrimitiveType::Fixed(size) => write!(f, "fixed({size})"), PrimitiveType::Binary => write!(f, "binary"), + PrimitiveType::Geometry(geometry) => write!(f, "geometry({})", geometry.crs()), + PrimitiveType::Geography(geography) => write!( + f, + "geography({}, {})", + geography.crs(), + geography.algorithm().as_str() + ), } } } @@ -944,10 +975,13 @@ impl<'de> Deserialize<'de> for VariantType { #[cfg(test)] mod tests { + use std::hash::{Hash, Hasher}; + use pretty_assertions::assert_eq; use uuid::Uuid; use super::*; + use crate::spec::geospatial::{DEFAULT_GEOSPATIAL_CRS, EdgeInterpolationAlgorithm}; use crate::spec::values::PrimitiveLiteral; fn check_type_serde(json: &str, expected_type: Type) { @@ -1076,6 +1110,67 @@ mod tests { ) } + #[test] + fn primitive_type_geospatial() { + let cases = vec![ + ( + r#""geometry""#, + PrimitiveType::Geometry(GeometryType::default()), + "geometry(OGC:CRS84)", + ), + ( + r#""GeOmEtRy ( EPSG:3857 )""#, + PrimitiveType::Geometry(GeometryType::new(Some("EPSG:3857".to_string())).unwrap()), + "geometry(EPSG:3857)", + ), + ( + r#""GEOGRAPHY""#, + PrimitiveType::Geography(GeographyType::default()), + "geography(OGC:CRS84, spherical)", + ), + ( + r#""GeOgRaPhY ( OGC:CRS27 , KARNEY )""#, + PrimitiveType::Geography( + GeographyType::new( + Some("OGC:CRS27".to_string()), + EdgeInterpolationAlgorithm::Karney, + ) + .unwrap(), + ), + "geography(OGC:CRS27, karney)", + ), + ]; + + for (json, expected, display) in cases { + let actual: PrimitiveType = serde_json::from_str(json).unwrap(); + assert_eq!( + actual, expected, + "parsed primitive type did not match expectation" + ); + assert_eq!(actual.to_string(), display, "display impl did not match"); + assert_eq!( + serde_json::to_string(&actual).unwrap(), + format!(r#""{display}""#), + "JSON serialization did not match expectation" + ); + } + + let equivalent_default = GeometryType::new(Some("EPSG:4326".to_string())).unwrap(); + assert_eq!(equivalent_default.crs(), "EPSG:4326"); + assert_ne!(equivalent_default, GeometryType::default()); + let lowercase_default = + GeometryType::new(Some(DEFAULT_GEOSPATIAL_CRS.to_ascii_lowercase())).unwrap(); + assert_eq!(lowercase_default, GeometryType::default()); + let mut lowercase_hash = std::collections::hash_map::DefaultHasher::new(); + lowercase_default.hash(&mut lowercase_hash); + let mut default_hash = std::collections::hash_map::DefaultHasher::new(); + GeometryType::default().hash(&mut default_hash); + assert_eq!(lowercase_hash.finish(), default_hash.finish()); + assert!( + serde_json::from_str::<PrimitiveType>(r#""geography(OGC:CRS27,unknown)""#).is_err() + ); + } + #[test] fn struct_type() { let record = r#" @@ -1551,6 +1646,14 @@ mod tests { ), (PrimitiveType::Fixed(8), PrimitiveLiteral::Binary(vec![1])), (PrimitiveType::Binary, PrimitiveLiteral::Binary(vec![1])), + ( + PrimitiveType::Geometry(GeometryType::default()), + PrimitiveLiteral::Binary(vec![1]), + ), + ( + PrimitiveType::Geography(GeographyType::default()), + PrimitiveLiteral::Binary(vec![1]), + ), ]; for (ty, literal) in pairs { assert!(ty.compatible(&literal)); diff --git a/crates/iceberg/src/spec/geospatial.rs b/crates/iceberg/src/spec/geospatial.rs new file mode 100644 index 000000000..df2ea5a2e --- /dev/null +++ b/crates/iceberg/src/spec/geospatial.rs @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Geospatial types defined by the Iceberg specification. + +use std::hash::{Hash, Hasher}; + +use serde::{Deserialize, Serialize}; + +use super::PrimitiveType; +use crate::error::{Result, invalid_data}; + +pub(crate) const DEFAULT_GEOSPATIAL_CRS: &str = "OGC:CRS84"; + +#[derive(Debug, Serialize, Deserialize, Clone)] +struct Crs(String); + +impl Crs { + fn new(crs: Option<String>) -> Result<Self> { + let crs = crs.unwrap_or_else(|| DEFAULT_GEOSPATIAL_CRS.to_string()); + let crs = crs.trim().to_string(); + if crs.is_empty() || crs.contains([',', ')']) { + return Err(crate::Error::new( + crate::ErrorKind::DataInvalid, + "Geospatial CRS must be non-empty and must not contain ',' or ')'", + )); + } + + Ok(Self(crs)) + } +} + +impl Default for Crs { + fn default() -> Self { + Self(DEFAULT_GEOSPATIAL_CRS.to_string()) + } +} + +impl PartialEq for Crs { + fn eq(&self, other: &Self) -> bool { + self.0.eq_ignore_ascii_case(&other.0) + } +} + +impl Eq for Crs {} + +impl Hash for Crs { + fn hash<H: Hasher>(&self, state: &mut H) { + for byte in self.0.bytes() { + state.write_u8(byte.to_ascii_lowercase()); + } + } +} + +/// Iceberg geometry type. +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash, Default)] +pub struct GeometryType { + crs: Crs, +} + +impl GeometryType { + /// Creates a geometry type with an optional coordinate reference system. + pub fn new(crs: Option<String>) -> Result<Self> { + Ok(Self { + crs: Crs::new(crs)?, + }) + } + + /// Returns the coordinate reference system. + pub fn crs(&self) -> &str { + &self.crs.0 + } +} + +/// Iceberg geography edge interpolation algorithm. +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy, Hash, Default)] +#[serde(rename_all = "lowercase")] +pub enum EdgeInterpolationAlgorithm { + /// Spherical edge interpolation. + #[default] + Spherical, + /// Vincenty edge interpolation. + Vincenty, + /// Thomas edge interpolation. + Thomas, + /// Andoyer edge interpolation. + Andoyer, + /// Karney edge interpolation. + Karney, +} + +impl EdgeInterpolationAlgorithm { + pub(super) fn as_str(self) -> &'static str { + match self { + Self::Spherical => "spherical", + Self::Vincenty => "vincenty", + Self::Thomas => "thomas", + Self::Andoyer => "andoyer", + Self::Karney => "karney", + } + } + + fn parse(value: &str) -> Result<Self> { + match value.trim().to_ascii_lowercase().as_str() { + "spherical" => Ok(Self::Spherical), + "vincenty" => Ok(Self::Vincenty), + "thomas" => Ok(Self::Thomas), + "andoyer" => Ok(Self::Andoyer), + "karney" => Ok(Self::Karney), + _ => Err(invalid_data!( + "Unknown geography edge interpolation algorithm: {value}" + )), + } + } +} + +/// Iceberg geography type. +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)] +pub struct GeographyType { + crs: Crs, + algorithm: EdgeInterpolationAlgorithm, +} + +impl Default for GeographyType { + fn default() -> Self { + Self { + crs: Crs::default(), + algorithm: EdgeInterpolationAlgorithm::Spherical, + } + } +} + +impl GeographyType { + /// Creates a geography type with an optional coordinate reference system and edge interpolation algorithm. + pub fn new(crs: Option<String>, algorithm: EdgeInterpolationAlgorithm) -> Result<Self> { + Ok(Self { + crs: Crs::new(crs)?, + algorithm, + }) + } + + /// Returns the coordinate reference system. + pub fn crs(&self) -> &str { + &self.crs.0 + } + + /// Returns the edge interpolation algorithm. + pub fn algorithm(&self) -> EdgeInterpolationAlgorithm { + self.algorithm + } +} + +fn parse_geospatial_params<'a>(value: &'a str, type_name: &str) -> Result<Vec<&'a str>> { + if value.eq_ignore_ascii_case(type_name) { + return Ok(vec![]); + } + + let params = value + .get(..type_name.len()) + .filter(|prefix| prefix.eq_ignore_ascii_case(type_name)) + .map(|_| &value[type_name.len()..]) + .map(str::trim_start) + .and_then(|s| s.strip_prefix('(')) + .and_then(|s| s.strip_suffix(')')) + .ok_or_else(|| invalid_data!("Invalid {type_name} type: {value}"))?; + + if params.trim().is_empty() { + return Err(invalid_data!("{type_name} requires a non-empty CRS")); + } + + Ok(params.split(',').map(str::trim).collect()) +} + +pub(super) fn parse_geometry(value: &str) -> Result<PrimitiveType> { + let params = parse_geospatial_params(value.trim(), "geometry")?; + let geometry = match params.as_slice() { + [] => GeometryType::default(), + [crs] if !crs.is_empty() => GeometryType::new(Some((*crs).to_string()))?, + _ => return Err(invalid_data!("Invalid geometry type: {value}")), + }; + + Ok(PrimitiveType::Geometry(geometry)) +} + +pub(super) fn parse_geography(value: &str) -> Result<PrimitiveType> { + let params = parse_geospatial_params(value.trim(), "geography")?; + let geography = match params.as_slice() { + [] => GeographyType::default(), + [crs] if !crs.is_empty() => GeographyType::new( + Some((*crs).to_string()), + EdgeInterpolationAlgorithm::Spherical, + )?, + [crs, algorithm] if !crs.is_empty() && !algorithm.is_empty() => GeographyType::new( + Some((*crs).to_string()), + EdgeInterpolationAlgorithm::parse(algorithm)?, + )?, + _ => return Err(invalid_data!("Invalid geography type: {value}")), + }; + + Ok(PrimitiveType::Geography(geography)) +} diff --git a/crates/iceberg/src/spec/mod.rs b/crates/iceberg/src/spec/mod.rs index 707ebbb63..7bbb37c79 100644 --- a/crates/iceberg/src/spec/mod.rs +++ b/crates/iceberg/src/spec/mod.rs @@ -19,6 +19,7 @@ mod datatypes; mod encrypted_key; +pub mod geospatial; mod manifest; mod manifest_list; mod name_mapping; @@ -39,6 +40,7 @@ mod view_version; pub use datatypes::*; pub use encrypted_key::*; +pub use geospatial::{GeographyType, GeometryType}; pub use manifest::*; pub use manifest_list::*; pub use name_mapping::*; diff --git a/crates/iceberg/src/spec/schema/mod.rs b/crates/iceberg/src/spec/schema/mod.rs index d55d0889d..4d539b36c 100644 --- a/crates/iceberg/src/spec/schema/mod.rs +++ b/crates/iceberg/src/spec/schema/mod.rs @@ -662,7 +662,7 @@ mod tests { #[test] fn test_calc_min_compatible_format() { - use crate::spec::{FormatVersion, VariantType}; + use crate::spec::{FormatVersion, GeometryType, VariantType}; fn schema_with(fields: Vec<NestedFieldRef>) -> Schema { Schema::builder().with_fields(fields).build().unwrap() @@ -692,6 +692,22 @@ mod tests { .is_err() ); + // Geospatial types are v3-only. + let geometry = schema_with(vec![ + NestedField::optional( + 1, + "geom", + Primitive(PrimitiveType::Geometry(GeometryType::default())), + ) + .into(), + ]); + assert_eq!(geometry.calc_min_compatible_format(), FormatVersion::V3); + assert!( + geometry + .check_format_compatibility(FormatVersion::V2) + .is_err() + ); + // A v3-only type nested inside a list inside a struct → V3 (flattened fields). let nested = schema_with(vec![ NestedField::required( diff --git a/crates/iceberg/src/spec/sort.rs b/crates/iceberg/src/spec/sort.rs index e40aaa65d..c1c9fdbad 100644 --- a/crates/iceberg/src/spec/sort.rs +++ b/crates/iceberg/src/spec/sort.rs @@ -27,7 +27,7 @@ use typed_builder::TypedBuilder; use super::transform::Transform; use crate::error::Result; -use crate::spec::Schema; +use crate::spec::{PrimitiveType, Schema, Type}; use crate::{Error, ErrorKind}; /// Reference to [`SortOrder`]. @@ -194,6 +194,19 @@ impl SortOrderBuilder { )); } + if matches!( + source_type, + Type::Primitive(PrimitiveType::Geometry(_) | PrimitiveType::Geography(_)) + ) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Cannot sort by geospatial source field: {}", + source_field.name + ), + )); + } + let field_transform = sort_field.transform; field_transform.result_type(source_type)?; } @@ -207,7 +220,7 @@ impl SortOrderBuilder { #[cfg(test)] mod tests { use super::*; - use crate::spec::{ListType, NestedField, PrimitiveType, Type}; + use crate::spec::{GeometryType, ListType, NestedField, PrimitiveType, Type}; #[test] fn test_sort_field() { @@ -474,6 +487,40 @@ mod tests { ); } + #[test] + fn test_build_should_reject_geospatial_source_fields() { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required( + 1, + "geom", + Type::Primitive(PrimitiveType::Geometry(GeometryType::default())), + ) + .into(), + ]) + .build() + .unwrap(); + + let err = SortOrder::builder() + .with_sort_field( + SortField::builder() + .source_id(1) + .direction(SortDirection::Ascending) + .null_order(NullOrder::First) + .transform(Transform::Void) + .build(), + ) + .build(&schema) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + "Cannot sort by geospatial source field: geom" + ); + } + #[test] fn test_build_should_return_err_if_source_field_type_is_not_supported_by_transform() { let schema = Schema::builder() diff --git a/crates/iceberg/src/spec/transform.rs b/crates/iceberg/src/spec/transform.rs index c28dc28c1..0d2b61e73 100644 --- a/crates/iceberg/src/spec/transform.rs +++ b/crates/iceberg/src/spec/transform.rs @@ -278,7 +278,15 @@ impl Transform { pub fn result_type(&self, input_type: &Type) -> Result<Type> { match self { Transform::Identity => { - if matches!(input_type, Type::Primitive(_)) { + if matches!( + input_type, + Type::Primitive(PrimitiveType::Geometry(_) | PrimitiveType::Geography(_)) + ) { + Err(Error::new( + ErrorKind::DataInvalid, + format!("{input_type} is not a valid input type of identity transform",), + )) + } else if matches!(input_type, Type::Primitive(_)) { Ok(input_type.clone()) } else { Err(invalid_data!( diff --git a/crates/iceberg/src/spec/values/datum.rs b/crates/iceberg/src/spec/values/datum.rs index c4c835f61..db1f6500c 100644 --- a/crates/iceberg/src/spec/values/datum.rs +++ b/crates/iceberg/src/spec/values/datum.rs @@ -269,6 +269,12 @@ impl PartialOrd for Datum { PrimitiveType::Binary, PrimitiveType::Binary, ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::Binary(_), + PrimitiveLiteral::Binary(_), + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_), + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_), + ) => None, ( PrimitiveLiteral::Int128(val), PrimitiveLiteral::Int128(other_val), @@ -374,6 +380,12 @@ impl Datum { "Cannot create datum for unknown type from bytes", )); } + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Cannot create geospatial datum from bytes yet", + )); + } PrimitiveType::Boolean => { if bytes.len() == 1 && bytes[0] == 0u8 { PrimitiveLiteral::Boolean(false) diff --git a/crates/iceberg/src/spec/values/literal.rs b/crates/iceberg/src/spec/values/literal.rs index 1d27ae995..52e4a5c73 100644 --- a/crates/iceberg/src/spec/values/literal.rs +++ b/crates/iceberg/src/spec/values/literal.rs @@ -32,7 +32,7 @@ use super::decimal_utils::{ use super::primitive::PrimitiveLiteral; use super::struct_value::Struct; use super::temporal::{date, time, timestamp, timestamptz}; -use crate::error::{Result, invalid_data}; +use crate::error::{Error, ErrorKind, Result, invalid_data}; use crate::spec::datatypes::{PrimitiveType, Type}; /// Values present in iceberg type @@ -508,6 +508,13 @@ impl Literal { (PrimitiveType::Binary, JsonValue::String(s)) => Ok(Some(Literal::Primitive( PrimitiveLiteral::Binary(decode_hex_bytes(&s)?), ))), + ( + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_), + JsonValue::String(_), + ) => Err(Error::new( + ErrorKind::FeatureUnsupported, + "WKT deserialization for geospatial types is not yet supported", + )), ( PrimitiveType::Decimal { precision: _, @@ -676,6 +683,13 @@ impl Literal { (PrimitiveType::Binary, PrimitiveLiteral::Binary(val)) => { Ok(JsonValue::String(encode_hex_bytes(&val))) } + ( + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_), + PrimitiveLiteral::Binary(_), + ) => Err(Error::new( + ErrorKind::FeatureUnsupported, + "WKT serialization for geospatial types is not yet supported", + )), (_, PrimitiveLiteral::Int128(val)) => match r#type { Type::Primitive(PrimitiveType::Decimal { precision: _precision, diff --git a/crates/iceberg/src/spec/values/tests.rs b/crates/iceberg/src/spec/values/tests.rs index 63ff9f636..b61646e69 100644 --- a/crates/iceberg/src/spec/values/tests.rs +++ b/crates/iceberg/src/spec/values/tests.rs @@ -548,6 +548,65 @@ fn avro_bytes_string() { check_avro_bytes_serde(bytes, Datum::string("iceberg"), &PrimitiveType::String); } +#[test] +fn geospatial_binary_deserialization_is_not_yet_supported() { + for primitive_type in [ + PrimitiveType::Geometry(Default::default()), + PrimitiveType::Geography(Default::default()), + ] { + let err = Datum::try_from_bytes(&[1, 2, 3], primitive_type).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + } +} + +#[test] +fn geospatial_wkt_json_is_not_yet_supported() { + for primitive_type in [ + PrimitiveType::Geometry(Default::default()), + PrimitiveType::Geography(Default::default()), + ] { + let err = Literal::try_from_json( + serde_json::json!("POINT (1 2)"), + &Primitive(primitive_type.clone()), + ) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + assert_eq!( + err.message(), + "WKT deserialization for geospatial types is not yet supported" + ); + + let err = Literal::Primitive(PrimitiveLiteral::Binary(vec![1, 2, 3])) + .try_into_json(&Primitive(primitive_type)) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + assert_eq!( + err.message(), + "WKT serialization for geospatial types is not yet supported" + ); + } +} + +#[test] +fn geospatial_datums_are_not_orderable() { + let geometry_type = PrimitiveType::Geometry(Default::default()); + let geography_type = PrimitiveType::Geography(Default::default()); + let geometry = Datum::new( + geometry_type.clone(), + PrimitiveLiteral::Binary(vec![1, 2, 3]), + ); + let other_geometry = Datum::new(geometry_type, PrimitiveLiteral::Binary(vec![4, 5, 6])); + let geography = Datum::new( + geography_type.clone(), + PrimitiveLiteral::Binary(vec![1, 2, 3]), + ); + let other_geography = Datum::new(geography_type, PrimitiveLiteral::Binary(vec![4, 5, 6])); + + assert_eq!(geometry.partial_cmp(&other_geometry), None); + assert_eq!(geography.partial_cmp(&other_geography), None); +} + #[test] fn avro_bytes_decimal() { // (input_bytes, decimal_num, expect_scale, expect_precision) diff --git a/crates/iceberg/src/transform/bucket.rs b/crates/iceberg/src/transform/bucket.rs index 128fb1728..33911eb07 100644 --- a/crates/iceberg/src/transform/bucket.rs +++ b/crates/iceberg/src/transform/bucket.rs @@ -297,12 +297,12 @@ mod test { use crate::Result; use crate::expr::PredicateOperator; use crate::spec::PrimitiveType::{ - Binary, Date, Decimal, Fixed, Int, Long, String as StringType, Time, Timestamp, + Binary, Date, Decimal, Fixed, Geometry, Int, Long, String as StringType, Time, Timestamp, TimestampNs, Timestamptz, TimestamptzNs, Uuid, }; use crate::spec::Type::{Primitive, Struct}; use crate::spec::decimal_utils::decimal_new; - use crate::spec::{Datum, NestedField, PrimitiveType, StructType, Transform}; + use crate::spec::{Datum, GeometryType, NestedField, PrimitiveType, StructType, Transform}; use crate::transform::TransformFunction; use crate::transform::test::{TestProjectionFixture, TestTransformFixture}; @@ -341,6 +341,7 @@ mod test { (Primitive(Timestamptz), Some(Primitive(Int))), (Primitive(TimestampNs), Some(Primitive(Int))), (Primitive(TimestamptzNs), Some(Primitive(Int))), + (Primitive(Geometry(GeometryType::default())), None), ( Struct(StructType::new(vec![ NestedField::optional(1, "a", Primitive(Timestamp)).into(), diff --git a/crates/iceberg/src/transform/identity.rs b/crates/iceberg/src/transform/identity.rs index dd096337a..d2d9cc476 100644 --- a/crates/iceberg/src/transform/identity.rs +++ b/crates/iceberg/src/transform/identity.rs @@ -37,11 +37,11 @@ impl TransformFunction for Identity { #[cfg(test)] mod test { use crate::spec::PrimitiveType::{ - Binary, Date, Decimal, Fixed, Int, Long, String as StringType, Time, Timestamp, + Binary, Date, Decimal, Fixed, Geometry, Int, Long, String as StringType, Time, Timestamp, TimestampNs, Timestamptz, TimestamptzNs, Uuid, }; use crate::spec::Type::{Primitive, Struct}; - use crate::spec::{NestedField, StructType, Transform}; + use crate::spec::{GeometryType, NestedField, StructType, Transform}; use crate::transform::test::TestTransformFixture; #[test] @@ -83,6 +83,7 @@ mod test { (Primitive(Timestamptz), Some(Primitive(Timestamptz))), (Primitive(TimestampNs), Some(Primitive(TimestampNs))), (Primitive(TimestamptzNs), Some(Primitive(TimestamptzNs))), + (Primitive(Geometry(GeometryType::default())), None), ( Struct(StructType::new(vec