This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new b567b6e6 fix(python): render field types as SQL strings, not Rust
Debug (#786)
b567b6e6 is described below
commit b567b6e6d916078e49f17b55b50c73a78013a307
Author: jackylee <[email protected]>
AuthorDate: Fri Sep 18 14:47:23 2026 +0800
fix(python): render field types as SQL strings, not Rust Debug (#786)
---
bindings/python/src/schema.rs | 6 +-
crates/paimon/src/spec/schema.rs | 13 +-
crates/paimon/src/spec/types.rs | 347 ++++++++++++++++++++++++++++++++++++---
3 files changed, 339 insertions(+), 27 deletions(-)
diff --git a/bindings/python/src/schema.rs b/bindings/python/src/schema.rs
index 194cb3ad..3783e3a1 100644
--- a/bindings/python/src/schema.rs
+++ b/bindings/python/src/schema.rs
@@ -77,9 +77,9 @@ impl PyDataField {
}
fn field_type(&self) -> String {
- // TODO(#284 follow-up): mirror Java DataType.asSQLString() once
- // a Display impl is added to paimon::spec::DataType.
- format!("{:?}", self.inner.data_type())
+ // Java `DataField.type().asSQLString()`, which is what
+ // `Display for DataType` renders.
+ self.inner.data_type().to_string()
}
fn is_nullable(&self) -> bool {
diff --git a/crates/paimon/src/spec/schema.rs b/crates/paimon/src/spec/schema.rs
index 1e020eff..662889ad 100644
--- a/crates/paimon/src/spec/schema.rs
+++ b/crates/paimon/src/spec/schema.rs
@@ -1119,8 +1119,10 @@ impl DataField {
}
}
+/// Quote an identifier the way Java `EncodingUtils.escapeIdentifier` does:
wrap it
+/// in backticks and double any backtick it contains.
pub fn escape_identifier(identifier: &str) -> String {
- identifier.replace('"', "\"\"")
+ format!("`{}`", identifier.replace('`', "``"))
}
pub fn escape_single_quotes(text: &str) -> String {
@@ -2386,8 +2388,13 @@ mod tests {
#[test]
fn test_escape_identifier() {
- let escaped_identifier = escape_identifier("\"identifier\"");
- assert_eq!(escaped_identifier, "\"\"identifier\"\"");
+ // Java doubles backticks and wraps in backticks; a double quote is not
+ // special (`EncodingUtils.escapeIdentifier`).
+ assert_eq!(escape_identifier("id"), "`id`");
+ assert_eq!(escape_identifier("a`b"), "`a``b`");
+ assert_eq!(escape_identifier("\"identifier\""), "`\"identifier\"`");
+ assert_eq!(escape_identifier("two words"), "`two words`");
+ assert_eq!(escape_identifier(""), "``");
}
#[test]
diff --git a/crates/paimon/src/spec/types.rs b/crates/paimon/src/spec/types.rs
index f14fc1cc..551dd08e 100644
--- a/crates/paimon/src/spec/types.rs
+++ b/crates/paimon/src/spec/types.rs
@@ -370,6 +370,18 @@ pub struct ArrayType {
element_type: Box<DataType>,
}
+impl Display for ArrayType {
+ /// Java `ArrayType.FORMAT` is `ARRAY<%s>`, filled with the element's own
SQL
+ /// string so the element's `NOT NULL` lands inside the angle brackets.
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "ARRAY<{}>", self.element_type)?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
impl ArrayType {
pub fn new(element_type: DataType) -> Self {
Self::with_nullable(true, element_type)
@@ -469,33 +481,13 @@ impl VectorType {
pub fn length(&self) -> u32 {
self.length
}
-
- /// SQL name of a valid vector element type.
- fn element_sql_name(&self) -> &'static str {
- match self.element_type.as_ref() {
- DataType::Boolean(_) => "BOOLEAN",
- DataType::TinyInt(_) => "TINYINT",
- DataType::SmallInt(_) => "SMALLINT",
- DataType::Int(_) => "INT",
- DataType::BigInt(_) => "BIGINT",
- DataType::Float(_) => "FLOAT",
- DataType::Double(_) => "DOUBLE",
- other => {
- unreachable!("vector element type validated at construction:
{other:?}")
- }
- }
- }
}
impl Display for VectorType {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
// Match Java VectorType.asSQLString: the element is rendered with its
own
// SQL string, which includes the element's nullability suffix.
- write!(f, "VECTOR<{}", self.element_sql_name())?;
- if !self.element_type.is_nullable() {
- write!(f, " NOT NULL")?;
- }
- write!(f, ", {}>", self.length)?;
+ write!(f, "VECTOR<{}, {}>", self.element_type, self.length)?;
if !self.nullable {
write!(f, " NOT NULL")?;
}
@@ -600,6 +592,16 @@ pub struct BigIntType {
nullable: bool,
}
+impl Display for BigIntType {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "BIGINT")?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
impl Default for BigIntType {
fn default() -> Self {
Self::new()
@@ -720,6 +722,16 @@ pub struct BooleanType {
nullable: bool,
}
+impl Display for BooleanType {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "BOOLEAN")?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
impl Default for BooleanType {
fn default() -> Self {
Self::new()
@@ -753,6 +765,16 @@ pub struct BlobType {
nullable: bool,
}
+impl Display for BlobType {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "BLOB")?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
impl Default for BlobType {
fn default() -> Self {
Self::new()
@@ -787,6 +809,16 @@ pub struct VariantType {
nullable: bool,
}
+impl Display for VariantType {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "VARIANT")?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
impl Default for VariantType {
fn default() -> Self {
Self::new()
@@ -908,6 +940,16 @@ pub struct DateType {
nullable: bool,
}
+impl Display for DateType {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "DATE")?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
impl Default for DateType {
fn default() -> Self {
Self::new()
@@ -1129,6 +1171,16 @@ pub struct FloatType {
nullable: bool,
}
+impl Display for FloatType {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "FLOAT")?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
impl Default for FloatType {
fn default() -> Self {
Self::new()
@@ -1162,6 +1214,16 @@ pub struct IntType {
nullable: bool,
}
+impl Display for IntType {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "INT")?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
impl Default for IntType {
fn default() -> Self {
Self::new()
@@ -1304,6 +1366,16 @@ pub struct SmallIntType {
nullable: bool,
}
+impl Display for SmallIntType {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "SMALLINT")?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
impl Default for SmallIntType {
fn default() -> Self {
Self::new()
@@ -1538,6 +1610,16 @@ pub struct TinyIntType {
nullable: bool,
}
+impl Display for TinyIntType {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "TINYINT")?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
impl Default for TinyIntType {
fn default() -> Self {
Self::new()
@@ -1788,6 +1870,17 @@ pub struct MapType {
value_type: Box<DataType>,
}
+impl Display for MapType {
+ /// Java `MapType.FORMAT` is `MAP<%s, %s>`.
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "MAP<{}, {}>", self.key_type, self.value_type)?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
impl MapType {
pub fn new(key_type: DataType, value_type: DataType) -> Self {
Self::with_nullable(true, key_type, value_type)
@@ -1830,6 +1923,17 @@ pub struct MultisetType {
element_type: Box<DataType>,
}
+impl Display for MultisetType {
+ /// Java `MultisetType.FORMAT` is `MULTISET<%s>`.
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "MULTISET<{}>", self.element_type)?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
impl MultisetType {
pub fn new(element_type: DataType) -> Self {
Self::with_nullable(true, element_type)
@@ -1868,6 +1972,79 @@ pub struct RowType {
fields: Vec<DataField>,
}
+/// Render one row field the way Java `DataField.asSQLString` does: the escaped
+/// name, a space, the field type's SQL string, then `COMMENT '...'` when the
+/// field carries a description. Java also appends `DEFAULT <value>`, which
has no
+/// counterpart on this struct.
+fn write_row_field(f: &mut Formatter<'_>, field: &DataField) ->
std::fmt::Result {
+ write!(
+ f,
+ "{} {}",
+ crate::spec::escape_identifier(field.name()),
+ field.data_type()
+ )?;
+ if let Some(description) = field.description().filter(|text|
!text.is_empty()) {
+ write!(
+ f,
+ " COMMENT '{}'",
+ crate::spec::escape_single_quotes(description)
+ )?;
+ }
+ Ok(())
+}
+
+impl Display for RowType {
+ /// Java `RowType.FORMAT` is `ROW<%s>`, filled with the fields' own SQL
strings
+ /// joined by `", "`.
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "ROW<")?;
+ for (index, field) in self.fields.iter().enumerate() {
+ if index > 0 {
+ write!(f, ", ")?;
+ }
+ write_row_field(f, field)?;
+ }
+ write!(f, ">")?;
+ if !self.nullable {
+ write!(f, " NOT NULL")?;
+ }
+ Ok(())
+ }
+}
+
+impl Display for DataType {
+ /// Dispatch to the variant's own SQL string, mirroring Java's abstract
+ /// `DataType.asSQLString`. Every arm is spelled out so a new variant
fails to
+ /// compile rather than falling into a catch-all that renders the wrong
text.
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ match self {
+ DataType::Boolean(t) => Display::fmt(t, f),
+ DataType::TinyInt(t) => Display::fmt(t, f),
+ DataType::SmallInt(t) => Display::fmt(t, f),
+ DataType::Int(t) => Display::fmt(t, f),
+ DataType::BigInt(t) => Display::fmt(t, f),
+ DataType::Decimal(t) => Display::fmt(t, f),
+ DataType::Double(t) => Display::fmt(t, f),
+ DataType::Float(t) => Display::fmt(t, f),
+ DataType::Binary(t) => Display::fmt(t, f),
+ DataType::VarBinary(t) => Display::fmt(t, f),
+ DataType::Variant(t) => Display::fmt(t, f),
+ DataType::Blob(t) => Display::fmt(t, f),
+ DataType::Char(t) => Display::fmt(t, f),
+ DataType::VarChar(t) => Display::fmt(t, f),
+ DataType::Date(t) => Display::fmt(t, f),
+ DataType::LocalZonedTimestamp(t) => Display::fmt(t, f),
+ DataType::Time(t) => Display::fmt(t, f),
+ DataType::Timestamp(t) => Display::fmt(t, f),
+ DataType::Array(t) => Display::fmt(t, f),
+ DataType::Map(t) => Display::fmt(t, f),
+ DataType::Multiset(t) => Display::fmt(t, f),
+ DataType::Row(t) => Display::fmt(t, f),
+ DataType::Vector(t) => Display::fmt(t, f),
+ }
+ }
+}
+
impl RowType {
pub const fn new(fields: Vec<DataField>) -> Self {
Self::with_nullable(true, fields)
@@ -2783,6 +2960,134 @@ mod tests {
));
}
+ /// Every variant must render its Java `asSQLString`. Spelled out one by
one so
+ /// a wrong keyword cannot hide behind another arm.
+ #[test]
+ fn test_data_type_display_matches_java_sql_string() {
+ let cases: Vec<(DataType, &str)> = vec![
+ (DataType::Boolean(BooleanType::new()), "BOOLEAN"),
+ (DataType::TinyInt(TinyIntType::new()), "TINYINT"),
+ (DataType::SmallInt(SmallIntType::new()), "SMALLINT"),
+ (DataType::Int(IntType::new()), "INT"),
+ (DataType::BigInt(BigIntType::new()), "BIGINT"),
+ (DataType::Float(FloatType::new()), "FLOAT"),
+ (DataType::Double(DoubleType::new()), "DOUBLE"),
+ (DataType::Date(DateType::new()), "DATE"),
+ (DataType::Variant(VariantType::new()), "VARIANT"),
+ (DataType::Blob(BlobType::new()), "BLOB"),
+ (DataType::Binary(BinaryType::new(1).unwrap()), "BINARY(1)"),
+ (
+ DataType::Decimal(DecimalType::new(10, 2).unwrap()),
+ "DECIMAL(10, 2)",
+ ),
+ (
+ DataType::VarChar(VarCharType::new(10).unwrap()),
+ "VARCHAR(10)",
+ ),
+ ];
+ for (data_type, expected) in cases {
+ assert_eq!(data_type.to_string(), expected);
+ assert_eq!(
+ data_type.copy_with_nullable(false).unwrap().to_string(),
+ format!("{expected} NOT NULL"),
+ "NOT NULL suffix for {expected}"
+ );
+ }
+ }
+
+ /// Java renders a constructed type by filling its `FORMAT` with the
children's
+ /// own SQL strings, so a child's `NOT NULL` sits inside the brackets
while the
+ /// outer type's sits after them.
+ #[test]
+ fn test_constructed_data_type_display_nests_children() {
+ let int = DataType::Int(IntType::new());
+ let not_null_int = int.copy_with_nullable(false).unwrap();
+
+ assert_eq!(
+ DataType::Array(ArrayType::new(int.clone())).to_string(),
+ "ARRAY<INT>"
+ );
+ assert_eq!(
+ DataType::Array(ArrayType::with_nullable(false,
not_null_int.clone())).to_string(),
+ "ARRAY<INT NOT NULL> NOT NULL"
+ );
+ assert_eq!(
+ DataType::Multiset(MultisetType::new(int.clone())).to_string(),
+ "MULTISET<INT>"
+ );
+ assert_eq!(
+ DataType::Map(MapType::new(
+ DataType::VarChar(VarCharType::new(8).unwrap()),
+ int.clone()
+ ))
+ .to_string(),
+ "MAP<VARCHAR(8), INT>"
+ );
+ assert_eq!(
+ DataType::Array(ArrayType::new(DataType::Map(MapType::new(
+ int.clone(),
+ DataType::Array(ArrayType::new(int.clone()))
+ ))))
+ .to_string(),
+ "ARRAY<MAP<INT, ARRAY<INT>>>"
+ );
+ }
+
+ /// Java `RowType.FORMAT` is `ROW<%s>` over `DataField.asSQLString`, which
is the
+ /// escaped name, the type, then `COMMENT '...'` when a description is
present.
+ #[test]
+ fn test_row_type_display_renders_fields_like_java() {
+ let row = RowType::new(vec![
+ DataField::new(0, "id".to_string(), DataType::Int(IntType::new())),
+ DataField::new(
+ 1,
+ "name".to_string(),
+ DataType::VarChar(VarCharType::new(10).unwrap()),
+ ),
+ ]);
+ assert_eq!(
+ DataType::Row(row).to_string(),
+ "ROW<`id` INT, `name` VARCHAR(10)>"
+ );
+
+ let described = RowType::with_nullable(
+ false,
+ vec![
+ DataField::new(0, "id".to_string(),
DataType::Int(IntType::new()))
+ .with_description(Some("the 'key'".to_string())),
+ ],
+ );
+ assert_eq!(
+ DataType::Row(described).to_string(),
+ "ROW<`id` INT COMMENT 'the ''key'''> NOT NULL"
+ );
+
+ // A double quote is not special to backtick quoting; a backtick is
doubled;
+ // a name with a space or a reserved word stays unambiguous.
+ let quoted = RowType::new(vec![DataField::new(
+ 0,
+ "we\"ird".to_string(),
+ DataType::Int(IntType::new()),
+ )]);
+ assert_eq!(DataType::Row(quoted).to_string(), "ROW<`we\"ird` INT>");
+
+ let backticked = RowType::new(vec![DataField::new(
+ 0,
+ "a`b".to_string(),
+ DataType::Int(IntType::new()),
+ )]);
+ assert_eq!(DataType::Row(backticked).to_string(), "ROW<`a``b` INT>");
+
+ let spaced = RowType::new(vec![
+ DataField::new(0, "two words".to_string(),
DataType::Int(IntType::new())),
+ DataField::new(1, "select".to_string(),
DataType::Int(IntType::new())),
+ ]);
+ assert_eq!(
+ DataType::Row(spaced).to_string(),
+ "ROW<`two words` INT, `select` INT>"
+ );
+ }
+
#[test]
fn test_vector_type_display() {
let v = VectorType::try_new(true, 128,
DataType::Float(FloatType::new())).unwrap();
