This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 3f8a653baf [common] Keep blank variant object keys unshredded during
schema inference (#9813)
3f8a653baf is described below
commit 3f8a653baf51f97f76dfc30d90aecdcf8e5db77e
Author: Xiangyi Zhu <[email protected]>
AuthorDate: Thu Sep 17 17:06:10 2026 +0800
[common] Keep blank variant object keys unshredded during schema inference
(#9813)
---
.../data/variant/InferVariantShreddingSchema.java | 8 +++++-
.../variant/InferVariantShreddingSchemaTest.java | 28 +++++++++++++++++++
.../writer/InferVariantShreddingWriteTest.java | 31 ++++++++++++++++++++++
.../apache/paimon/spark/sql/VariantTestBase.scala | 14 ++++++++++
4 files changed, 80 insertions(+), 1 deletion(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java
b/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java
index 318266eb67..7c80064c24 100644
---
a/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java
+++
b/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java
@@ -26,6 +26,7 @@ import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.DecimalType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.VariantType;
+import org.apache.paimon.utils.StringUtils;
import java.math.BigDecimal;
import java.util.ArrayList;
@@ -301,6 +302,11 @@ public class InferVariantShreddingSchema {
for (int i = 0; i < size; i++) {
GenericVariant.ObjectField field = v.getFieldAtIndex(i);
+ if (StringUtils.isNullOrWhitespaceOnly(field.key)) {
+ // A blank key is a valid variant object key but not a
RowType field
+ // name, so leave it in the unshredded value instead
of failing the write
+ continue;
+ }
DataType fieldType = schemaOf(field.value, maxDepth - 1);
// Store count in description temporarily (will be used in
mergeRowTypes)
DataField dataField = new DataField(i, field.key,
fieldType, "1");
@@ -308,7 +314,7 @@ public class InferVariantShreddingSchema {
}
// According to the variant spec, object fields must be sorted
alphabetically
- for (int i = 1; i < size; i++) {
+ for (int i = 1; i < fields.size(); i++) {
if (fields.get(i -
1).name().compareTo(fields.get(i).name()) >= 0) {
throw new RuntimeException(
"Variant object fields must be sorted
alphabetically");
diff --git
a/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java
b/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java
index c1f8628009..2b7835d7b1 100644
---
a/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java
@@ -31,6 +31,7 @@ import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -211,6 +212,33 @@ public class InferVariantShreddingSchemaTest {
.isEqualTo(variantShreddingSchema(expectedType));
}
+ @Test
+ void testInferSchemaSkipsBlankKeys() {
+ RowType schema = RowType.of(new DataType[] {DataTypes.VARIANT()}, new
String[] {"v"});
+
+ // An empty or blank key is a valid variant object key but not a
RowType field name;
+ // it must stay in the unshredded value rather than fail the inference
+ GenericVariant variant1 = GenericVariant.fromJson("{\"\": 1, \" \": 2,
\"a\": 3}");
+ GenericVariant variant2 = GenericVariant.fromJson("{\"\": 4, \"a\":
5}");
+
+ List<InternalRow> rows = Arrays.asList(GenericRow.of(variant1),
GenericRow.of(variant2));
+
+ InferVariantShreddingSchema inferrer =
defaultInferVariantShreddingSchema(schema);
+ RowType inferredSchema = inferrer.inferSchema(rows);
+
+ RowType expectedType = RowType.of(new DataType[] {DataTypes.BIGINT()},
new String[] {"a"});
+ assertThat(inferredSchema.getField("v").type())
+ .isEqualTo(variantShreddingSchema(expectedType));
+
+ // an object made only of blank keys has nothing to shred and stays
unshredded
+ RowType blankOnly =
+ inferrer.inferSchema(
+ Collections.singletonList(
+ GenericRow.of(GenericVariant.fromJson("{\"\":
1}"))));
+ assertThat(blankOnly.getField("v").type())
+ .isEqualTo(variantShreddingSchema(DataTypes.VARIANT()));
+ }
+
@Test
void testInferSchemaWithNullValues() {
// Schema: row<v: variant>
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java
index f59ffa462e..f1187f2b54 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java
@@ -465,6 +465,37 @@ public class InferVariantShreddingWriteTest {
assertThat(result2.get(1)).isEqualTo(GenericRow.of(GenericRow.of(20.5,
7L)));
}
+ @Test
+ public void testInferSchemaSkipsBlankKeys() throws Exception {
+ ParquetFileFormat format = createFormat();
+ RowType writeType = DataTypes.ROW(DataTypes.FIELD(0, "v",
DataTypes.VARIANT()));
+
+ // an empty key is a valid variant object key but not a RowType field
name; it used
+ // to fail the whole file write and must stay in the unshredded value
instead
+ FormatWriterFactory factory = format.createWriterFactory(writeType);
+ writeRows(
+ factory,
+ GenericRow.of(GenericVariant.fromJson("{\"\":1,\"a\":2}")),
+ GenericRow.of(GenericVariant.fromJson("{\"\":3,\"
\":4,\"a\":5}")));
+
+ RowType expectShreddedType =
+ RowType.of(new DataType[] {DataTypes.BIGINT()}, new String[]
{"a"});
+ verifyShreddingSchema(expectShreddedType);
+
+ List<InternalRow> result = readRows(format, writeType);
+
assertThat(result.get(0).getVariant(0).toJson()).isEqualTo("{\"\":1,\"a\":2}");
+ assertThat(result.get(1).getVariant(0).toJson()).isEqualTo("{\"\":3,\"
\":4,\"a\":5}");
+
+ RowType variantRowType =
+ VariantMetadataUtils.VariantRowTypeBuilder.builder()
+ .field(DataTypes.BIGINT(), "$.a")
+ .build();
+ RowType readType = DataTypes.ROW(DataTypes.FIELD(0, "v",
variantRowType));
+ List<InternalRow> result2 = readRows(format, readType);
+ assertThat(result2.get(0)).isEqualTo(GenericRow.of(GenericRow.of(2L)));
+ assertThat(result2.get(1)).isEqualTo(GenericRow.of(GenericRow.of(5L)));
+ }
+
@Test
public void testInferSchemaWithNullValues() throws Exception {
ParquetFileFormat format = createFormat();
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
index 70b58e0fe9..0b50b12870 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
@@ -445,6 +445,20 @@ abstract class VariantTestBase extends PaimonSparkTestBase
{
Seq(Row(1, 10.0, 100L), Row(2, 20.5, 7L)))
}
+ test("Paimon Variant: blank object keys under inferred shredding") {
+ sql("CREATE TABLE T (id INT, v VARIANT)")
+ // an empty key cannot become a shredded column and used to fail
inferred-shredding writes
+ sql("""INSERT INTO T VALUES
+ | (1, parse_json('{"":1,"a":2}')),
+ | (2, parse_json('{"":3," ":4,"a":5}'))
+ |""".stripMargin)
+
+ checkAnswer(
+ sql("SELECT id, to_json(v), variant_get(v, '$.a', 'bigint') FROM T ORDER
BY id"),
+ Seq(Row(1, """{"":1,"a":2}""", 2L), Row(2, """{"":3," ":4,"a":5}""", 5L))
+ )
+ }
+
test("Paimon Variant: read and write variant with null value") {
withTable("source_tbl", "target_tbl") {
sql("CREATE TABLE source_tbl (id INT, js STRING) USING paimon")
