This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit b540c00762a1c9e466d9ca7540a3f4ad04899103 Author: Xiangyi Zhu <[email protected]> AuthorDate: Tue Sep 15 11:17:27 2026 +0800 [common] Fix inferred variant shredding failing on decimals with trailing zeros (#9811) --- .../data/variant/InferVariantShreddingSchema.java | 17 ++++++----- .../variant/InferVariantShreddingSchemaTest.java | 32 +++++++++++++++++++++ .../writer/InferVariantShreddingWriteTest.java | 33 ++++++++++++++++++++++ .../apache/paimon/spark/sql/VariantTestBase.scala | 14 +++++++++ 4 files changed, 87 insertions(+), 9 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java b/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java index e983f28fdd..318266eb67 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java @@ -354,16 +354,15 @@ public class InferVariantShreddingSchema { case DECIMAL: BigDecimal dec = v.getDecimal(); - int decPrecision = dec.precision(); - int decScale = dec.scale(); - // Ensure precision is at least scale + 1 to be valid - if (decPrecision < decScale) { - decPrecision = decScale; - } - // Ensure precision is at least 1 - if (decPrecision == 0) { - decPrecision = 1; + if (dec.scale() < 0) { + // getDecimal() strips trailing zeros, which turns 10.0 into 1E+1; a negative + // scale is not a valid Paimon decimal, so fold the exponent back into digits + dec = dec.setScale(0); } + int decScale = dec.scale(); + // precision() counts the digits of the unscaled value, so it is below the scale + // for a value under 0.1, which DecimalType rejects + int decPrecision = Math.max(dec.precision(), decScale); return DataTypes.DECIMAL(decPrecision, decScale); case DATE: diff --git a/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java b/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java index 1a38025e7b..c1f8628009 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java @@ -179,6 +179,38 @@ public class InferVariantShreddingSchemaTest { .isEqualTo(variantShreddingSchema(expectedType)); } + @Test + void testInferSchemaWithDecimalTrailingZeros() { + RowType schema = RowType.of(new DataType[] {DataTypes.VARIANT()}, new String[] {"v"}); + + // getDecimal() strips trailing zeros, so 10.0 and 100.00 arrive with a negative scale + // and 0.05 with a precision below its scale; none of them may break inference + GenericVariant variant1 = + GenericVariant.fromJson( + "{\"price\": 10.0, \"whole\": 100.00, \"tiny\": 0.05," + + " \"big\": 100000000000000000000}"); + GenericVariant variant2 = + GenericVariant.fromJson( + "{\"price\": 20.5, \"whole\": 7, \"tiny\": 0.001, \"big\": 1}"); + + List<InternalRow> rows = Arrays.asList(GenericRow.of(variant1), GenericRow.of(variant2)); + + InferVariantShreddingSchema inferrer = defaultInferVariantShreddingSchema(schema); + RowType inferredSchema = inferrer.inferSchema(rows); + + RowType expectedType = + RowType.of( + new DataType[] { + DataTypes.DECIMAL(38, 0), + DataTypes.DECIMAL(18, 1), + DataTypes.DECIMAL(18, 3), + DataTypes.BIGINT() + }, + new String[] {"big", "price", "tiny", "whole"}); + assertThat(inferredSchema.getField("v").type()) + .isEqualTo(variantShreddingSchema(expectedType)); + } + @Test void testInferSchemaWithNullValues() { // Schema: row<v: variant> diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java index 8e12c08ddc..f59ffa462e 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java @@ -432,6 +432,39 @@ public class InferVariantShreddingWriteTest { verifyShreddingSchema(expectShreddedType); } + @Test + public void testInferSchemaWithDecimalTrailingZeros() throws Exception { + ParquetFileFormat format = createFormat(); + RowType writeType = DataTypes.ROW(DataTypes.FIELD(0, "v", DataTypes.VARIANT())); + + // 10.0 and 100.00 strip to a negative scale, which used to fail the whole file write + FormatWriterFactory factory = format.createWriterFactory(writeType); + writeRows( + factory, + GenericRow.of(GenericVariant.fromJson("{\"price\":10.0,\"whole\":100.00}")), + GenericRow.of(GenericVariant.fromJson("{\"price\":20.5,\"whole\":7}"))); + + RowType expectShreddedType = + RowType.of( + new DataType[] {DataTypes.DECIMAL(18, 1), DataTypes.BIGINT()}, + new String[] {"price", "whole"}); + verifyShreddingSchema(expectShreddedType); + + List<InternalRow> result = readRows(format, writeType); + assertThat(result.get(0).getVariant(0).toJson()).isEqualTo("{\"price\":10,\"whole\":100}"); + assertThat(result.get(1).getVariant(0).toJson()).isEqualTo("{\"price\":20.5,\"whole\":7}"); + + RowType variantRowType = + VariantMetadataUtils.VariantRowTypeBuilder.builder() + .field(DataTypes.DOUBLE(), "$.price") + .field(DataTypes.BIGINT(), "$.whole") + .build(); + RowType readType = DataTypes.ROW(DataTypes.FIELD(0, "v", variantRowType)); + List<InternalRow> result2 = readRows(format, readType); + assertThat(result2.get(0)).isEqualTo(GenericRow.of(GenericRow.of(10.0, 100L))); + assertThat(result2.get(1)).isEqualTo(GenericRow.of(GenericRow.of(20.5, 7L))); + } + @Test public void testInferSchemaWithNullValues() throws Exception { ParquetFileFormat format = createFormat(); diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala index 0799310b74..0878e2c00a 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala @@ -431,6 +431,20 @@ abstract class VariantTestBase extends PaimonSparkTestBase { } } + test("Paimon Variant: decimals with trailing zeros under inferred shredding") { + sql("CREATE TABLE T (id INT, v VARIANT)") + // 10.0 and 100.00 strip to a negative scale, which used to fail inferred-shredding writes + sql("""INSERT INTO T VALUES + | (1, parse_json('{"price":10.0,"whole":100.00}')), + | (2, parse_json('{"price":20.5,"whole":7}')) + |""".stripMargin) + + checkAnswer( + sql( + "SELECT id, variant_get(v, '$.price', 'double'), variant_get(v, '$.whole', 'bigint') FROM T ORDER BY id"), + Seq(Row(1, 10.0, 100L), Row(2, 20.5, 7L))) + } + test("Paimon Variant: read and write variant with null value") { withTable("source_tbl", "target_tbl") { sql("CREATE TABLE source_tbl (id INT, js STRING) USING paimon")
