This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new d0d2dec0ae [common] Reject out-of-range integral casts when extracting
a variant scalar (#9809)
d0d2dec0ae is described below
commit d0d2dec0ae4c568fd1b1f92d5b77d8376d286196
Author: Xiangyi Zhu <[email protected]>
AuthorDate: Thu Sep 17 17:08:43 2026 +0800
[common] Reject out-of-range integral casts when extracting a variant
scalar (#9809)
---
.../paimon/data/variant/BaseVariantReader.java | 7 +-
.../org/apache/paimon/data/variant/VariantGet.java | 88 ++++++++++++++++++++--
.../paimon/data/variant/GenericVariantTest.java | 37 +++++++++
.../parquet/reader/VariantShreddingReadTest.java | 50 ++++++++++++
.../apache/paimon/spark/sql/VariantTestBase.scala | 14 ++++
5 files changed, 183 insertions(+), 13 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java
b/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java
index 0d0177c607..e5c5011d20 100644
---
a/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java
+++
b/paimon-common/src/main/java/org/apache/paimon/data/variant/BaseVariantReader.java
@@ -543,11 +543,8 @@ public class BaseVariantReader {
if (noNeedCast) {
return i;
}
- try {
- return resolve.cast(i);
- } catch (Exception e) {
- return invalidCast(row, topLevelMetadata);
- }
+ Object result = VariantGet.castScalar(i, scalaType, targetType,
resolve);
+ return result == null ? invalidCast(row, topLevelMetadata) :
result;
}
}
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java
b/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java
index 7e355939eb..5a22fc1752 100644
--- a/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java
+++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java
@@ -34,7 +34,10 @@ import org.apache.paimon.types.MapType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.VariantType;
+import javax.annotation.Nullable;
+
import java.math.BigDecimal;
+import java.math.RoundingMode;
import java.util.HashMap;
/** Utils for variant get. */
@@ -172,15 +175,84 @@ public class VariantGet {
CastExecutor<Object, Object> resolve =
(CastExecutor<Object, Object>)
CastExecutors.resolve(inputType, dataType);
- if (resolve != null) {
- try {
- return resolve.cast(input);
- } catch (Exception e) {
- return invalidCast(v, dataType, castArgs);
- }
- }
+ Object result = castScalar(input, inputType, dataType, resolve);
+ return result == null ? invalidCast(v, dataType, castArgs) :
result;
+ }
+ }
- return invalidCast(v, dataType, castArgs);
+ /**
+ * Casts a non-null scalar read from a variant to {@code targetType},
returning null when the
+ * cast is invalid. The generic cast rules wrap a numeric value that does
not fit the target, so
+ * an out-of-range value is rejected here first, matching Spark's TRY cast
semantics.
+ */
+ @Nullable
+ static Object castScalar(
+ Object input,
+ DataType inputType,
+ DataType targetType,
+ @Nullable CastExecutor<Object, Object> executor) {
+ if (executor == null || !fitsIntegralTarget(input, inputType,
targetType)) {
+ return null;
+ }
+ try {
+ return executor.cast(input);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ /** Whether a numeric {@code input} lies within the range of an integral
{@code targetType}. */
+ private static boolean fitsIntegralTarget(
+ Object input, DataType inputType, DataType targetType) {
+ long min;
+ long max;
+ switch (targetType.getTypeRoot()) {
+ case TINYINT:
+ min = Byte.MIN_VALUE;
+ max = Byte.MAX_VALUE;
+ break;
+ case SMALLINT:
+ min = Short.MIN_VALUE;
+ max = Short.MAX_VALUE;
+ break;
+ case INTEGER:
+ min = Integer.MIN_VALUE;
+ max = Integer.MAX_VALUE;
+ break;
+ case BIGINT:
+ min = Long.MIN_VALUE;
+ max = Long.MAX_VALUE;
+ break;
+ default:
+ return true;
+ }
+
+ switch (inputType.getTypeRoot()) {
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ case BIGINT:
+ long value = ((Number) input).longValue();
+ return value >= min && value <= max;
+ case FLOAT:
+ case DOUBLE:
+ // The fractional part is truncated by the cast, so any finite
value strictly
+ // between min - 1 and max + 1 fits. Both bounds are exact
doubles; for BIGINT
+ // max + 1 is 2^63 and min - 1 rounds to -2^63, which is
itself in range.
+ double d = ((Number) input).doubleValue();
+ if (Double.isNaN(d) || Double.isInfinite(d)) {
+ return false;
+ }
+ return max == Long.MAX_VALUE
+ ? d >= -0x1p63 && d < 0x1p63
+ : d > min - 1.0 && d < max + 1.0;
+ case DECIMAL:
+ BigDecimal truncated =
+ ((Decimal) input).toBigDecimal().setScale(0,
RoundingMode.DOWN);
+ return truncated.compareTo(BigDecimal.valueOf(min)) >= 0
+ && truncated.compareTo(BigDecimal.valueOf(max)) <= 0;
+ default:
+ return true;
}
}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java
b/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java
index e5e507c455..46b5b8c992 100644
---
a/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java
@@ -41,6 +41,7 @@ import static
org.apache.paimon.data.variant.PaimonShreddingUtils.buildVariantSc
import static org.apache.paimon.data.variant.PaimonShreddingUtils.castShredded;
import static
org.apache.paimon.data.variant.PaimonShreddingUtils.variantShreddingSchema;
import static org.apache.paimon.types.DataTypesTest.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Test of {@link GenericVariant}. */
public class GenericVariantTest {
@@ -308,6 +309,42 @@ public class GenericVariantTest {
.isEqualTo(Decimal.fromBigDecimal(new BigDecimal("100.0"), 5,
1));
}
+ @Test
+ public void testVariantGetIntegralOverflow() {
+ // The generic cast rules wrap or saturate a value that does not fit
the target, which
+ // would turn an invalid cast into a wrong number; Spark's TRY cast
rejects it instead.
+ Variant variant =
+ GenericVariant.fromJson(
+ "{\"big\": 99999999999, \"huge\": 1e30, \"negHuge\":
-1e30,"
+ + " \"wide\": 12345678901234567890.5, \"nan\":
1.5,"
+ + " \"fits\": 2147483647, \"neg\":
-2147483648.5}");
+ VariantCastArgs tryCast = new VariantCastArgs(false, ZoneOffset.UTC);
+ VariantCastArgs strict = new VariantCastArgs(true, ZoneOffset.UTC);
+
+ assertThat(variant.variantGet("$.big", DataTypes.INT(),
tryCast)).isNull();
+ assertThat(variant.variantGet("$.big", DataTypes.SMALLINT(),
tryCast)).isNull();
+ assertThat(variant.variantGet("$.big", DataTypes.TINYINT(),
tryCast)).isNull();
+ assertThat(variant.variantGet("$.huge", DataTypes.BIGINT(),
tryCast)).isNull();
+ assertThat(variant.variantGet("$.huge", DataTypes.INT(),
tryCast)).isNull();
+ assertThat(variant.variantGet("$.negHuge", DataTypes.INT(),
tryCast)).isNull();
+ assertThat(variant.variantGet("$.wide", DataTypes.BIGINT(),
tryCast)).isNull();
+ assertThatThrownBy(() -> variant.variantGet("$.big", DataTypes.INT(),
strict))
+ .hasMessageContaining("Invalid cast 99999999999 to INT");
+ assertThatThrownBy(() -> variant.variantGet("$.huge",
DataTypes.BIGINT(), strict))
+ .hasMessageContaining("Invalid cast 1.0E30 to BIGINT");
+
+ // in-range values still truncate the fractional part and widen as
before
+ assertThat(variant.variantGet("$.big", DataTypes.BIGINT(), tryCast))
+ .isEqualTo(99999999999L);
+ assertThat(variant.variantGet("$.nan", DataTypes.INT(),
tryCast)).isEqualTo(1);
+ assertThat(variant.variantGet("$.fits", DataTypes.INT(), tryCast))
+ .isEqualTo(Integer.MAX_VALUE);
+ assertThat(variant.variantGet("$.neg", DataTypes.INT(), tryCast))
+ .isEqualTo(Integer.MIN_VALUE);
+ assertThat(variant.variantGet("$.wide", DataTypes.DOUBLE(), tryCast))
+ .isEqualTo(1.2345678901234567E19);
+ }
+
@Test
public void testObjectFieldOrderingCompatibility() {
String bmpKey = "\uE000";
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/parquet/reader/VariantShreddingReadTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/reader/VariantShreddingReadTest.java
index 34c4bdd382..5b93ff3c73 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/parquet/reader/VariantShreddingReadTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/parquet/reader/VariantShreddingReadTest.java
@@ -54,6 +54,7 @@ import java.util.List;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Tests for variant shredding read. */
public class VariantShreddingReadTest {
@@ -544,6 +545,55 @@ public class VariantShreddingReadTest {
0.0));
}
+ @ParameterizedTest
+ @ValueSource(
+ strings = {
+ "null",
+
"{\"type\":\"ROW\",\"fields\":[{\"name\":\"v\",\"type\":{\"type\":\"ROW\",\"fields\":[{\"name\":\"n\",\"type\":\"BIGINT\"},{\"name\":\"d\",\"type\":\"DOUBLE\"}]}}]}"
+ })
+ public void testReadIntegralOverflowAsInvalidCast(String shreddingSchema)
throws Exception {
+ // A shredded typed_value is narrowed by the scalar reader and an
unshredded value by
+ // VariantGet; both must reject an out-of-range number instead of
wrapping it.
+ Options options = new Options();
+ if (!shreddingSchema.equals("null")) {
+ options.set("parquet.variant.shreddingSchema", shreddingSchema);
+ }
+ ParquetFileFormat format =
+ new ParquetFileFormat(new
FileFormatFactory.FormatContext(options, 1024, 1024));
+
+ RowType writeType = DataTypes.ROW(DataTypes.FIELD(0, "v",
DataTypes.VARIANT()));
+ writeRows(
+ format.createWriterFactory(writeType),
+
GenericRow.of(GenericVariant.fromJson("{\"n\":99999999999,\"d\":1e30}")),
+ GenericRow.of(GenericVariant.fromJson("{\"n\":7,\"d\":1.5}")));
+
+ RowType tryReadType =
+ DataTypes.ROW(
+ DataTypes.FIELD(
+ 0,
+ "v",
+
VariantMetadataUtils.VariantRowTypeBuilder.builder()
+ .field(DataTypes.INT(), "$.n", false,
"UTC")
+ .field(DataTypes.BIGINT(), "$.d",
false, "UTC")
+ .build()));
+ List<InternalRow> result = readRows(format, tryReadType);
+ assertThat(result.get(0).getRow(0, 2).isNullAt(0)).isTrue();
+ assertThat(result.get(0).getRow(0, 2).isNullAt(1)).isTrue();
+ assertThat(result.get(1).getRow(0, 2).getInt(0)).isEqualTo(7);
+ assertThat(result.get(1).getRow(0, 2).getLong(1)).isEqualTo(1L);
+
+ RowType strictReadType =
+ DataTypes.ROW(
+ DataTypes.FIELD(
+ 0,
+ "v",
+
VariantMetadataUtils.VariantRowTypeBuilder.builder()
+ .field(DataTypes.INT(), "$.n", true,
"UTC")
+ .build()));
+ assertThatThrownBy(() -> readRows(format, strictReadType))
+ .hasMessageContaining("Invalid cast 99999999999 to INT");
+ }
+
protected List<InternalRow> readRows(ParquetFileFormat format, RowType
rowType)
throws IOException {
List<InternalRow> result = new ArrayList<>();
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
index 0b50b12870..4501713510 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
@@ -1099,6 +1099,20 @@ abstract class VariantTestBase extends
PaimonSparkTestBase {
)
}
+ test("Paimon Variant pushdown: out-of-range extraction is NULL rather than a
wrapped value") {
+ sql("CREATE TABLE T (id INT, v VARIANT)")
+ sql("""INSERT INTO T VALUES
+ | (1, parse_json('{"n":99999999999,"d":1e30}')),
+ | (2, parse_json('{"n":7,"d":1.5}'))
+ |""".stripMargin)
+
+ // With pushdown the cast runs in the Paimon reader, which must agree with
Spark's TRY cast.
+ checkAnswer(
+ sql(
+ "SELECT id, try_variant_get(v, '$.n', 'int'), try_variant_get(v,
'$.d', 'bigint') FROM T ORDER BY id"),
+ Seq(Row(1, null, null), Row(2, 7, 1L)))
+ }
+
test("Paimon Variant pushdown: nested variant column inside a struct") {
assume(gteqSpark4_1)
sql("CREATE TABLE T (id INT, nested STRUCT<v: VARIANT, x: INT>)")
