This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 5a63d7d0d3 [format] Keep DECIMAL precision when reading JSON numbers
(#10376)
5a63d7d0d3 is described below
commit 5a63d7d0d3e646e0f6cb63153b45dbb83bc693f6
Author: Eunbin Son <[email protected]>
AuthorDate: Fri Oct 16 14:03:06 2026 +0900
[format] Keep DECIMAL precision when reading JSON numbers (#10376)
---
.../apache/paimon/format/json/JsonFileReader.java | 37 ++++++++-
.../paimon/format/json/JsonFileFormatTest.java | 95 ++++++++++++++++++++++
2 files changed, 130 insertions(+), 2 deletions(-)
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/json/JsonFileReader.java
b/paimon-format/src/main/java/org/apache/paimon/format/json/JsonFileReader.java
index e776170816..6cb07fe4cc 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/json/JsonFileReader.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/json/JsonFileReader.java
@@ -29,6 +29,9 @@ import org.apache.paimon.fs.Path;
import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypeChecks;
+import org.apache.paimon.types.DataTypeFamily;
+import org.apache.paimon.types.DataTypeRoot;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.RowType;
@@ -36,7 +39,9 @@ import org.apache.paimon.types.VectorType;
import org.apache.paimon.utils.JsonSerdeUtil;
import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
+import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.DeserializationFeature;
import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
+import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectReader;
import javax.annotation.Nullable;
@@ -53,6 +58,7 @@ public class JsonFileReader extends AbstractTextFileReader {
private static final Base64.Decoder BASE64_DECODER = Base64.getDecoder();
private final JsonOptions options;
+ private final ObjectReader jsonReader;
public JsonFileReader(
FileIO fileIO,
@@ -64,12 +70,34 @@ public class JsonFileReader extends AbstractTextFileReader {
throws IOException {
super(fileIO, filePath, rowType, options.getLineDelimiter(), offset,
length);
this.options = options;
+ // Parse floating-point numbers as BigDecimal only when needed, so
DECIMAL values keep
+ // their precision instead of being rounded through double.
+ this.jsonReader =
+ containsDecimal(rowType)
+ ? JsonSerdeUtil.OBJECT_MAPPER_INSTANCE
+ .reader()
+
.with(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS)
+ : JsonSerdeUtil.OBJECT_MAPPER_INSTANCE.reader();
+ }
+
+ private static boolean containsDecimal(DataType type) {
+ if (type.is(DataTypeRoot.DECIMAL)) {
+ return true;
+ }
+ if (type.is(DataTypeFamily.CONSTRUCTED)) {
+ for (DataType nested : DataTypeChecks.getNestedTypes(type)) {
+ if (containsDecimal(nested)) {
+ return true;
+ }
+ }
+ }
+ return false;
}
@Override
protected InternalRow parseLine(String line) throws IOException {
try {
- JsonNode jsonNode =
JsonSerdeUtil.OBJECT_MAPPER_INSTANCE.readTree(line);
+ JsonNode jsonNode = jsonReader.readTree(line);
return (InternalRow) convertJsonValue(jsonNode, rowType, options);
} catch (JsonProcessingException e) {
if (options.ignoreParseErrors()) {
@@ -108,7 +136,12 @@ public class JsonFileReader extends AbstractTextFileReader
{
case ROW:
return convertJsonRow(node, (RowType) dataType, options);
default:
- return convertPrimitiveStringToType(node.asText(), dataType,
options);
+ // Non-DECIMAL types keep the double text they got before
BigDecimal parsing.
+ String text =
+ node.isBigDecimal() &&
!dataType.is(DataTypeRoot.DECIMAL)
+ ? String.valueOf(node.doubleValue())
+ : node.asText();
+ return convertPrimitiveStringToType(text, dataType, options);
}
}
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/json/JsonFileFormatTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/json/JsonFileFormatTest.java
index ddb5be035f..3ddd28b5ee 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/json/JsonFileFormatTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/json/JsonFileFormatTest.java
@@ -44,6 +44,7 @@ import org.apache.paimon.types.RowType;
import org.junit.jupiter.api.Test;
import java.io.IOException;
+import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -666,6 +667,77 @@ public class JsonFileFormatTest extends
FormatReadWriteTest {
assertThat(result.get(0).getVector(1).toFloatArray()).isEqualTo(values);
}
+ @Test
+ public void testReadNumericDecimalKeepsPrecision() throws IOException {
+ // DECIMAL maps to a JSON number, but Paimon's own writer emits
DECIMAL as a string, so
+ // numeric literals only come from external writers.
+ RowType rowType =
+ RowType.of(
+ new DataType[] {
+ DataTypes.DECIMAL(20, 2),
+ DataTypes.DECIMAL(38, 18),
+ DataTypes.ARRAY(DataTypes.DECIMAL(20, 2)),
+ DataTypes.MAP(DataTypes.STRING(),
DataTypes.DECIMAL(20, 2)),
+ DataTypes.ROW(DataTypes.FIELD(0, "d",
DataTypes.DECIMAL(20, 2)))
+ },
+ new String[] {"d1", "d2", "arr", "m", "r"});
+ String json =
+ "{\"d1\":12345678901234567.89,\"d2\":123456789.123456789,"
+ +
"\"arr\":[12345678901234567.89],\"m\":{\"k\":12345678901234567.89},"
+ + "\"r\":{\"d\":12345678901234567.89}}";
+
+ List<InternalRow> result = readRawJson(rowType, json);
+
+ assertThat(result).hasSize(1);
+ InternalRow row = result.get(0);
+ BigDecimal expected = new BigDecimal("12345678901234567.89");
+ assertThat(row.getDecimal(0, 20,
2).toBigDecimal()).isEqualTo(expected);
+ assertThat(row.getDecimal(1, 38, 18).toBigDecimal())
+ .isEqualTo(new BigDecimal("123456789.123456789").setScale(18));
+ assertThat(row.getArray(2).getDecimal(0, 20,
2).toBigDecimal()).isEqualTo(expected);
+ assertThat(row.getMap(3).valueArray().getDecimal(0, 20,
2).toBigDecimal())
+ .isEqualTo(expected);
+ assertThat(row.getRow(4, 1).getDecimal(0, 20,
2).toBigDecimal()).isEqualTo(expected);
+ }
+
+ @Test
+ public void testReadNumericNonDecimalFieldsInDecimalRowUnchanged() throws
IOException {
+ RowType rowType =
+ RowType.of(
+ new DataType[] {
+ DataTypes.DECIMAL(20, 2),
+ DataTypes.STRING(),
+ DataTypes.STRING(),
+ DataTypes.DOUBLE()
+ },
+ new String[] {"d", "s1", "s2", "dbl"});
+ String json = "{\"d\":1.5,\"s1\":1e20,\"s2\":1000.0,\"dbl\":1000.0}";
+
+ List<InternalRow> result = readRawJson(rowType, json);
+
+ assertThat(result).hasSize(1);
+ InternalRow row = result.get(0);
+ assertThat(row.getDecimal(0, 20, 2).toBigDecimal()).isEqualTo(new
BigDecimal("1.50"));
+ assertThat(row.getString(1).toString()).isEqualTo("1.0E20");
+ assertThat(row.getString(2).toString()).isEqualTo("1000.0");
+ assertThat(row.getDouble(3)).isEqualTo(1000.0d);
+ }
+
+ @Test
+ public void testReadNegativeZeroDoubleWithoutDecimal() throws IOException {
+ RowType rowType =
+ RowType.of(
+ new DataType[] {DataTypes.DOUBLE(),
DataTypes.STRING()},
+ new String[] {"dbl", "s"});
+
+ List<InternalRow> result = readRawJson(rowType,
"{\"dbl\":-0.0,\"s\":-0.0}");
+
+ assertThat(result).hasSize(1);
+ assertThat(Double.doubleToRawLongBits(result.get(0).getDouble(0)))
+ .isEqualTo(Double.doubleToRawLongBits(-0.0d));
+ assertThat(result.get(0).getString(1).toString()).isEqualTo("-0.0");
+ }
+
@Override
public boolean supportDataFileWithoutExtension() {
return true;
@@ -713,6 +785,29 @@ public class JsonFileFormatTest extends
FormatReadWriteTest {
return new String(Files.readAllBytes(Paths.get(testFile.toUri())),
StandardCharsets.UTF_8);
}
+ private List<InternalRow> readRawJson(RowType rowType, String jsonLine)
throws IOException {
+ FileFormat format =
+ new JsonFileFormat(new FileFormatFactory.FormatContext(new
Options(), 1024, 1024));
+ Path testFile = new Path(parent, "raw_json_" + UUID.randomUUID() +
".json");
+ try (PositionOutputStream out = fileIO.newOutputStream(testFile,
false)) {
+ out.write((jsonLine + "\n").getBytes(StandardCharsets.UTF_8));
+ }
+ try (RecordReader<InternalRow> reader =
+ format.createReaderFactory(rowType, rowType, new ArrayList<>())
+ .createReader(
+ new FormatReaderContext(
+ fileIO,
+ testFile,
+ fileIO.getFileSize(testFile),
+ null,
+ null))) {
+ InternalRowSerializer serializer = new
InternalRowSerializer(rowType);
+ List<InternalRow> result = new ArrayList<>();
+ reader.forEachRemaining(row -> result.add(serializer.copy(row)));
+ return result;
+ }
+ }
+
private List<InternalRow> writeThenRead(
Options options, RowType rowType, List<InternalRow> testData,
String testPrefix)
throws IOException {
