This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 3f2bc2c835 [format] Treat an unrecognized boolean string as malformed
in CSV and JSON (#10194)
3f2bc2c835 is described below
commit 3f2bc2c8355d50b15173ac9f6be9e3bd70d32a8f
Author: jackylee <[email protected]>
AuthorDate: Sat Oct 3 17:09:09 2026 +0800
[format] Treat an unrecognized boolean string as malformed in CSV and JSON
(#10194)
---
.../org/apache/paimon/format/csv/CsvParser.java | 13 ++++++-
.../paimon/format/json/JsonMapKeyConverter.java | 12 +++++-
.../paimon/format/csv/CsvFileFormatTest.java | 21 +++++++++++
.../paimon/format/json/JsonFileFormatTest.java | 43 ++++++++++++++++++++++
4 files changed, 87 insertions(+), 2 deletions(-)
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvParser.java
b/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvParser.java
index c72b72be8e..d17b84d3b4 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvParser.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvParser.java
@@ -243,7 +243,8 @@ public class CsvParser {
case DOUBLE:
return Pair.of(true, Double.parseDouble(field));
case BOOLEAN:
- return Pair.of(true, Boolean.parseBoolean(field));
+ Boolean boolVal = parseBoolean(field);
+ return Pair.of(boolVal != null, boolVal);
case CHAR:
case VARCHAR:
return Pair.of(true, BinaryString.fromString(field));
@@ -269,6 +270,16 @@ public class CsvParser {
return BinaryString.fromString(field);
}
+ private static Boolean parseBoolean(String s) {
+ if ("true".equalsIgnoreCase(s)) {
+ return Boolean.TRUE;
+ }
+ if ("false".equalsIgnoreCase(s)) {
+ return Boolean.FALSE;
+ }
+ return null;
+ }
+
private static Integer parseInt(String s) {
if (s == null || s.isEmpty()) {
return null;
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/json/JsonMapKeyConverter.java
b/paimon-format/src/main/java/org/apache/paimon/format/json/JsonMapKeyConverter.java
index d638c5e684..ab900837e8 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/json/JsonMapKeyConverter.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/json/JsonMapKeyConverter.java
@@ -44,7 +44,7 @@ final class JsonMapKeyConverter {
case DOUBLE:
return Double.parseDouble(value);
case BOOLEAN:
- return Boolean.parseBoolean(value);
+ return parseBoolean(value);
case CHAR:
case VARCHAR:
return BinaryString.fromString(value);
@@ -57,4 +57,14 @@ final class JsonMapKeyConverter {
return cast.cast(BinaryString.fromString(value));
}
}
+
+ private static boolean parseBoolean(String value) {
+ if ("true".equalsIgnoreCase(value)) {
+ return true;
+ }
+ if ("false".equalsIgnoreCase(value)) {
+ return false;
+ }
+ throw new IllegalArgumentException("Invalid boolean value: " + value);
+ }
}
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
index d2c26deb0f..435df580dc 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
@@ -39,6 +39,7 @@ import org.apache.paimon.options.Options;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Pair;
import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.dataformat.csv.CsvMapper;
import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema;
@@ -715,6 +716,26 @@ public class CsvFileFormatTest extends FormatReadWriteTest
{
assertThat(parser.parseField("32768",
DataTypes.SMALLINT()).getValue()).isNull();
}
+ @Test
+ public void testCsvParserParseBoolean() {
+ RowType rowType = DataTypes.ROW(DataTypes.BOOLEAN());
+ CsvParser parser = new CsvParser(rowType, new int[] {0}, new
CsvOptions(new Options()));
+
+ // Recognized boolean literals parse (case-insensitive), matching the
writer output.
+ assertThat(parser.parseField("true",
DataTypes.BOOLEAN()).getValue()).isEqualTo(true);
+ assertThat(parser.parseField("TRUE",
DataTypes.BOOLEAN()).getValue()).isEqualTo(true);
+ assertThat(parser.parseField("false",
DataTypes.BOOLEAN()).getValue()).isEqualTo(false);
+ assertThat(parser.parseField("False",
DataTypes.BOOLEAN()).getValue()).isEqualTo(false);
+
+ // Unrecognized values are malformed, not silently false, so csv.mode
can
+ // null/drop/fail them the same way it does for a malformed number.
+ for (String invalid : new String[] {"1", "0", "t", "yes", "abc"}) {
+ Pair<Boolean, Object> parsed = parser.parseField(invalid,
DataTypes.BOOLEAN());
+ assertThat(parsed.getLeft()).as(invalid).isFalse();
+ assertThat(parsed.getValue()).as(invalid).isNull();
+ }
+ }
+
private List<InternalRow> read(
FileFormat format, RowType fullRowType, RowType readRowType, Path
testFile)
throws IOException {
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/json/JsonFileFormatTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/json/JsonFileFormatTest.java
index 0629c3fe83..0908a2ab94 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/json/JsonFileFormatTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/json/JsonFileFormatTest.java
@@ -193,6 +193,49 @@ public class JsonFileFormatTest extends
FormatReadWriteTest {
}
}
+ @Test
+ public void testIgnoreParseErrorsNullsMalformedBoolean() throws
IOException {
+ RowType rowType = DataTypes.ROW(DataTypes.INT().notNull(),
DataTypes.BOOLEAN());
+
+ Options options = new Options();
+ options.set(JsonOptions.JSON_IGNORE_PARSE_ERRORS, true);
+
+ FileFormat format =
+ new JsonFileFormat(new
FileFormatFactory.FormatContext(options, 1024, 1024));
+
+ Path testFile = new Path(parent, "test_bool_" + UUID.randomUUID() +
".json");
+ try (PositionOutputStream out = fileIO.newOutputStream(testFile,
false)) {
+ out.write("{\"f0\":1,\"f1\":true}\n".getBytes());
+ out.write("{\"f0\":2,\"f1\":\"FALSE\"}\n".getBytes());
+ out.write("{\"f0\":3,\"f1\":\"maybe\"}\n".getBytes());
+ out.write("{\"f0\":4,\"f1\":1}\n".getBytes());
+ }
+
+ try (RecordReader<InternalRow> reader =
+ format.createReaderFactory(rowType, rowType, new ArrayList<>())
+ .createReader(
+ new FormatReaderContext(
+ fileIO,
+ testFile,
+ fileIO.getFileSize(testFile),
+ null,
+ null))) {
+
+ InternalRowSerializer serializer = new
InternalRowSerializer(rowType);
+ List<InternalRow> result = new ArrayList<>();
+ reader.forEachRemaining(row -> result.add(serializer.copy(row)));
+
+ assertThat(result).hasSize(4);
+ // Recognized literals parse, case-insensitively.
+ assertThat(result.get(0).getBoolean(1)).isTrue();
+ assertThat(result.get(1).getBoolean(1)).isFalse();
+ // An unrecognized string or a non-boolean number is malformed, so
it is
+ // nulled like any other bad value instead of silently becoming
false.
+ assertThat(result.get(2).isNullAt(1)).isTrue();
+ assertThat(result.get(3).isNullAt(1)).isTrue();
+ }
+ }
+
@Test
public void testIgnoreParseErrorsDisabled() throws IOException {
RowType rowType = DataTypes.ROW(DataTypes.INT().notNull(),
DataTypes.STRING());
