This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new be41a82370 [format] Support lineSep as fallback key of
json.line-delimiter (#10377)
be41a82370 is described below
commit be41a82370e882e46e2ef965cf858a69157c4b64
Author: Eunbin Son <[email protected]>
AuthorDate: Mon Oct 5 21:42:18 2026 +0900
[format] Support lineSep as fallback key of json.line-delimiter (#10377)
---
.../org/apache/paimon/format/json/JsonOptions.java | 1 +
.../paimon/format/json/JsonFileFormatTest.java | 59 ++++++++++++++++++++++
2 files changed, 60 insertions(+)
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/json/JsonOptions.java
b/paimon-format/src/main/java/org/apache/paimon/format/json/JsonOptions.java
index 1b21db3e87..fc49ac7e29 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/json/JsonOptions.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/json/JsonOptions.java
@@ -52,6 +52,7 @@ public class JsonOptions {
ConfigOptions.key("json.line-delimiter")
.stringType()
.defaultValue("\n")
+ .withFallbackKeys("lineSep")
.withDescription("The line delimiter for JSON format");
/** Enum for handling null keys in JSON maps. */
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/json/JsonFileFormatTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/json/JsonFileFormatTest.java
index 0908a2ab94..ddb5be035f 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/json/JsonFileFormatTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/json/JsonFileFormatTest.java
@@ -592,6 +592,65 @@ public class JsonFileFormatTest extends
FormatReadWriteTest {
}
}
+ @Test
+ public void testJsonLineDelimiterFallbackKey() throws IOException {
+ RowType rowType = DataTypes.ROW(DataTypes.INT().notNull(),
DataTypes.STRING());
+
+ Options options = new Options();
+ options.set("lineSep", "|");
+
+ assertThat(options.get(JsonOptions.LINE_DELIMITER)).isEqualTo("|");
+
+ // the writer uses the delimiter given by the fallback key
+ String json =
+ writeToJson(
+ options,
+ rowType,
+ GenericRow.of(1, BinaryString.fromString("Alice")),
+ "test_line_sep");
+ assertThat(json).endsWith("|").doesNotContain("\n");
+
+ // the reader splits a '|' delimited file into rows
+ Path testFile = new Path(parent, "test_line_sep_read_" +
UUID.randomUUID() + ".json");
+ try (PositionOutputStream out = fileIO.newOutputStream(testFile,
false)) {
+ out.write(
+ "{\"f0\":1,\"f1\":\"Alice\"}|{\"f0\":2,\"f1\":\"Bob\"}|"
+ .getBytes(StandardCharsets.UTF_8));
+ }
+ FileFormat format =
+ new JsonFileFormat(new
FileFormatFactory.FormatContext(options, 1024, 1024));
+ List<InternalRow> result = new ArrayList<>();
+ try (RecordReader<InternalRow> reader =
+ format.createReaderFactory(rowType, rowType, new ArrayList<>())
+ .createReader(
+ new FormatReaderContext(
+ fileIO,
+ testFile,
+ fileIO.getFileSize(testFile),
+ null,
+ null))) {
+ InternalRowSerializer serializer = new
InternalRowSerializer(rowType);
+ reader.forEachRemaining(row -> result.add(serializer.copy(row)));
+ }
+ assertThat(result).hasSize(2);
+ assertThat(result.get(0).getInt(0)).isEqualTo(1);
+ assertThat(result.get(0).getString(1).toString()).isEqualTo("Alice");
+ assertThat(result.get(1).getInt(0)).isEqualTo(2);
+ assertThat(result.get(1).getString(1).toString()).isEqualTo("Bob");
+
+ List<InternalRow> roundTrip =
+ writeThenRead(
+ options,
+ rowType,
+ Arrays.asList(
+ GenericRow.of(1,
BinaryString.fromString("Alice")),
+ GenericRow.of(2,
BinaryString.fromString("Bob"))),
+ "test_line_sep_round_trip");
+ assertThat(roundTrip).hasSize(2);
+
assertThat(roundTrip.get(0).getString(1).toString()).isEqualTo("Alice");
+ assertThat(roundTrip.get(1).getString(1).toString()).isEqualTo("Bob");
+ }
+
@Test
public void testVectorTypeReadWrite() throws IOException {
RowType rowType = DataTypes.ROW(DataTypes.INT(), DataTypes.VECTOR(3,
DataTypes.FLOAT()));
