This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 9b138f28e7 [format] Simplify buffered Avro encoder selection
9b138f28e7 is described below
commit 9b138f28e7ba6493e1f4fea68e9add57ee744d15
Author: JingsongLi <[email protected]>
AuthorDate: Fri Sep 18 13:15:57 2026 +0800
[format] Simplify buffered Avro encoder selection
---
.../java/org/apache/paimon/format/FileFormat.java | 9 ++------
.../apache/paimon/format/FileFormatFactory.java | 25 ----------------------
.../apache/paimon/format/avro/AvroFileFormat.java | 4 +---
.../paimon/format/avro/AvroFileFormatTest.java | 4 ++--
4 files changed, 5 insertions(+), 37 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
index 7a06e544ea..f17c23b255 100644
--- a/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
+++ b/paimon-common/src/main/java/org/apache/paimon/format/FileFormat.java
@@ -74,10 +74,6 @@ public abstract class FileFormat {
}
public static FileFormat fromIdentifier(String identifier, Options
options) {
- return fromIdentifier(identifier, options, false);
- }
-
- private static FileFormat fromIdentifier(String identifier, Options
options, boolean manifest) {
return fromIdentifier(
normalizeFileFormat(identifier),
new FormatContext(
@@ -86,8 +82,7 @@ public abstract class FileFormat {
options.get(CoreOptions.WRITE_BATCH_SIZE),
options.get(CoreOptions.WRITE_BATCH_MEMORY),
options.get(CoreOptions.FILE_COMPRESSION_ZSTD_LEVEL),
- options.get(CoreOptions.FILE_BLOCK_SIZE),
- manifest));
+ options.get(CoreOptions.FILE_BLOCK_SIZE)));
}
/** Create a {@link FileFormat} from format identifier and format options.
*/
@@ -125,6 +120,6 @@ public abstract class FileFormat {
// Manifest blocks retain the Avro default independently of data-file
block sizing.
Options manifestOptions = new Options(options.toMap());
manifestOptions.remove(CoreOptions.FILE_BLOCK_SIZE);
- return FileFormat.fromIdentifier(CoreOptions.FILE_FORMAT_AVRO,
manifestOptions, true);
+ return FileFormat.fromIdentifier(CoreOptions.FILE_FORMAT_AVRO,
manifestOptions);
}
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
b/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
index 157f151ad2..b5cba150fe 100644
---
a/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
+++
b/paimon-common/src/main/java/org/apache/paimon/format/FileFormatFactory.java
@@ -41,7 +41,6 @@ public interface FileFormatFactory {
private final MemorySize writeBatchMemory;
private final int zstdLevel;
@Nullable private final MemorySize blockSize;
- private final boolean manifest;
@VisibleForTesting
public FormatContext(Options options, int readBatchSize, int
writeBatchSize) {
@@ -64,31 +63,12 @@ public interface FileFormatFactory {
MemorySize writeBatchMemory,
int zstdLevel,
@Nullable MemorySize blockSize) {
- this(
- options,
- readBatchSize,
- writeBatchSize,
- writeBatchMemory,
- zstdLevel,
- blockSize,
- false);
- }
-
- public FormatContext(
- Options options,
- int readBatchSize,
- int writeBatchSize,
- MemorySize writeBatchMemory,
- int zstdLevel,
- @Nullable MemorySize blockSize,
- boolean manifest) {
this.options = options;
this.readBatchSize = readBatchSize;
this.writeBatchSize = writeBatchSize;
this.writeBatchMemory = writeBatchMemory;
this.zstdLevel = zstdLevel;
this.blockSize = blockSize;
- this.manifest = manifest;
}
public Options options() {
@@ -116,11 +96,6 @@ public interface FileFormatFactory {
return blockSize;
}
- /** Whether the format is used for manifest metadata rather than data
files. */
- public boolean isManifest() {
- return manifest;
- }
-
public boolean caseSensitive() {
return
options.getOptional(CatalogOptions.CASE_SENSITIVE).orElse(true);
}
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
index 845cce65d5..ad243b6132 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java
@@ -68,7 +68,6 @@ public class AvroFileFormat extends FileFormat {
private final Options options;
private final int zstdLevel;
@Nullable private final MemorySize blockSize;
- private final boolean useBufferedEncoder;
public AvroFileFormat(FormatContext context) {
super(IDENTIFIER);
@@ -76,7 +75,6 @@ public class AvroFileFormat extends FileFormat {
this.options = getIdentifierPrefixOptions(context.options());
this.zstdLevel = context.zstdLevel();
this.blockSize = context.blockSize();
- this.useBufferedEncoder = !context.isManifest();
}
@Override
@@ -160,7 +158,7 @@ public class AvroFileFormat extends FileFormat {
@Override
public FormatWriter create(PositionOutputStream out, String
compression)
throws IOException {
- return createBlockWriter(out, rowType, compression,
useBufferedEncoder);
+ return createBlockWriter(out, rowType, compression, true);
}
}
}
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
index 251f3ca974..bef9ef8873 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
@@ -261,7 +261,7 @@ public class AvroFileFormatTest {
@ParameterizedTest
@ValueSource(strings = {"avro", "parquet", "orc"})
- void testManifestWriterUsesDirectEncoder(String dataFileFormat) throws
IOException {
+ void testManifestRowWriterUsesBufferedEncoder(String dataFileFormat)
throws IOException {
Options options = new Options();
options.set(CoreOptions.FILE_FORMAT, dataFileFormat);
FileFormat manifestFormat = FileFormat.manifestFormat(new
CoreOptions(options));
@@ -275,7 +275,7 @@ public class AvroFileFormatTest {
assertThat(writer)
.extracting("writer")
.extracting("bufOut")
- .isInstanceOf(DirectBinaryEncoder.class);
+ .isInstanceOf(BufferedBinaryEncoder.class);
writer.addElement(GenericRow.of(42));
}
}
