This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 394cbaaa11e141362f5a5d6c5dadd07d72b56430 Author: JingsongLi <[email protected]> AuthorDate: Thu Sep 17 11:35:21 2026 +0800 Revert "[format] Support file.block-size for Avro (#9901)" This reverts commit 7f8c08e16977101c9207816f89e69576f0861717. --- docs/generated/core_configuration.html | 2 +- .../main/java/org/apache/paimon/CoreOptions.java | 3 +- .../apache/paimon/format/avro/AvroFileFormat.java | 6 -- .../paimon/format/avro/AvroFileFormatTest.java | 71 ---------------------- 4 files changed, 2 insertions(+), 80 deletions(-) diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 19f6d52603..cf48d473c4 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -738,7 +738,7 @@ under the License. <td><h5>file.block-size</h5></td> <td style="word-wrap: break-word;">(none)</td> <td>MemorySize</td> - <td>File block size of format, default value of orc stripe is 64 MB, parquet row group is 128 MB, and avro block is 64000 bytes. For avro, this is the uncompressed block size threshold.</td> + <td>File block size of format, default value of orc stripe is 64 MB, and parquet row group is 128 MB.</td> </tr> <tr> <td><h5>file.compression</h5></td> diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 20c59b1f95..9bb324b780 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -426,8 +426,7 @@ public class CoreOptions implements Serializable { .memoryType() .noDefaultValue() .withDescription( - "File block size of format, default value of orc stripe is 64 MB, parquet row group is 128 MB, " - + "and avro block is 64000 bytes. For avro, this is the uncompressed block size threshold."); + "File block size of format, default value of orc stripe is 64 MB, and parquet row group is 128 MB."); public static final ConfigOption<MemorySize> FILE_INDEX_IN_MANIFEST_THRESHOLD = key("file-index.in-manifest-threshold") diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java index a23c66af21..69512d5ee7 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java @@ -29,7 +29,6 @@ import org.apache.paimon.fs.CloseShieldOutputStream; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.ConfigOptions; -import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.statistics.SimpleColStatsCollector; @@ -66,14 +65,12 @@ public class AvroFileFormat extends FileFormat { private final Options options; private final int zstdLevel; - @Nullable private final MemorySize blockSize; public AvroFileFormat(FormatContext context) { super(IDENTIFIER); this.options = getIdentifierPrefixOptions(context.options()); this.zstdLevel = context.zstdLevel(); - this.blockSize = context.blockSize(); } @Override @@ -96,9 +93,6 @@ public class AvroFileFormat extends FileFormat { AvroRowDatumWriter datumWriter = new AvroRowDatumWriter(rowType); DataFileWriter<InternalRow> writer = new DataFileWriter<>(datumWriter); writer.setCodec(createCodecFactory(compression)); - if (blockSize != null) { - writer.setSyncInterval(Math.toIntExact(blockSize.getBytes())); - } writer.setFlushOnEveryBlock(false); writer.create(schema, new CloseShieldOutputStream(out)); return new AvroBlockWriter(writer, out, schema); diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java index 33b5798b5b..c7ca45b88f 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java @@ -48,9 +48,6 @@ import org.apache.avro.io.EncoderFactory; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.ValueSource; import java.io.ByteArrayOutputStream; import java.io.File; @@ -79,74 +76,6 @@ public class AvroFileFormatTest { fileFormat = new AvroFileFormat(new FormatContext(new Options(), 1024, 1024)); } - @ParameterizedTest - @CsvSource({ - ", 64000, null", - ", 64000, deflate", - "1 kb, 1024, null", - "1 kb, 1024, deflate", - "4 kb, 4096, null", - "4 kb, 4096, deflate", - "128 kb, 131072, null", - "128 kb, 131072, deflate" - }) - void testFileBlockSize(String blockSize, int expectedBlockSize, String compression) - throws IOException { - Options options = new Options(); - if (blockSize != null) { - options.setString("file.block-size", blockSize); - } - FileFormat format = FileFormat.fromIdentifier("avro", options); - RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()).notNull(); - LocalFileIO fileIO = LocalFileIO.create(); - Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); - int numRecords = 300_000; - - try (PositionOutputStream out = fileIO.newOutputStream(file, false); - FormatWriter writer = - format.createWriterFactory(rowType).create(out, compression)) { - // Each record is encoded as one byte, so record counts also give uncompressed sizes. - for (int i = 0; i < numRecords; i++) { - writer.addElement(GenericRow.of(0)); - } - } - - long records = 0; - try (AvroBlockReader reader = new AvroBlockReader(fileIO.newInputStream(file))) { - while (reader.hasNextBlock()) { - AvroRawBlock block = reader.nextBorrowedRawBlock(); - long expectedRecords = Math.min(expectedBlockSize, numRecords - records); - assertThat(block.recordCount()).isEqualTo(expectedRecords); - assertThat(block.decompress(null).remaining()).isEqualTo((int) expectedRecords); - records += block.recordCount(); - } - } - assertThat(records).isEqualTo(numRecords); - } - - @ParameterizedTest - @ValueSource(longs = {2147483648L, 4294968320L, Long.MAX_VALUE}) - void testFileBlockSizeOverflow(long blockSize) throws IOException { - Options options = new Options(); - options.setString("file.block-size", Long.toString(blockSize)); - FileFormat format = FileFormat.fromIdentifier("avro", options); - RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()).notNull(); - LocalFileIO fileIO = LocalFileIO.create(); - Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); - - try (PositionOutputStream out = fileIO.newOutputStream(file, false)) { - assertThatThrownBy( - () -> { - try (FormatWriter writer = - format.createWriterFactory(rowType).create(out, "null")) { - writer.addElement(GenericRow.of(0)); - } - }) - .isInstanceOf(ArithmeticException.class) - .hasMessage("integer overflow"); - } - } - @Test public void testSupportedDataTypes() { ArrayList<DataField> dataFields = new ArrayList<>();
