This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new f582e30ab9 [core] Add compression extension to text format data file
names (#9853)
f582e30ab9 is described below
commit f582e30ab98b7bffc5d3a065a43a69eb5a164732
Author: jackylee <[email protected]>
AuthorDate: Thu Sep 17 14:44:04 2026 +0800
[core] Add compression extension to text format data file names (#9853)
---
.../org/apache/paimon/io/DataFilePathFactory.java | 8 ++++++--
.../apache/paimon/io/DataFilePathFactoryTest.java | 22 ++++++++++++++++++++++
2 files changed, 28 insertions(+), 2 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java
b/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java
index 33068859dc..7c53ebf6b8 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java
@@ -18,6 +18,7 @@
package org.apache.paimon.io;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.format.HadoopCompressionType;
import org.apache.paimon.fs.ExternalPathProvider;
@@ -207,8 +208,11 @@ public class DataFilePathFactory {
}
private static boolean isTextFormat(String formatIdentifier) {
- return "json".equalsIgnoreCase(formatIdentifier)
- || "csv".equalsIgnoreCase(formatIdentifier);
+ // Keep this list in step with CoreOptions.formatTableFileCompression,
which groups the
+ // same three formats: they are the ones whose codec is recovered from
the file name.
+ return CoreOptions.FILE_FORMAT_JSON.equalsIgnoreCase(formatIdentifier)
+ ||
CoreOptions.FILE_FORMAT_CSV.equalsIgnoreCase(formatIdentifier)
+ ||
CoreOptions.FILE_FORMAT_TEXT.equalsIgnoreCase(formatIdentifier);
}
@Nullable
diff --git
a/paimon-core/src/test/java/org/apache/paimon/io/DataFilePathFactoryTest.java
b/paimon-core/src/test/java/org/apache/paimon/io/DataFilePathFactoryTest.java
index d2d0eb8f36..bb24940c3d 100644
---
a/paimon-core/src/test/java/org/apache/paimon/io/DataFilePathFactoryTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/io/DataFilePathFactoryTest.java
@@ -113,6 +113,28 @@ public class DataFilePathFactoryTest {
assertThat(DataFilePathFactory.formatIdentifier(video.getName())).isEqualTo("video");
}
+ @Test
+ public void testTextFormatCarriesTheCompressionExtension() {
+ // A text file is always written through the configured codec, while
the reader recovers
+ // that codec from the file name alone, so the name has to end with
the codec extension.
+ // The compression is passed explicitly rather than taken from the
option default so that
+ // the expected name does not silently change with it.
+ DataFilePathFactory pathFactory =
+ new DataFilePathFactory(
+ new Path(tempDir + "/bucket-123"),
+ CoreOptions.FILE_FORMAT_TEXT,
+ CoreOptions.DATA_FILE_PREFIX.defaultValue(),
+ CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
+
CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
+ "zstd",
+ null);
+
+ String name = pathFactory.newPath().getName();
+ assertThat(name).endsWith(".text.zst");
+ assertThat(DataFilePathFactory.formatIdentifier(name))
+ .isEqualTo(CoreOptions.FILE_FORMAT_TEXT);
+ }
+
@Test
public void testEntropyInjectWithNoPartition() {
EntropyInjectExternalPathProvider externalPathProvider =
