This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 1ab8b32729 [core] Reject hidden partition values in value-only format
table layout (#10750)
1ab8b32729 is described below
commit 1ab8b32729b76893a6c4c1720913c5354535d701
Author: suntectec <[email protected]>
AuthorDate: Mon Oct 26 14:22:28 2026 +0800
[core] Reject hidden partition values in value-only format table layout
(#10750)
---
.../paimon/table/format/FormatTableCommit.java | 6 +-
.../paimon/table/format/FormatTableFileWriter.java | 12 +-
.../apache/paimon/utils/PartitionPathUtils.java | 40 +++++++
.../FormatTableValueOnlyHiddenPartitionTest.java | 123 +++++++++++++++++++++
.../paimon/utils/PartitionPathUtilsTest.java | 36 ++++++
5 files changed, 211 insertions(+), 6 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
index d71ab7e258..2b67c26751 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
@@ -728,7 +728,7 @@ public class FormatTableCommit implements BatchTableCommit {
return partitionSpec;
}
- private static Path buildPartitionPath(
+ private Path buildPartitionPath(
String location,
Map<String, String> partitionSpec,
boolean formatTablePartitionOnlyValueInPath,
@@ -751,6 +751,10 @@ public class FormatTableCommit implements BatchTableCommit
{
throw new RuntimeException("partitionSpec does not contain
key: " + key);
}
}
+ // Fail loudly on value-only partition values that a scan would hide
('_'/'.') rather than
+ // silently dropping the partition; the configured default partition
name is exempt.
+ PartitionPathUtils.validatePartitionSpecForWrite(
+ orderedSpec.values(), formatTablePartitionOnlyValueInPath,
defaultPartName);
return new Path(
location,
PartitionPathUtils.generatePartitionPathUtil(
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableFileWriter.java
b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableFileWriter.java
index a3b48ee7fa..66453d402a 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableFileWriter.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableFileWriter.java
@@ -29,6 +29,7 @@ import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.FileStorePathFactory;
import org.apache.paimon.utils.IOUtils;
+import org.apache.paimon.utils.PartitionPathUtils;
import java.util.ArrayList;
import java.util.HashMap;
@@ -134,11 +135,12 @@ public class FormatTableFileWriter {
if (partition.getFieldCount() > 0) {
LinkedHashMap<String, String> partValues =
pathFactory.partitionComputer().generatePartValues(partition);
- parent =
- new Path(
- parent,
- generatePartitionPathUtil(
- partValues,
options.formatTablePartitionOnlyValueInPath()));
+ boolean onlyValueInPath =
options.formatTablePartitionOnlyValueInPath();
+ // Reject value-only partition values that scans would hide
('_'/'.') instead of
+ // silently dropping the whole partition on read; the default
partition name is exempt.
+ PartitionPathUtils.validatePartitionSpecForWrite(
+ partValues.values(), onlyValueInPath,
options.partitionDefaultName());
+ parent = new Path(parent, generatePartitionPathUtil(partValues,
onlyValueInPath));
}
return new FormatTableRecordWriter(
fileIO,
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
index aec2b21aae..630a45c21d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
@@ -166,6 +166,46 @@ public class PartitionPathUtils {
validatePartitionSpecForPath(partitionSpec, true);
}
+ /**
+ * Write-side guard for the value-only partition layout.
+ *
+ * <p>Scans treat any value-only directory whose name starts with {@code
'_'} or {@code '.'} as
+ * a hidden staging or metadata directory (see {@link #isHiddenName}) and
skip it, so a
+ * partition written under such a value and all of its data would silently
disappear from every
+ * read with no error at write or scan time. Validating the value here
turns that silent data
+ * loss into a loud write-time failure.
+ *
+ * <p>The configured default partition name stays allowed because scans
deliberately un-hide
+ * exactly that name (see {@link #isHiddenFile}). Callers only reach this
for the value-only
+ * layout; the key-value layout writes {@code pt=<value>} directories
whose leading {@code '_'}
+ * or {@code '.'} is contributed by the column name, not the value, and is
therefore never
+ * hidden.
+ */
+ public static void validatePartitionValueForWrite(
+ String value, boolean onlyValueInPath, @Nullable String
defaultPartValue) {
+ validatePartitionValueForPath(value, onlyValueInPath);
+ if (onlyValueInPath && !value.equals(defaultPartValue) &&
isHiddenName(value)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Partition value '%s' cannot be written in a
value-only partition "
+ + "layout: scans treat a directory whose
name starts with '_' or "
+ + "'.' as hidden and would silently drop
this partition and all "
+ + "of its data. Use a partition value that
does not start with "
+ + "'_' or '.', or switch to the key-value
partition layout.",
+ value));
+ }
+ }
+
+ /** Validate every value of a partition spec against the write-side
value-only rules. */
+ public static void validatePartitionSpecForWrite(
+ Iterable<String> partitionValues,
+ boolean onlyValueInPath,
+ @Nullable String defaultPartValue) {
+ for (String value : partitionValues) {
+ validatePartitionValueForWrite(value, onlyValueInPath,
defaultPartValue);
+ }
+ }
+
/**
* Build the partition-name prefix pattern pushed down to a
partition-managing catalog from the
* leading equality prefix of a partition predicate.
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableValueOnlyHiddenPartitionTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableValueOnlyHiddenPartitionTest.java
new file mode 100644
index 0000000000..175ab032ae
--- /dev/null
+++
b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableValueOnlyHiddenPartitionTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * Regression tests for writing hidden-name partitions into a value-only
format table layout.
+ *
+ * <p>Scans skip any value-only directory whose name starts with {@code '_'}
or {@code '.'} (see
+ * {@link org.apache.paimon.utils.PartitionPathUtils}), so a partition written
under such a value
+ * used to vanish silently from every read. The write path must now fail
loudly instead. The
+ * configured default partition name stays writable because scans deliberately
un-hide exactly it.
+ */
+class FormatTableValueOnlyHiddenPartitionTest {
+
+ private static final String DEFAULT_PART_NAME = "__null__";
+
+ private static final RowType TABLE_TYPE =
+ RowType.builder()
+ .field("value", DataTypes.STRING())
+ .field("pt", DataTypes.STRING())
+ .build();
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @ParameterizedTest
+ @ValueSource(strings = {"_x", ".tmp", "__hidden"})
+ void testHiddenValueOnlyPartitionFailsLoudlyAtWrite(String pt) {
+ // Before the fix this write succeeded and the partition silently
disappeared on read.
+ assertThatThrownBy(() -> writeOneRow(pt))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(pt)
+ .hasMessageContaining("value-only");
+ }
+
+ @Test
+ void testVisibleValueOnlyPartitionStillWrites() throws Exception {
+ List<CommitMessage> messages = writeOneRow("2025");
+ assertThat(messages).isNotEmpty();
+ }
+
+ @Test
+ void testDefaultPartitionNameIsWritableInValueOnlyLayout() throws
Exception {
+ // A null partition value is substituted with the default name, which
starts with '_' but is
+ // exempted because scans un-hide exactly that name.
+ List<CommitMessage> messages = writeOneRow(null);
+ assertThat(messages).isNotEmpty();
+ assertThat(tempDir.resolve(DEFAULT_PART_NAME)).exists();
+ }
+
+ private List<CommitMessage> writeOneRow(String pt) throws Exception {
+ FormatTable table = valueOnlyTable();
+ BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = writeBuilder.newWrite()) {
+ write.write(
+ GenericRow.of(
+ BinaryString.fromString("row"),
+ pt == null ? null : BinaryString.fromString(pt)));
+ return write.prepareCommit();
+ }
+ }
+
+ private FormatTable valueOnlyTable() {
+ Path tablePath = new Path(tempDir.toUri());
+ Map<String, String> options = new HashMap<>();
+ options.put(CoreOptions.PATH.key(), tablePath.toString());
+ options.put(CoreOptions.FILE_FORMAT.key(), "csv");
+ options.put(
+ CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
+ Boolean.TRUE.toString());
+ options.put(CoreOptions.PARTITION_DEFAULT_NAME.key(),
DEFAULT_PART_NAME);
+ return FormatTable.builder()
+ .fileIO(LocalFileIO.create())
+ .identifier(Identifier.create("test_db", "value_only"))
+ .rowType(TABLE_TYPE)
+ .partitionKeys(Collections.singletonList("pt"))
+ .location(tablePath.toString())
+ .format(FormatTable.Format.CSV)
+ .options(options)
+ .build();
+ }
+}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java
b/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java
index 6026eadc04..5525afe0ca 100644
---
a/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java
@@ -72,6 +72,42 @@ class PartitionPathUtilsTest {
.containsExactly(entry("pt", rawValue));
}
+ @ParameterizedTest
+ @ValueSource(strings = {"_x", ".tmp", "._y", "__hidden", ".", ".."})
+ void testValueOnlyHiddenPrefixRejectedAtWrite(String rawValue) {
+ // A value-only directory starting with '_' or '.' is hidden to scans
(see isHiddenName), so
+ // writing it would silently drop the partition; the write path must
reject it loudly.
+ assertThat(PartitionPathUtils.isHiddenName(rawValue)).isTrue();
+ assertThatThrownBy(
+ () ->
+
PartitionPathUtils.validatePartitionValueForWrite(
+ rawValue, true,
"__DEFAULT_PARTITION__"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(rawValue);
+ }
+
+ @Test
+ void testValueOnlyDefaultPartitionNameIsAllowedAtWrite() {
+ // The configured default name starts with '_' yet scans un-hide
exactly that name, so it
+ // must stay writable.
+ String defaultName = "__DEFAULT_PARTITION__";
+ PartitionPathUtils.validatePartitionValueForWrite(defaultName, true,
defaultName);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"2025", "a:b", "a b"})
+ void testVisibleValueOnlyValueAcceptedAtWrite(String rawValue) {
+ PartitionPathUtils.validatePartitionValueForWrite(rawValue, true,
null);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"_x", ".tmp"})
+ void testKeyValueLayoutKeepsHiddenPrefixValuesAtWrite(String rawValue) {
+ // In a key=value layout the directory is 'pt=_x'; the leading char
comes from the column
+ // name, not the value, so the directory is never hidden and the value
stays writable.
+ PartitionPathUtils.validatePartitionValueForWrite(rawValue, false,
null);
+ }
+
@Test
void testTrailingPartitionExtractionStopsAtTableBoundary() {
Path partitionPath = new
Path("file:///warehouse/env=prod/table/dt=20260718/hh=10");
