This is an automated email from the ASF dual-hosted git repository.
nsivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 3b5bb09b1d36 fix(writer): reject directory-traversal partition paths
across all writers (#19342)
3b5bb09b1d36 is described below
commit 3b5bb09b1d365f5e31cf30a8dfdd0a48ec96b2b0
Author: Sivabalan Narayanan <[email protected]>
AuthorDate: Mon Oct 5 22:54:44 2026 -0700
fix(writer): reject directory-traversal partition paths across all writers
(#19342)
fix(writer): reject directory-traversal partition paths across all writers
A partition-path field value containing ".." segments could escape the
table base path and write Hudi-managed files into arbitrary directories
reachable by the writer's credentials. URL-encoding of partition values
is opt-in and disabled by default, and nothing else validated the value.
PartitionPathEncodeUtils.validateNoPathTraversal rejects whole ".."
segments regardless of the urlencode setting. It runs in the write
handles (HoodieWriteHandle, HoodieRowCreateHandle,
HoodieRowDataCreateHandle), once per partition rather than per record,
right before the partition directory is created.
Only a whole ".." segment can escape, so 2024-01-01, region=us-west-2
and hive-style rider=../evil are still accepted. Encoding instead of
throwing was rejected: escapePathName leaves "." untouched, and encoding
here would desync the directory from the partition path already recorded
in the HoodieKey, markers and WriteStatus.
---
.../apache/hudi/cli/commands/RepairsCommand.java | 4 +-
.../java/org/apache/hudi/io/BaseCreateHandle.java | 2 +-
.../hudi/io/FileGroupReaderBasedMergeHandle.java | 2 +-
.../apache/hudi/io/HoodieAbstractMergeHandle.java | 2 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 2 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 2 +-
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 2 +-
.../hudi/common/model/HoodiePartitionMetadata.java | 12 ++-
.../hudi/common/util/PartitionPathEncodeUtils.java | 60 +++++++++++++++
.../common/testutils/HoodieTestDataGenerator.java | 2 +-
.../common/util/TestPartitionPathEncodeUtils.java | 59 ++++++++++++++-
.../common/model/TestHoodiePartitionMetadata.java | 88 +++++++++++++++++++++-
.../hudi/common/testutils/HoodieTestTable.java | 2 +-
.../hudi/common/util/TestTablePathUtils.java | 4 +-
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 2 +-
.../AlterHoodieTableAddPartitionCommand.scala | 8 +-
.../RepairAddpartitionmetaProcedure.scala | 2 +-
.../RepairMigratePartitionMetaProcedure.scala | 2 +-
18 files changed, 234 insertions(+), 23 deletions(-)
diff --git
a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
index 9d6e1872aa68..a4698a083a61 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
@@ -142,7 +142,7 @@ public class RepairsCommand {
HoodiePartitionMetadata partitionMetadata =
new HoodiePartitionMetadata(HoodieCLI.storage, latestCommit,
basePath, partitionPath,
client.getTableConfig().getPartitionMetafileFormat());
- partitionMetadata.trySave();
+ partitionMetadata.trySave(partition);
row[2] = "Repaired";
}
}
@@ -305,7 +305,7 @@ public class RepairsCommand {
HoodiePartitionMetadata partitionMetadata =
new HoodiePartitionMetadata(HoodieCLI.storage, latestCommit,
basePath, partition,
Option.of(client.getTableConfig().getBaseFileFormat()));
- partitionMetadata.trySave();
+ partitionMetadata.trySave(partitionPath);
}
// delete it, in case we failed midway last time.
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/BaseCreateHandle.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/BaseCreateHandle.java
index 1ce8807e425b..1e5c5b83c05d 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/BaseCreateHandle.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/BaseCreateHandle.java
@@ -86,7 +86,7 @@ public abstract class BaseCreateHandle<T, I, K, O> extends
HoodieWriteHandle<T,
new StoragePath(config.getBasePath()),
FSUtils.constructAbsolutePath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
- partitionMetadata.trySave();
+ partitionMetadata.trySave(partitionPath);
createMarkerFile(partitionPath,
FSUtils.makeBaseFileName(this.instantTime, this.writeToken,
this.fileId, hoodieTable.getBaseFileExtension()));
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java
index c7abeda7eeed..4b5a227d4ff9 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/FileGroupReaderBasedMergeHandle.java
@@ -234,7 +234,7 @@ public class FileGroupReaderBasedMergeHandle<T, I, K, O>
extends HoodieWriteMerg
new StoragePath(config.getBasePath()),
FSUtils.constructAbsolutePath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
- partitionMetadata.trySave();
+ partitionMetadata.trySave(partitionPath);
String oldFileName = latestValidFilePath.isPresent() ?
latestValidFilePath.get() : null;
String newFileName = createNewFileName(oldFileName);
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAbstractMergeHandle.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAbstractMergeHandle.java
index 36c4525a7cf2..93b2ae6277c1 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAbstractMergeHandle.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAbstractMergeHandle.java
@@ -117,7 +117,7 @@ public abstract class HoodieAbstractMergeHandle<T, I, K, O>
extends HoodieWriteH
new StoragePath(config.getBasePath()),
FSUtils.constructAbsolutePath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
- partitionMetadata.trySave();
+ partitionMetadata.trySave(partitionPath);
String newFileName = createNewFileName(latestValidFilePath);
oldFilePath = makeNewFilePath(partitionPath, latestValidFilePath);
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
index 56a48a82dd80..00e62f361575 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
@@ -272,7 +272,7 @@ public abstract class HoodieAppendHandle<T, I, K, O>
extends HoodieWriteHandle<T
new StoragePath(config.getBasePath()),
FSUtils.constructAbsolutePath(config.getBasePath(), partitionPath),
hoodieTable.getPartitionMetafileFormat());
- partitionMetadata.trySave();
+ partitionMetadata.trySave(partitionPath);
String logInstantTime =
config.getWriteVersion().greaterThanOrEquals(HoodieTableVersion.EIGHT)
? getInstantTimeForLogFile(record) : deltaWriteStat.getPrevCommit();
diff --git
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
index f5cd0c739930..9fd73e261545 100644
---
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
+++
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
@@ -127,7 +127,7 @@ public class HoodieRowDataCreateHandle implements
Serializable {
new StoragePath(writeConfig.getBasePath()),
FSUtils.constructAbsolutePath(writeConfig.getBasePath(),
partitionPath),
table.getPartitionMetafileFormat());
- partitionMetadata.trySave();
+ partitionMetadata.trySave(partitionPath);
createMarkerFile(partitionPath,
FSUtils.makeBaseFileName(this.instantTime, getWriteToken(), this.fileId,
table.getBaseFileExtension()));
this.fileWriter = createNewFileWriter(path, table, writeConfig,
this.instantTime);
} catch (IOException e) {
diff --git
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
index d5d573d4d89a..33a615cb089b 100644
---
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
+++
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java
@@ -140,7 +140,7 @@ public class HoodieRowCreateHandle implements Serializable {
new StoragePath(writeConfig.getBasePath()),
FSUtils.constructAbsolutePath(writeConfig.getBasePath(),
partitionPath),
table.getPartitionMetafileFormat());
- partitionMetadata.trySave();
+ partitionMetadata.trySave(partitionPath);
createMarkerFile(partitionPath, fileName, instantTime, table,
writeConfig);
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
index be4fbabb95f9..ec18bffe065f 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
@@ -20,6 +20,7 @@ package org.apache.hudi.common.model;
import org.apache.hudi.common.util.FileFormatUtils;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.PartitionPathEncodeUtils;
import org.apache.hudi.common.util.RetryHelper;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.core.io.storage.HoodieIOFactory;
@@ -42,6 +43,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import static
org.apache.hudi.common.table.HoodieTableMetaClient.COMMIT_TIME_KEY;
+import static org.apache.hudi.common.util.ValidationUtils.checkArgument;
/**
* The metadata that goes into the meta file in each partition.
@@ -95,9 +97,14 @@ public class HoodiePartitionMetadata {
}
/**
- * Write the metadata safely into partition atomically.
+ * Write missing partition metadata safely and atomically, validating the
original relative partition path
+ * immediately before creating the file. Existing metadata files are not
revalidated.
+ *
+ * @param relativePartitionPath partition path before StoragePath
normalization, including any key-generator URL encoding;
+ * use an empty string for non-partitioned
tables
*/
- public void trySave() throws HoodieIOException {
+ public void trySave(String relativePartitionPath) throws HoodieIOException {
+ checkArgument(relativePartitionPath != null, "Relative partition path must
not be null");
StoragePath metaPath = new StoragePath(
partitionPath, HOODIE_PARTITION_METAFILE_PREFIX +
getMetafileExtension());
@@ -106,6 +113,7 @@ public class HoodiePartitionMetadata {
RetryHelper<Void, HoodieIOException> retryHelper = new RetryHelper(1000,
3, 1000, HoodieIOException.class.getName())
.tryWith(() -> {
if (!storage.exists(metaPath)) {
+
PartitionPathEncodeUtils.validateNoPathTraversal(relativePartitionPath);
if (format.isPresent()) {
writeMetafileInFormat(metaPath, format.get());
} else {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/PartitionPathEncodeUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/PartitionPathEncodeUtils.java
index a1c45d316740..6990a526d77b 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/util/PartitionPathEncodeUtils.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/util/PartitionPathEncodeUtils.java
@@ -17,6 +17,8 @@
package org.apache.hudi.common.util;
+import org.apache.hudi.exception.HoodieKeyException;
+
import java.util.BitSet;
import java.util.function.Function;
@@ -174,4 +176,62 @@ public class PartitionPathEncodeUtils {
return escapePathName(value);
}
}
+
+ /**
+ * Validates that the given (relative) partition path does not contain a
directory-traversal
+ * segment, throwing {@link HoodieKeyException} if it does. This is enforced
regardless of the
+ * {@code hoodie.datasource.write.partitionpath.urlencode} setting, since
url-encoding is opt-in
+ * (disabled by default) and does not escape {@code '.'}, so it never
neutralizes {@code ".."}.
+ *
+ * @param partitionPath the relative partition path (or a single partition
field value).
+ * @return the same {@code partitionPath} if it is safe.
+ * @throws HoodieKeyException if the partition path contains a {@code ".."}
traversal segment.
+ */
+ public static String validateNoPathTraversal(String partitionPath) {
+ if (hasPathTraversal(partitionPath)) {
+ throw new HoodieKeyException("Invalid partition path \"" + partitionPath
+ "\": partition paths "
+ + "must not contain \"..\" path-traversal segments, which could let
a record write Hudi files "
+ + "outside the table base path. This is most often caused by an
unsanitized data field being "
+ + "used as the partition path; sanitize or remap the offending value
in the upstream source "
+ + "or via a transformer before ingesting it.");
+ }
+ return partitionPath;
+ }
+
+ /**
+ * Returns {@code true} if the given (relative) partition path contains a
directory-traversal
+ * segment (a path segment equal to {@code ".."}). Such a partition path,
once resolved against
+ * the table base path, can escape the base path and write Hudi-managed
files into arbitrary
+ * directories reachable by the writer's credentials.
+ *
+ * <p>The check is intentionally value-content only: it tolerates {@code
'.'} inside a segment
+ * (e.g. date partitions like {@code 2024.01.01}) and only rejects the
standalone {@code ".."}
+ * segment. Both forward slash {@code '/'} and the platform-independent
literal are treated as
+ * separators, since a partition path is always stored using forward slashes.
+ *
+ * @param partitionPath the relative partition path (or a single partition
field value).
+ * @return {@code true} if a {@code ".."} traversal segment is present,
{@code false} otherwise.
+ */
+ public static boolean hasPathTraversal(String partitionPath) {
+ if (partitionPath == null || partitionPath.isEmpty()) {
+ return false;
+ }
+ // Normalize backslashes to forward slashes so a "..\.." style value
cannot slip through on
+ // any platform. Partition paths are always persisted using forward
slashes.
+ String normalized = partitionPath.replace('\\', '/');
+ int start = 0;
+ int len = normalized.length();
+ while (start <= len) {
+ int end = normalized.indexOf('/', start);
+ if (end < 0) {
+ end = len;
+ }
+ // A segment of exactly ".." is a traversal segment.
+ if (end - start == 2 && normalized.charAt(start) == '.' &&
normalized.charAt(start + 1) == '.') {
+ return true;
+ }
+ start = end + 1;
+ }
+ return false;
+ }
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
index ccaaa1b5fa81..67bdd8eee5e4 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
@@ -368,7 +368,7 @@ public class HoodieTestDataGenerator implements
AutoCloseable {
String basePath) {
for (String partitionPath : partitionPaths) {
new HoodiePartitionMetadata(storage, "000", new StoragePath(basePath),
- new StoragePath(basePath, partitionPath), Option.empty()).trySave();
+ new StoragePath(basePath, partitionPath),
Option.empty()).trySave(partitionPath);
}
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/util/TestPartitionPathEncodeUtils.java
b/hudi-common/src/test/java/org/apache/hudi/common/util/TestPartitionPathEncodeUtils.java
index 84e754a475a7..16621e1643d9 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/util/TestPartitionPathEncodeUtils.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/util/TestPartitionPathEncodeUtils.java
@@ -17,15 +17,22 @@
package org.apache.hudi.common.util;
+import org.apache.hudi.exception.HoodieKeyException;
+
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Tests for {@link PartitionPathEncodeUtils}.
*/
-public class TestPartitionPathEncodeUtils {
+class TestPartitionPathEncodeUtils {
@Test
public void testEscapeFileNameWithDotInColumnName() {
@@ -80,4 +87,54 @@ public class TestPartitionPathEncodeUtils {
assertEquals("date%3D2024.01.01", escaped);
assertEquals(original, PartitionPathEncodeUtils.unescapePathName(escaped));
}
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "..",
+ "../foo",
+ "foo/..",
+ "foo/../bar",
+ "../../../../tmp/evil",
+ "a/../../b",
+ "..\\foo",
+ "foo\\..\\bar"
+ })
+ void detectsPathTraversal(String partitionPath) {
+ assertTrue(PartitionPathEncodeUtils.hasPathTraversal(partitionPath),
+ "Expected traversal to be detected for: " + partitionPath);
+ assertThrows(HoodieKeyException.class,
+ () -> PartitionPathEncodeUtils.validateNoPathTraversal(partitionPath));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "2024/01/01",
+ "2024-01-01",
+ "region=us-west-2",
+ "year=2024/month=01",
+ "rider=..",
+ "rider=../evil",
+ "..foo",
+ "foo..",
+ "foo..bar",
+ "a.b.c",
+ "...",
+ "....",
+ "a/.hidden/b",
+ "."
+ })
+ void allowsLegitimatePartitionPaths(String partitionPath) {
+ assertFalse(PartitionPathEncodeUtils.hasPathTraversal(partitionPath),
+ "Expected no traversal for: " + partitionPath);
+ // validate should return the same value unchanged.
+ assertEquals(partitionPath,
PartitionPathEncodeUtils.validateNoPathTraversal(partitionPath));
+ }
+
+ @Test
+ void nullAndEmptyAreSafe() {
+ assertFalse(PartitionPathEncodeUtils.hasPathTraversal(null));
+ assertFalse(PartitionPathEncodeUtils.hasPathTraversal(""));
+ assertEquals(null, PartitionPathEncodeUtils.validateNoPathTraversal(null));
+ assertEquals("", PartitionPathEncodeUtils.validateNoPathTraversal(""));
+ }
}
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java
index ef01aa7deedf..7604670b588c 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java
@@ -20,7 +20,9 @@ package org.apache.hudi.common.model;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.PartitionPathEncodeUtils;
import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.exception.HoodieKeyException;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
@@ -36,6 +38,7 @@ import java.util.Arrays;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -70,14 +73,14 @@ public class TestHoodiePartitionMetadata extends
HoodieCommonTestHarness {
@MethodSource("formatProviderFn")
public void testTextFormatMetaFile(Option<HoodieFileFormat> format) throws
IOException {
// given
- final StoragePath partitionPath = new StoragePath(basePath, "a/b/"
- + format.map(Enum::name).orElse("text"));
+ String relativePartitionPath = "a/b/" +
format.map(Enum::name).orElse("text");
+ final StoragePath partitionPath = new StoragePath(basePath,
relativePartitionPath);
storage.createDirectory(partitionPath);
final String commitTime = "000000000001";
HoodiePartitionMetadata writtenMetadata = new HoodiePartitionMetadata(
metaClient.getStorage(), commitTime, new StoragePath(basePath),
partitionPath,
format);
- writtenMetadata.trySave();
+ writtenMetadata.trySave(relativePartitionPath);
// when
HoodiePartitionMetadata readMetadata = new HoodiePartitionMetadata(
@@ -89,6 +92,85 @@ public class TestHoodiePartitionMetadata extends
HoodieCommonTestHarness {
assertEquals(3, readMetadata.getPartitionDepth());
}
+ @ParameterizedTest
+ @MethodSource("formatProviderFn")
+ void testRejectsTraversalBeforeCreatingMetadata(Option<HoodieFileFormat>
format) throws IOException {
+ StoragePath tablePath = new StoragePath(basePath, "table");
+ for (String relativePartitionPath : Arrays.asList("..", "../outside",
"a/../b", "..\\outside")) {
+ StoragePath partitionPath = new StoragePath(tablePath,
relativePartitionPath);
+ storage.createDirectory(partitionPath);
+ HoodiePartitionMetadata metadata = new HoodiePartitionMetadata(
+ storage, "0001", tablePath, partitionPath, format);
+
+ assertThrows(HoodieKeyException.class, () ->
metadata.trySave(relativePartitionPath));
+ assertFalse(HoodiePartitionMetadata.hasPartitionMetadata(storage,
partitionPath));
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("formatProviderFn")
+ void testValidatesOriginalPathBeforeNormalization(Option<HoodieFileFormat>
format) throws IOException {
+ String relativePartitionPath = "a/../b";
+ StoragePath partitionPath = new StoragePath(basePath,
relativePartitionPath);
+ assertEquals(new StoragePath(basePath, "b"), partitionPath);
+ storage.createDirectory(partitionPath);
+ HoodiePartitionMetadata metadata = new HoodiePartitionMetadata(
+ storage, "0001", new StoragePath(basePath), partitionPath, format);
+ assertThrows(HoodieKeyException.class, () ->
metadata.trySave(relativePartitionPath));
+ assertTrue(storage.listDirectEntries(partitionPath).isEmpty());
+ }
+
+ @ParameterizedTest
+ @MethodSource("formatProviderFn")
+ void testExistingMetadataSkipsTraversalValidation(Option<HoodieFileFormat>
format) throws IOException {
+ StoragePath partitionPath = new StoragePath(basePath, "b");
+ storage.createDirectory(partitionPath);
+ HoodiePartitionMetadata original = new HoodiePartitionMetadata(
+ storage, "0001", new StoragePath(basePath), partitionPath, format);
+ original.trySave("b");
+
+ HoodiePartitionMetadata subsequent = new HoodiePartitionMetadata(
+ storage, "0002", new StoragePath(basePath), partitionPath, format);
+ subsequent.trySave("a/../b");
+ HoodiePartitionMetadata saved = new HoodiePartitionMetadata(storage,
partitionPath);
+ assertEquals(Option.of("0001"), saved.readPartitionCreatedCommitTime());
+ }
+
+ @ParameterizedTest
+ @MethodSource("formatProviderFn")
+ void testExistingEncodingIsAccepted(Option<HoodieFileFormat> format) throws
IOException {
+ String partitionValue = "../a/./b";
+ String encodedPath =
PartitionPathEncodeUtils.escapePathName(partitionValue);
+ StoragePath partitionPath = new StoragePath(basePath, encodedPath);
+ storage.createDirectory(partitionPath);
+ HoodiePartitionMetadata metadata = new HoodiePartitionMetadata(
+ storage, "0001", new StoragePath(basePath), partitionPath, format);
+ metadata.trySave(encodedPath);
+ assertTrue(HoodiePartitionMetadata.hasPartitionMetadata(storage,
partitionPath));
+ assertEquals(new StoragePath(basePath), partitionPath.getParent());
+ assertEquals(partitionValue,
PartitionPathEncodeUtils.unescapePathName(encodedPath));
+ }
+
+ @ParameterizedTest
+ @MethodSource("formatProviderFn")
+ void testEmptyRelativePartitionPath(Option<HoodieFileFormat> format) {
+ StoragePath tablePath = new StoragePath(basePath);
+ HoodiePartitionMetadata metadata = new HoodiePartitionMetadata(
+ storage, "0001", tablePath, tablePath, format);
+ metadata.trySave("");
+ assertTrue(HoodiePartitionMetadata.hasPartitionMetadata(storage,
tablePath));
+ assertEquals(0, metadata.getPartitionDepth());
+ }
+
+ @Test
+ void testRejectsNullRelativePartitionPath() {
+ StoragePath tablePath = new StoragePath(basePath);
+ HoodiePartitionMetadata metadata = new HoodiePartitionMetadata(
+ storage, "0001", tablePath, tablePath, Option.empty());
+ assertThrows(IllegalArgumentException.class, () -> metadata.trySave(null));
+ assertFalse(HoodiePartitionMetadata.hasPartitionMetadata(storage,
tablePath));
+ }
+
@Test
public void testErrorIfAbsent() throws IOException {
final StoragePath partitionPath = new StoragePath(basePath,
"a/b/not-a-partition");
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
index f731a3ff0bc0..1fe77dc7d7ad 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
@@ -732,7 +732,7 @@ public class HoodieTestTable implements AutoCloseable {
StoragePath partitionPathObj = FSUtils.getAbsolutePartitionPath(new
StoragePath(basePath), partitionPath);
HoodiePartitionMetadata partitionMetadata = new
HoodiePartitionMetadata(storage,
currentInstantTime, new StoragePath(basePath), partitionPathObj,
Option.empty());
- partitionMetadata.trySave();
+ partitionMetadata.trySave(partitionPath);
}
return this;
}
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java
index 048826e30eb4..896b520f7040 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java
@@ -81,11 +81,11 @@ public final class TestTablePathUtils {
HoodiePartitionMetadata partitionMetadata1 = new HoodiePartitionMetadata(
storage, Instant.now().toString(), tablePath,
partitionPath1, partitionMetafileFormat);
- partitionMetadata1.trySave();
+ partitionMetadata1.trySave("key1=abc/key2=def");
HoodiePartitionMetadata partitionMetadata2 = new HoodiePartitionMetadata(
storage, Instant.now().toString(), tablePath,
partitionPath2, partitionMetafileFormat);
- partitionMetadata2.trySave();
+ partitionMetadata2.trySave("key1=xyz/key2=def");
// Create files
URI filePathURI1 =
diff --git
a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java
b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java
index fb40535a6275..88e993520449 100644
---
a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java
+++
b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java
@@ -547,7 +547,7 @@ public class InputFormatTestUtil {
new StoragePath(partitionPath.toAbsolutePath().toString()),
Option.of(HoodieFileFormat.PARQUET));
- partitionMetadata.trySave();
+
partitionMetadata.trySave(basePath.toAbsolutePath().relativize(partitionPath.toAbsolutePath()).toString());
}
}
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddPartitionCommand.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddPartitionCommand.scala
index f56f73504dfa..4a4793555bc5 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddPartitionCommand.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddPartitionCommand.scala
@@ -73,10 +73,14 @@ case class AlterHoodieTableAddPartitionCommand(
throw new HoodieAnalysisException(s"Partition metadata already
exists for path: $fullPartitionPath")
}
None
- } else Some(new HoodiePartitionMetadata(storage,
HoodieTimeline.INIT_INSTANT_TS, basePath, fullPartitionPath, format))
+ } else {
+ Some((new HoodiePartitionMetadata(storage,
HoodieTimeline.INIT_INSTANT_TS, basePath, fullPartitionPath, format),
partitionPath))
+ }
(metadata, CatalogTablePartition(spec, table.storage.copy(locationUri =
Some(fullPartitionPath.toUri))))
}.unzip
- partitionMetadata.flatten.foreach(_.trySave)
+ partitionMetadata.flatten.foreach { case (metadata, relativePartitionPath)
=>
+ metadata.trySave(relativePartitionPath)
+ }
// Sync new partitions in batch, enable ignoreIfExists to be silent for
existing partitions.
val batchSize =
sparkSession.sparkContext.conf.getInt("spark.sql.addPartitionInBatch.size", 100)
diff --git
a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
index 90c145220dcb..6a90632827a5 100644
---
a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
+++
b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
@@ -69,7 +69,7 @@ class RepairAddpartitionmetaProcedure extends BaseProcedure
with ProcedureBuilde
if (!dryRun) {
val partitionMetadata: HoodiePartitionMetadata = new
HoodiePartitionMetadata(
metaClient.getStorage, latestCommit, basePath, partitionPath,
metaClient.getTableConfig.getPartitionMetafileFormat)
- partitionMetadata.trySave()
+ partitionMetadata.trySave(partition)
action = "Repaired"
}
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
index 1f7ebbfde237..57a172f9722e 100644
---
a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
+++
b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairMigratePartitionMetaProcedure.scala
@@ -80,7 +80,7 @@ class RepairMigratePartitionMetaProcedure extends
BaseProcedure with ProcedureBu
val partitionMetadata: HoodiePartitionMetadata = new
HoodiePartitionMetadata(
metaClient.getStorage, latestCommit,
basePath, partition,
Option.of(getWriteConfig(basePath.toString).getBaseFileFormat))
- partitionMetadata.trySave()
+ partitionMetadata.trySave(partitionPath)
}
// delete it, in case we failed midway last time.
textFormatFile.ifPresent(
