This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 6cd1196795d8 fix(spark): make partition DDL commands honor slash
separated date partitioning (#19703)
6cd1196795d8 is described below
commit 6cd1196795d875adcc935a956e2790a769d39bb0
Author: Sepuri Sai Krishna <[email protected]>
AuthorDate: Wed Aug 26 07:55:28 2026 +0530
fix(spark): make partition DDL commands honor slash separated date
partitioning (#19703)
* fix(spark): make partition DDL commands honor slash separated date
partitioning
HoodieSqlCommonUtils#makePartitionPath read hive-style partitioning and URL
encoding from the table config but never getSlashSeparatedDatePartitioning,
so
on a slash-partitioned table the DDL commands computed 2026-01-05 while the
writer had laid the partition out as 2026/01/05. ADD PARTITION created a
stray
dashed directory (and its existence check never saw the real partition),
while
DROP PARTITION and TRUNCATE PARTITION targeted a directory that does not
exist
and silently removed nothing.
Apply the same substitution the write path performs, confined to a single
partition field and skipped for hive-style partitioning, with a leading dash
left alone so the partition path never becomes absolute.
Closes #19702
* fix(spark): reuse the write path's path-breaking dash guard in partition
DDL
The dash-to-slash substitution these commands perform carried its own guard,
which only refused a leading dash. #19648 widened the write-side rule to
reject
any dash-delimited token that is empty, "." or "..", after finding that
"..-a"
became "../a" and resolved outside the table base path.
Duplicating the rule is what let the two drift, so ask KeyGenUtils rather
than
restate it: the commands have to name the directory the writer created,
which
means agreeing on every value, not just on the well formed dates.
Covers the three token classes at the DDL level, where the stray directory
is
observable: "..-a" must not create a sibling of the table, "2026-" must not
lose
its trailing dash to a normalized-away slash, and "a--b" must not split into
nested directories.
---
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 38 ++++++++-
.../sql/hudi/ddl/TestAlterTableAddPartition.scala | 97 ++++++++++++++++++++++
.../sql/hudi/ddl/TestAlterTableDropPartition.scala | 41 +++++++++
.../spark/sql/hudi/ddl/TestTruncateTable.scala | 35 ++++++++
4 files changed, 208 insertions(+), 3 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
index 1462ceab78dd..9f2027405152 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala
@@ -29,6 +29,7 @@ import
org.apache.hudi.common.table.timeline.{HoodieInstantTimeGenerator, Hoodie
import
org.apache.hudi.common.table.timeline.TimelineUtils.parseDateFromInstantTime
import org.apache.hudi.common.util.PartitionPathEncodeUtils
import org.apache.hudi.exception.HoodieException
+import org.apache.hudi.keygen.KeyGenUtils
import org.apache.hudi.storage.{HoodieStorage, StoragePath, StoragePathInfo}
import org.apache.hudi.util.SparkConfigUtils
@@ -406,24 +407,55 @@ object HoodieSqlCommonUtils extends SparkAdapterSupport {
private def makePartitionPath(partitionFields: Seq[String],
normalizedSpecs: Map[String, String],
enableEncodeUrl: Boolean,
- enableHiveStylePartitioning: Boolean): String
= {
+ enableHiveStylePartitioning: Boolean,
+ slashSeparatedDatePartitioning: Boolean):
String = {
+ // NOTE: Slash-separated date partitioning only kicks in for a table
partitioned by a single
+ // (date) column, mirroring the guard in
[[KeyGenUtils#getRecordPartitionPath]] that drives
+ // the write path -- these commands have to name the very same
directory the writer created.
+ // Hive-style partitioning is excluded because the config documents
the two as mutually
+ // exclusive, and the write paths do not agree on what the
combination should produce
+ // (tracked in HUDI issue #19669), so there is no single directory
to name here
+ val useSlashSeparatedDates =
+ slashSeparatedDatePartitioning && !enableHiveStylePartitioning &&
partitionFields.length == 1
partitionFields.map { partitionColumn =>
val encodedPartitionValue = if (enableEncodeUrl) {
PartitionPathEncodeUtils.escapePathName(normalizedSpecs(partitionColumn))
} else {
normalizedSpecs(partitionColumn)
}
- if (enableHiveStylePartitioning)
s"$partitionColumn=$encodedPartitionValue" else encodedPartitionValue
+ if (enableHiveStylePartitioning) {
+ s"$partitionColumn=$encodedPartitionValue"
+ } else if (useSlashSeparatedDates) {
+ toSlashSeparatedDate(encodedPartitionValue)
+ } else {
+ encodedPartitionValue
+ }
}.mkString("/")
}
+ /**
+ * Turns a `yyyy-MM-dd` formatted partition value into the `yyyy/MM/dd`
directory structure
+ * requested by `hoodie.datasource.write.slash.separated.date.partitioning`,
mirroring the
+ * substitution the write path performs in
`KeyGenUtils#getRecordPartitionPath`.
+ *
+ * The path-breaking values the writer refuses to slash are refused here for
the same reasons, by
+ * asking [[KeyGenUtils#hasPathBreakingDash]] rather than restating the
rule: these commands have
+ * to name the directory the writer created, so the two have to agree on
every value, not just on
+ * the dates. See that method for what each excluded case does to the path.
+ */
+ private def toSlashSeparatedDate(partitionValue: String): String = {
+ if (KeyGenUtils.hasPathBreakingDash(partitionValue)) partitionValue else
partitionValue.replace('-', '/')
+ }
+
def makePartitionPath(hoodieCatalogTable: HoodieCatalogTable,
normalizedSpecs: Map[String, String]): String = {
val tableConfig = hoodieCatalogTable.tableConfig
val enableHiveStylePartitioning =
java.lang.Boolean.parseBoolean(tableConfig.getHiveStylePartitioningEnable)
val enableEncodeUrl =
java.lang.Boolean.parseBoolean(tableConfig.getUrlEncodePartitioning)
+ val slashSeparatedDatePartitioning =
tableConfig.getSlashSeparatedDatePartitioning
- makePartitionPath(hoodieCatalogTable.partitionFields, normalizedSpecs,
enableEncodeUrl, enableHiveStylePartitioning)
+ makePartitionPath(hoodieCatalogTable.partitionFields, normalizedSpecs,
enableEncodeUrl,
+ enableHiveStylePartitioning, slashSeparatedDatePartitioning)
}
private def validateInstant(queryInstant: String): Unit = {
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableAddPartition.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableAddPartition.scala
index 3fcb3d4a3e12..0862ab506003 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableAddPartition.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableAddPartition.scala
@@ -18,6 +18,9 @@
package org.apache.spark.sql.hudi.ddl
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
+import org.junit.jupiter.api.Assertions.{assertFalse, assertTrue}
+
+import java.io.File
class TestAlterTableAddPartition extends HoodieSparkSqlTestBase {
@@ -227,4 +230,98 @@ class TestAlterTableAddPartition extends
HoodieSparkSqlTestBase {
}
}
}
+
+ test("Add partition for a slash separated date partitioned table") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = s"${tmp.getCanonicalPath}/$tableName"
+ // create table
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id bigint,
+ | name string,
+ | ts string,
+ | dt string
+ | )
+ | using hudi
+ | tblproperties (
+ | primaryKey = 'id',
+ | orderingFields = 'ts',
+ | hoodie.datasource.write.slash.separated.date.partitioning =
'true'
+ | )
+ | partitioned by (dt)
+ | location '$tablePath'
+ |""".stripMargin)
+
+ // The writer lays a partition value out as yyyy/MM/dd, so the DDL
command has to name that
+ // very directory rather than the dashed value
+ spark.sql(s"""insert into $tableName values (1, "a1", "v1",
"2026-01-05")""")
+
+ spark.sql(s"alter table $tableName add partition (dt='2026-02-06')")
+
+ assertTrue(new File(tablePath, "2026/02/06").exists(),
+ "ADD PARTITION should create the slash separated directory")
+ assertFalse(new File(tablePath, "2026-02-06").exists(),
+ "ADD PARTITION should not leave a dashed directory behind")
+
+ // naming the right directory also lets the existence check see the
partition the writer created
+ spark.sql(s"alter table $tableName add if not exists partition
(dt='2026-01-05')")
+ checkExceptionContain(s"alter table $tableName add partition
(dt='2026-01-05')")(
+ "Partition metadata already exists for path")
+ }
+ }
+
+ test("Add partition leaves path breaking values alone under slash separated
date partitioning") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = s"${tmp.getCanonicalPath}/$tableName"
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id bigint,
+ | name string,
+ | ts string,
+ | dt string
+ | )
+ | using hudi
+ | tblproperties (
+ | primaryKey = 'id',
+ | orderingFields = 'ts',
+ | hoodie.datasource.write.slash.separated.date.partitioning =
'true'
+ | )
+ | partitioned by (dt)
+ | location '$tablePath'
+ |""".stripMargin)
+
+ // NOTE: The writer refuses to slash these values -- see
[[KeyGenUtils#hasPathBreakingDash]]
+ // for what each one does to the path -- so the DDL command has to
refuse them too, or
+ // it creates a directory the writer would never have named
+ spark.sql(s"alter table $tableName add partition (dt='..-a')")
+ assertTrue(new File(tablePath, "..-a").exists(),
+ "ADD PARTITION should create the literal directory, not a dot segment")
+ // "../a" would resolve to a sibling of the table directory
+ assertFalse(new File(tmp.getCanonicalPath, "a").exists(),
+ "ADD PARTITION must not create a directory outside the table base
path")
+
+ // "2026/" is normalized back to "2026" by StoragePath#normalize
+ spark.sql(s"alter table $tableName add partition (dt='2026-')")
+ assertTrue(new File(tablePath, "2026-").exists(),
+ "ADD PARTITION should keep a trailing dash")
+ assertFalse(new File(tablePath, "2026").exists(),
+ "ADD PARTITION should not drop a trailing dash by way of a trailing
slash")
+
+ // "a//b" is collapsed to "a/b" by URI#normalize
+ spark.sql(s"alter table $tableName add partition (dt='a--b')")
+ assertTrue(new File(tablePath, "a--b").exists(),
+ "ADD PARTITION should keep a doubled dash")
+ assertFalse(new File(tablePath, "a").exists(),
+ "ADD PARTITION should not split a doubled dash into nested
directories")
+
+ // a single interior dash is still a separator
+ spark.sql(s"alter table $tableName add partition (dt='2026-02-06')")
+ assertTrue(new File(tablePath, "2026/02/06").exists(),
+ "ADD PARTITION should still slash a well formed date")
+ }
+ }
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableDropPartition.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableDropPartition.scala
index 1e7ac9c4c5ff..8ad58019b0cd 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableDropPartition.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableDropPartition.scala
@@ -692,4 +692,45 @@ class TestAlterTableDropPartition extends
HoodieSparkSqlTestBase {
}
}
}
+
+ test("Drop partition for a slash separated date partitioned table") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = s"${tmp.getCanonicalPath}/$tableName"
+ // create table
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id bigint,
+ | name string,
+ | ts string,
+ | dt string
+ | )
+ | using hudi
+ | tblproperties (
+ | primaryKey = 'id',
+ | orderingFields = 'ts',
+ | hoodie.datasource.write.slash.separated.date.partitioning =
'true'
+ | )
+ | partitioned by (dt)
+ | location '$tablePath'
+ |""".stripMargin)
+ // insert data
+ spark.sql(s"""insert into $tableName values (1, "z3", "v1",
"2026-01-05"), (2, "l4", "v1", "2026-01-06")""")
+
+ // the writer laid these out as 2026/01/05 and 2026/01/06, so DROP
PARTITION has to name the
+ // same directory -- naming the dashed value drops nothing while still
reporting success
+ spark.sql(s"alter table $tableName drop partition (dt='2026-01-05')")
+
+ // trigger clean so that partition deletion kicks in.
+ withSQLConf(HoodieCleanConfig.CLEANER_POLICY.key() ->
HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS.name()) {
+ spark.sql(s"call run_clean(table => '$tableName', retain_commits =>
1)")
+ .collect()
+ }
+
+ checkAnswer(s"select id, name, ts, dt from $tableName")(
+ Seq(2, "l4", "v1", "2026-01-06")
+ )
+ }
+ }
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestTruncateTable.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestTruncateTable.scala
index ba824c9c68f5..075164a57dff 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestTruncateTable.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestTruncateTable.scala
@@ -148,4 +148,39 @@ class TestTruncateTable extends HoodieSparkSqlTestBase {
}
}
}
+
+ test("Test Truncate partition for a slash separated date partitioned table")
{
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ val tablePath = s"${tmp.getCanonicalPath}/$tableName"
+
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id bigint,
+ | name string,
+ | ts string,
+ | dt string
+ | )
+ | using hudi
+ | tblproperties (
+ | primaryKey = 'id',
+ | orderingFields = 'ts',
+ | hoodie.datasource.write.slash.separated.date.partitioning =
'true'
+ | )
+ | partitioned by (dt)
+ | location '$tablePath'
+ |""".stripMargin)
+
+ spark.sql(s"""insert into $tableName values (1, "z3", "v1",
"2026-01-05"), (2, "l4", "v1", "2026-01-06")""")
+
+ // the writer laid these out as 2026/01/05 and 2026/01/06, so TRUNCATE
PARTITION has to name
+ // the same directory -- naming the dashed value truncates nothing
+ spark.sql(s"truncate table $tableName partition (dt='2026-01-05')")
+
+ checkAnswer(s"select id, name, ts, dt from $tableName")(
+ Seq(2, "l4", "v1", "2026-01-06")
+ )
+ }
+ }
}
