This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 3670decd5a [spark] Deduplicate partitions for markDone (#5328)
3670decd5a is described below
commit 3670decd5a63a2c1a8a7c3703840db9d72488de3
Author: Zouxxyy <[email protected]>
AuthorDate: Sat Mar 22 20:41:00 2025 +0800
[spark] Deduplicate partitions for markDone (#5328)
---
.../scala/org/apache/paimon/spark/commands/WriteIntoPaimonTable.scala | 1 +
1 file changed, 1 insertion(+)
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/WriteIntoPaimonTable.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/WriteIntoPaimonTable.scala
index 84b44984f8..d526ac2c3b 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/WriteIntoPaimonTable.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/WriteIntoPaimonTable.scala
@@ -109,6 +109,7 @@ case class WriteIntoPaimonTable(
)
val partitions = commitMessages
.map(c => c.partition())
+ .distinct
.map(p =>
PartitionPathUtils.generatePartitionPath(partitionComputer.generatePartValues(p)))
for (partition <- partitions) {
actions.forEach(a => a.markDone(partition))
