This is an automated email from the ASF dual-hosted git repository.
ethanfeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 7f08eb8f1 [CELEBORN-2137] Remove unused MAPGROUP PartitionType
7f08eb8f1 is described below
commit 7f08eb8f1dde495cfccd20098952589f543a2826
Author: SteNicholas <[email protected]>
AuthorDate: Wed Sep 3 09:58:07 2025 +0800
[CELEBORN-2137] Remove unused MAPGROUP PartitionType
### What changes were proposed in this pull request?
Remove unused `MAPGROUP` `PartitionType`.
### Why are the changes needed?
`PartitionType` `MAPGROUP` is unused at present, which could be removed.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI.
Closes #3459 from SteNicholas/CELEBORN-2137.
Authored-by: SteNicholas <[email protected]>
Signed-off-by: mingji <[email protected]>
---
.../main/java/org/apache/celeborn/common/protocol/PartitionType.java | 3 +--
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala | 3 +--
.../src/main/scala/org/apache/celeborn/common/util/PbSerDeUtils.scala | 2 --
common/src/main/scala/org/apache/celeborn/common/util/Utils.scala | 1 -
.../celeborn/service/deploy/worker/storage/StorageManager.scala | 4 ----
.../apache/celeborn/service/deploy/worker/storage/StoragePolicy.scala | 2 --
6 files changed, 2 insertions(+), 13 deletions(-)
diff --git
a/common/src/main/java/org/apache/celeborn/common/protocol/PartitionType.java
b/common/src/main/java/org/apache/celeborn/common/protocol/PartitionType.java
index aab4a4ea3..59e0f3e72 100644
---
a/common/src/main/java/org/apache/celeborn/common/protocol/PartitionType.java
+++
b/common/src/main/java/org/apache/celeborn/common/protocol/PartitionType.java
@@ -19,8 +19,7 @@ package org.apache.celeborn.common.protocol;
public enum PartitionType {
REDUCE(0),
- MAP(1),
- MAPGROUP(2);
+ MAP(1);
private final int value;
diff --git
a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
index 75c0dae22..19cc15982 100644
--- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -5043,8 +5043,7 @@ object CelebornConf extends Logging {
.transform(_.toUpperCase(Locale.ROOT))
.checkValues(Set(
PartitionType.REDUCE.name,
- PartitionType.MAP.name,
- PartitionType.MAPGROUP.name))
+ PartitionType.MAP.name))
.createWithDefault(PartitionType.REDUCE.name)
val SHUFFLE_PARTITION_SPLIT_THRESHOLD: ConfigEntry[Long] =
diff --git
a/common/src/main/scala/org/apache/celeborn/common/util/PbSerDeUtils.scala
b/common/src/main/scala/org/apache/celeborn/common/util/PbSerDeUtils.scala
index b74a31fdf..1e96ece3e 100644
--- a/common/src/main/scala/org/apache/celeborn/common/util/PbSerDeUtils.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/util/PbSerDeUtils.scala
@@ -113,8 +113,6 @@ object PbSerDeUtils {
// writer always closed as this is committed file info.
fileMeta.setIsWriterClosed(true)
fileMeta
- case PartitionType.MAPGROUP =>
- throw new NotImplementedError("Map group is not implemented")
}
new DiskFileInfo(
userIdentifier,
diff --git a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
index cddcfec40..e71db2ead 100644
--- a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
@@ -1085,7 +1085,6 @@ object Utils extends Logging {
value match {
case 0 => PartitionType.REDUCE
case 1 => PartitionType.MAP
- case 2 => PartitionType.MAPGROUP
case _ =>
logWarning(s"invalid partitionType $value, fallback to
ReducePartition")
PartitionType.REDUCE
diff --git
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
index f6699ebd6..13dcd0460 100644
---
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
+++
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
@@ -1053,8 +1053,6 @@ final private[worker] class StorageManager(conf:
CelebornConf, workerSource: Abs
new ReduceFileMeta(conf.shuffleChunkSize)
case PartitionType.MAP =>
new MapFileMeta()
- case PartitionType.MAPGROUP =>
- throw new NotImplementedError("Map group is not implemented")
}
val shuffleKey = Utils.makeShuffleKey(appId, shuffleId)
val memoryFileInfo =
@@ -1229,8 +1227,6 @@ final private[worker] class StorageManager(conf:
CelebornConf, workerSource: Abs
mapFileMeta.setMountPoint(mountPoint)
}
mapFileMeta
- case PartitionType.MAPGROUP =>
- throw new NotImplementedError("Map group is not implemented")
}
}
}
diff --git
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StoragePolicy.scala
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StoragePolicy.scala
index a475d12f2..6dd236da4 100644
---
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StoragePolicy.scala
+++
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StoragePolicy.scala
@@ -90,8 +90,6 @@ class StoragePolicy(conf: CelebornConf, storageManager:
StorageManager, source:
fileInfo.asInstanceOf[DiskFileInfo],
notifier)
}
- case PartitionType.MAPGROUP =>
- null
}
}
