RexXiong commented on code in PR #3365:
URL: https://github.com/apache/celeborn/pull/3365#discussion_r2289728295
##########
master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java:
##########
@@ -249,6 +254,34 @@ public void
removeWorkersUnavailableInfoMeta(List<WorkerInfo> unavailableWorkers
}
}
+ private boolean hasAvailableStorage(WorkerInfo workerInfo) {
+ Map<String, DiskInfo> disks = workerInfo.diskInfos();
+ Pair<Boolean, Long> exceedCheckResult =
isExceedingUnhealthyThreshold(disks);
+
+ boolean hasDisk = !disks.isEmpty();
+ boolean isExceeding = exceedCheckResult.getLeft();
+ long unhealthyCount = exceedCheckResult.getRight();
+
+ if (hasDisk) {
+ if (!isExceeding) {
+ return true;
+ } else {
+ LOG.info(
+ "Worker {} doesn't have enough healthy local disk (unhealthy
count: {})",
+ workerInfo,
+ unhealthyCount);
+ }
+ }
+
+ if (hasRemoteStorage) {
+ LOG.info("Worker {} has remote storage", workerInfo);
Review Comment:
No need print log
##########
common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala:
##########
@@ -6536,6 +6540,25 @@ object CelebornConf extends Logging {
.checkValue(v => v > 0.0 && v <= 1.0, "Should be in (0.0, 1.0].")
.createWithDefault(1)
+ val MASTER_AUTO_RELEASE_HIGH_WORKLOAD_WORKER_ENABLE: ConfigEntry[Boolean] =
Review Comment:
MASTER_AUTO_RELEASE_HIGH_WORKLOAD_WORKER_ENABLED
##########
master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java:
##########
@@ -249,6 +254,34 @@ public void
removeWorkersUnavailableInfoMeta(List<WorkerInfo> unavailableWorkers
}
}
+ private boolean hasAvailableStorage(WorkerInfo workerInfo) {
+ Map<String, DiskInfo> disks = workerInfo.diskInfos();
+ Pair<Boolean, Long> exceedCheckResult =
isExceedingUnhealthyThreshold(disks);
+
+ boolean hasDisk = !disks.isEmpty();
+ boolean isExceeding = exceedCheckResult.getLeft();
+ long unhealthyCount = exceedCheckResult.getRight();
+
+ if (hasDisk) {
+ if (!isExceeding) {
+ return true;
+ } else {
+ LOG.info(
Review Comment:
`LOG.info` -> `LOG.warn`
##########
master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java:
##########
@@ -89,6 +91,9 @@ public abstract class AbstractMetaManager implements
IMetadataHandler {
public long initialEstimatedPartitionSize;
public long estimatedPartitionSize;
public double unhealthyDiskRatioThreshold;
+ public boolean autoReleaseHighWorkLoadEnabled;
Review Comment:
protected scope is enough?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
