[hadoop] branch branch-3.3 updated: YARN-11482. Fix bug of DRF comparision DominantResourceFairnessComparator2 in fair scheduler. (#5607). Contributed by Xiaoqiao He.
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new f6850b89f36 YARN-11482. Fix bug of DRF comparision
DominantResourceFairnessComparator2 in fair scheduler. (#5607). Contributed by
Xiaoqiao He.
f6850b89f36 is described below
commit f6850b89f36aa0a68a5d3a350be70d1a4eddb7a2
Author: Hexiaoqiao
AuthorDate: Sat Apr 29 11:18:42 2023 +0800
YARN-11482. Fix bug of DRF comparision DominantResourceFairnessComparator2
in fair scheduler. (#5607). Contributed by Xiaoqiao He.
Reviewed-by: Shilun Fan
(cherry picked from commit 70c0aa342e6a6a12b647bbfe30bb73313958e450)
---
.../scheduler/fair/policies/DominantResourceFairnessPolicy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
index 83af8b64945..7ddfff548e7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
@@ -390,7 +390,7 @@ public class DominantResourceFairnessPolicy extends
SchedulingPolicy {
// share for that resource
boolean s1Needy = resourceInfo1[dominant1].getValue() <
minShareInfo1[dominant1].getValue();
- boolean s2Needy = resourceInfo1[dominant2].getValue() <
+ boolean s2Needy = resourceInfo2[dominant2].getValue() <
minShareInfo2[dominant2].getValue();
int res;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11079. Make an AbstractParentQueue to store common ParentQueue and ManagedParentQueue functionality. Contributed by Susheel Gupta
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c7699d3dcd4 YARN-11079. Make an AbstractParentQueue to store common
ParentQueue and ManagedParentQueue functionality. Contributed by Susheel Gupta
c7699d3dcd4 is described below
commit c7699d3dcd4f8feaf2c5ae5943b8a4cec738e95d
Author: Szilard Nemeth
AuthorDate: Thu May 4 22:16:18 2023 -0400
YARN-11079. Make an AbstractParentQueue to store common ParentQueue and
ManagedParentQueue functionality. Contributed by Susheel Gupta
---
.../ProportionalCapacityPreemptionPolicy.java |6 +-
.../monitor/capacity/TempQueuePerPartition.java|3 +-
.../csmappingrule/MappingRuleValidationHelper.java |5 +-
.../scheduler/capacity/AbstractCSQueue.java|6 +-
.../capacity/AbstractManagedParentQueue.java |2 +-
.../{ParentQueue.java => AbstractParentQueue.java} | 233 +--
.../capacity/AutoCreatedQueueManagementPolicy.java |4 +-
.../capacity/CSMaxRunningAppsEnforcer.java | 12 +-
.../scheduler/capacity/CapacityScheduler.java |2 +-
.../capacity/CapacitySchedulerConfigValidator.java |4 +-
.../capacity/CapacitySchedulerQueueManager.java|9 +-
.../scheduler/capacity/ManagedParentQueue.java |2 +-
.../scheduler/capacity/ParentQueue.java| 1572 +---
.../scheduler/capacity/PlanQueue.java |1 +
.../GuaranteedOrZeroCapacityOverTimePolicy.java|8 +-
.../event/QueueManagementChangeEvent.java |9 +-
.../webapp/dao/CapacitySchedulerInfo.java |6 +-
.../webapp/dao/CapacitySchedulerQueueInfo.java |6 +-
.../dao/helper/CapacitySchedulerInfoHelper.java|3 +-
.../TestCapacitySchedulerNewQueueAutoCreation.java |4 +-
.../capacity/TestCapacitySchedulerQueues.java |2 +-
.../scheduler/capacity/TestChildQueueOrder.java|4 +-
.../scheduler/capacity/TestParentQueue.java|4 +-
23 files changed, 133 insertions(+), 1774 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 1c4d60962ee..443241a664a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -21,6 +21,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractParentQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -41,7 +42,6 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
.ManagedParentQueue;
-import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptableQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent;
@@ -630,9 +630,9 @@ public class ProportionalCapacityPreemptionPolicy
partitionToLookAt, killable, absCap, absMaxCap, partitionResource,
reserved, curQueue, effMinRes, effMaxRes);
- if (curQueue instanceof ParentQueue) {
+ if (curQueue instanceof AbstractParentQueue) {
String configuredOrderingPolicy =
-((ParentQueue) curQueue).getQueueOrderingPolicy().getConfigName();
+((AbstractParentQueue)
curQueue).getQueueOrderingPolicy().getConfigName();
// Recursively add children
for (CSQueue c : curQueue.getChildQueues()) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
[hadoop] branch branch-3.3 updated: HADOOP-18727. Fix WriteOperations.listMultipartUploads function description (#5613)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 4670f9e8b08 HADOOP-18727. Fix WriteOperations.listMultipartUploads
function description (#5613)
4670f9e8b08 is described below
commit 4670f9e8b08d36b99e67d1a632e1762c88f5561a
Author: Dongjoon Hyun
AuthorDate: Thu May 4 05:03:48 2023 -0700
HADOOP-18727. Fix WriteOperations.listMultipartUploads function description
(#5613)
Contributed by Dongjoon Hyun
---
.../src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
index 321390446f7..91e0342cf7b 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
@@ -191,11 +191,11 @@ public interface WriteOperations extends AuditSpanSource,
Closeable {
throws IOException;
/**
- * Abort multipart uploads under a path: limited to the first
+ * List in-progress multipart uploads under a path: limited to the first
* few hundred.
- * @param prefix prefix for uploads to abort
- * @return a count of aborts
- * @throws IOException trouble; FileNotFoundExceptions are swallowed.
+ * @param prefix prefix for uploads to list
+ * @return a list of in-progress multipart uploads
+ * @throws IOException on problems
*/
List listMultipartUploads(String prefix)
throws IOException;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (27776ac45ea -> ceb8878d4fa)
This is an automated email from the ASF dual-hosted git repository. inigoiri pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git from 27776ac45ea HADOOP-18727. Fix WriteOperations.listMultipartUploads function description (#5613) add ceb8878d4fa HDFS-16998. RBF: Add ops metrics for getSlowDatanodeReport in RouterClientActivity (#5615) No new revisions were added by this update. Summary of changes: .../hadoop/hdfs/server/federation/router/RouterClientMetrics.java | 8 .../hdfs/server/federation/metrics/TestRouterClientMetrics.java | 7 +++ 2 files changed, 15 insertions(+) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (bd607951c02 -> 27776ac45ea)
This is an automated email from the ASF dual-hosted git repository. stevel pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git from bd607951c02 YARN-11463. Node Labels root directory creation doesn't have a retry logic - addendum (#5614) add 27776ac45ea HADOOP-18727. Fix WriteOperations.listMultipartUploads function description (#5613) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java | 8 1 file changed, 4 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11463. Node Labels root directory creation doesn't have a retry logic - addendum (#5614)
This is an automated email from the ASF dual-hosted git repository.
bteke pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new bd607951c02 YARN-11463. Node Labels root directory creation doesn't
have a retry logic - addendum (#5614)
bd607951c02 is described below
commit bd607951c02f17d1b443cfa3f5690385db67da08
Author: Peter Szucs <116345192+p-sz...@users.noreply.github.com>
AuthorDate: Thu May 4 12:27:25 2023 +0200
YARN-11463. Node Labels root directory creation doesn't have a retry logic
- addendum (#5614)
---
.../yarn/nodelabels/store/AbstractFSNodeStore.java | 6 +--
.../nodelabels/TestFileSystemNodeLabelsStore.java | 44 +++---
2 files changed, 41 insertions(+), 9 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java
index a697be19512..6e47b2afb08 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java
@@ -69,9 +69,9 @@ public abstract class AbstractFSNodeStore {
int maxRetries =
conf.getInt(YarnConfiguration.NODE_STORE_ROOT_DIR_NUM_RETRIES,
YarnConfiguration.NODE_STORE_ROOT_DIR_NUM_DEFAULT_RETRIES);
int retryCount = 0;
-boolean success = fs.mkdirs(fsWorkingPath);
+boolean success = false;
-while (!success && retryCount < maxRetries) {
+while (!success && retryCount <= maxRetries) {
try {
if (!fs.exists(fsWorkingPath)) {
success = fs.mkdirs(fsWorkingPath);
@@ -80,7 +80,7 @@ public abstract class AbstractFSNodeStore {
}
} catch (IOException e) {
retryCount++;
-if (retryCount >= maxRetries) {
+if (retryCount > maxRetries) {
throw e;
}
try {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
index a861b0654ea..517958367d8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
@@ -348,24 +348,56 @@ public class TestFileSystemNodeLabelsStore extends
NodeLabelTestBase {
@MethodSource("getParameters")
@ParameterizedTest
- void testRootMkdirOnInitStore(String className) throws Exception {
+ void testRootMkdirOnInitStoreWhenRootDirectoryAlreadyExists(String
className) throws Exception {
initTestFileSystemNodeLabelsStore(className);
final FileSystem mockFs = Mockito.mock(FileSystem.class);
+final FileSystemNodeLabelsStore mockStore =
createMockNodeLabelsStore(mockFs);
+final int expectedMkdirsCount = 0;
+
+Mockito.when(mockStore.getFs().exists(Mockito.any(Path.class)))
+.thenReturn(true);
+verifyMkdirsCount(mockStore, expectedMkdirsCount);
+ }
+
+ @MethodSource("getParameters")
+ @ParameterizedTest
+ void testRootMkdirOnInitStoreWhenRootDirectoryNotExists(String className)
throws Exception {
+initTestFileSystemNodeLabelsStore(className);
+final FileSystem mockFs = Mockito.mock(FileSystem.class);
+final FileSystemNodeLabelsStore mockStore =
createMockNodeLabelsStore(mockFs);
+final int expectedMkdirsCount = 1;
+
+Mockito.when(mockStore.getFs().exists(Mockito.any(Path.class)))
+.thenReturn(false).thenReturn(true);
+verifyMkdirsCount(mockStore, expectedMkdirsCount);
+ }
+
+ @MethodSource("getParameters")
+ @ParameterizedTest
+ void testRootMkdirOnInitStoreRetryLogic(String className) throws Exception {
+initTestFileSystemNodeLabelsStore(className);
+final FileSystem mockFs = Mockito.mock(FileSystem.class);
+final FileSystemNodeLabelsStore mockStore =
createMockNodeLabelsStore(mockFs);
+final int expectedMkdirsCount = 2;
+
+Mockito.when(mockStore.getFs().exists(Mockito.any(Path.class)))
+.thenReturn(false).thenReturn(false).thenReturn(true);
+verifyMkdirsCount(mockStore, expectedMkdirsCount);
+ }
+
+ private FileSystemNodeLabelsStore createMockNodeLabelsStore(FileSystem
mockFs) {
FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore() {
public void initFileSystem(Configuration config) throws IOException {
setFs(mockFs);
}
};
-
