[hadoop] branch trunk updated: YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new de526e1 YARN-10947. Simplify AbstractCSQueue#initializeQueueState.
Contributed by Andras Gyori
de526e1 is described below
commit de526e166c349e49d657c5980901670c086d36b2
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 21:05:01 2022 +0100
YARN-10947. Simplify AbstractCSQueue#initializeQueueState. Contributed by
Andras Gyori
---
.../yarn/client/api/impl/AHSv2ClientImpl.java | 4 +-
.../yarn/client/api/impl/TestAHSv2ClientImpl.java | 70 ++
2 files changed, 71 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
index e3653ba..5f90930 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
@@ -137,9 +137,7 @@ public class AHSv2ClientImpl extends AHSClient {
ApplicationId appId = applicationAttemptId.getApplicationId();
ApplicationReport appReport = getApplicationReport(appId);
Map filters = new HashMap<>();
-filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" +
-applicationAttemptId.toString() +
-"\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}");
+filters.put("appattemptId", applicationAttemptId.toString());
List entities = readerClient.getContainerEntities(
appId, "ALL", filters, 0, null);
List containers =
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
index f4d7a56..9664e40 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
@@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
@@ -45,8 +46,11 @@ import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -108,6 +112,72 @@ public class TestAHSv2ClientImpl {
}
@Test
+ public void testGetContainerByAppAttempt() throws IOException, YarnException
{
+int applicationId = 1;
+ApplicationId appId = ApplicationId.newInstance(0, applicationId);
+
+ApplicationAttemptId appAttemptId =
+ApplicationAttemptId.newInstance(appId, 1);
+
+ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
+
+when(spyTimelineReaderClient.getContainerEntities(
+appId, "ALL",
+ImmutableMap.of("appattemptId", appAttemptId.toString()),
+0, null))
+.thenReturn(Arrays.asList(createContainerEntity(containerId)));
+
+when(spyTimelineReaderClient.getApplicationEntity(appId, "ALL", null))
+.thenReturn(createApplicationTimelineEntity(appId, true,
+false));
+
+List containerList = client.getContainers(appAttemptId);
+
+assertThat(containerList.size()).isEqualTo(1);
+
+assertThat(containerList.get(0).getContainerId().getApplicationAttemptId()
+.getApplicationId().getId()).isEqualTo(applicationId);
+ }
+
+ @Test
+ public void testGetMultipleContainersByAppAttempt() throws IOException,
YarnException {
+int numContainers = 4;
+int applicationId = 3;
+int applicationAttemptId = 3;
+ApplicationId appMultiple = ApplicationId.newInstance(0, applicationId);
+ApplicationAttemptId appAttemptIdMultiple =
+ApplicationAttemptId.newInstance(appMultiple,
applicationAttemptId);
+
+List containerEntities = new ArrayList<>();
+for (int containerId = 0; containerId < numContainers; ++containerId) {
+ containerEntities.add(createContainerEntity(ContainerId.newContainerId(
+
[hadoop] branch trunk updated: YARN-11070. Minimum resource ratio is overridden by subsequent labels. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7dd288c YARN-11070. Minimum resource ratio is overridden by
subsequent labels. Contributed by Andras Gyori
7dd288c is described below
commit 7dd288ce8ccc93acd4f182e3db4b7936402f1592
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 19:58:14 2022 +0100
YARN-11070. Minimum resource ratio is overridden by subsequent labels.
Contributed by Andras Gyori
---
.../scheduler/capacity/AbstractCSQueue.java| 2 +-
.../scheduler/capacity/ParentQueue.java| 22
.../TestAbsoluteResourceConfiguration.java | 61 ++
3 files changed, 73 insertions(+), 12 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 809a860..4eb7b68 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -1298,7 +1298,7 @@ public abstract class AbstractCSQueue implements CSQueue {
CapacityConfigType.ABSOLUTE_RESOURCE)) {
newEffectiveMinResource = createNormalizedMinResource(
usageTracker.getQueueResourceQuotas().getConfiguredMinResource(label),
-((ParentQueue) parent).getEffectiveMinRatioPerResource());
+((ParentQueue) parent).getEffectiveMinRatio(label));
// Max resource of a queue should be the minimum of {parent's
maxResources,
// this queue's maxResources}. Both parent's maxResources and this
queue's
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index b77a90a..c624aab 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
@@ -101,11 +102,11 @@ public class ParentQueue extends AbstractCSQueue {
private AutoCreatedQueueTemplate autoCreatedQueueTemplate;
- // effective min ratio per resource, it is used during updateClusterResource,
- // leaf queue can use this to calculate effective resources.
- // This field will not be edited, reference will point to a new immutable map
- // after every time recalculation
- private volatile Map effectiveMinRatioPerResource;
+ // A ratio of the queue's effective minimum resource and the summary of the
configured
+ // minimum resource of its children grouped by labels and calculated for
each resource names
+ // distinctively.
+ private final Map> effectiveMinResourceRatio =
+ new ConcurrentHashMap<>();
public ParentQueue(CapacitySchedulerQueueContext queueContext,
String queueName, CSQueue parent, CSQueue old) throws IOException {
@@ -1328,8 +1329,8 @@ public class ParentQueue extends AbstractCSQueue {
}
}
-effectiveMinRatioPerResource = getEffectiveMinRatioPerResource(
-configuredMinResources, numeratorForMinRatio);
+effectiveMinResourceRatio.put(label, getEffectiveMinRatio(
+configuredMinResources, numeratorForMinRatio));
// Update effective resources for my self;
if (rootQueue) {
@@ -1340,7 +1341,7 @@ public class ParentQueue extends AbstractCSQueue {
}
}
- private Map getEffectiveMinRatioPerResource(
+ private Map getEffectiveMinRatio(
Resource configuredMinResources, Resource numeratorForMinRatio) {
Map effectiveMinRatioPerResource = new HashMap<>();
if
[hadoop] branch trunk updated: YARN-11075. Explicitly declare serialVersionUID in LogMutation class. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new ad0a1dc YARN-11075. Explicitly declare serialVersionUID in
LogMutation class. Contributed by Benjamin Teke
ad0a1dc is described below
commit ad0a1dc89723ed1630399203da614d6a338b1920
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 19:48:42 2022 +0100
YARN-11075. Explicitly declare serialVersionUID in LogMutation class.
Contributed by Benjamin Teke
---
.../resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 4480bc3..425d63f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore {
* audit logging and recovery.
*/
public static class LogMutation implements Serializable {
+private static final long serialVersionUID = 7754046036718906356L;
private Map updates;
private String user;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10995. Move PendingApplicationComparator from GuaranteedOrZeroCapacityOverTimePolicy. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4c05d25 YARN-10995. Move PendingApplicationComparator from
GuaranteedOrZeroCapacityOverTimePolicy. Contributed by Benjamin Teke
4c05d25 is described below
commit 4c05d257ba3f3311b5bbc993f6e5e35637487d88
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 19:43:37 2022 +0100
YARN-10995. Move PendingApplicationComparator from
GuaranteedOrZeroCapacityOverTimePolicy. Contributed by Benjamin Teke
---
.../scheduler/capacity/CapacityScheduler.java | 31 +
.../capacity/CapacitySchedulerContext.java | 3 +-
.../capacity/CapacitySchedulerQueueContext.java| 7 ++---
.../GuaranteedOrZeroCapacityOverTimePolicy.java| 32 +-
4 files changed, 35 insertions(+), 38 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 16c18bc..44e80a6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -21,6 +21,7 @@ package
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -201,6 +202,9 @@ public class CapacityScheduler extends
private int threadNum = 0;
+ private final PendingApplicationComparator applicationComparator =
+ new PendingApplicationComparator();
+
@Override
public void setConf(Configuration conf) {
yarnConf = conf;
@@ -3555,4 +3559,31 @@ public class CapacityScheduler extends
return asyncSchedulerThreads;
}
}
+
+ @Override
+ public PendingApplicationComparator getPendingApplicationComparator(){
+return applicationComparator;
+ }
+
+ /**
+ * Comparator that orders applications by their submit time.
+ */
+ class PendingApplicationComparator
+ implements Comparator {
+
+@Override
+public int compare(FiCaSchedulerApp app1, FiCaSchedulerApp app2) {
+ RMApp rmApp1 = rmContext.getRMApps().get(app1.getApplicationId());
+ RMApp rmApp2 = rmContext.getRMApps().get(app2.getApplicationId());
+ if (rmApp1 != null && rmApp2 != null) {
+return Long.compare(rmApp1.getSubmitTime(), rmApp2.getSubmitTime());
+ } else if (rmApp1 != null) {
+return -1;
+ } else if (rmApp2 != null) {
+return 1;
+ } else{
+return 0;
+ }
+}
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
index 1d0600f..5795a7e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import java.util.Comparator;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -104,4 +102,5 @@ public interface CapacitySchedulerContext {
*/
Clock getClock();
+ CapacityScheduler.PendingApplicationComparator
getPendingApplicationComparator();
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java
[hadoop] branch trunk updated: YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 51001fc YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok 51001fc is described below commit 51001fc7bf3bf2b83aaa5f4d2b4af0397ad5dc4c Author: Szilard Nemeth AuthorDate: Thu Feb 17 19:38:18 2022 +0100 YARN-11022. Fix the documentation for max-parallel-apps in CS. Contributed by Tamas Domok --- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index a945e14..9173cb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -151,7 +151,7 @@ Configuration |: |: | | `yarn.scheduler.capacity.maximum-applications` / `yarn.scheduler.capacity..maximum-applications` | Maximum number of applications in the system which can be concurrently active both running and pending. Limits on each queue are directly proportional to their queue capacities and user limits. This is a hard limit and any applications submitted when this limit is reached will be rejected. Default is 1. This can be set for all queues with `yarn.scheduler.capacity.maximum-a [...] | `yarn.scheduler.capacity.maximum-am-resource-percent` / `yarn.scheduler.capacity..maximum-am-resource-percent` | Maximum percent of resources in the cluster which can be used to run application masters - controls number of concurrent active applications. Limits on each queue are directly proportional to their queue capacities and user limits. Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with `yarn.scheduler.capacity.maximum-am-resource [...] -| `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity..max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.sch [...] +| `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity..max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.sch [...] You can also limit the number of parallel applications on a per user basis. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10580. Fix some issues in TestRMWebServicesCapacitySchedDynamicConfig. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 48e02f7 YARN-10580. Fix some issues in
TestRMWebServicesCapacitySchedDynamicConfig. Contributed by Tamas Domok
48e02f7 is described below
commit 48e02f79983b32b64d1ae10e8ddf162c06f700d6
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 19:17:31 2022 +0100
YARN-10580. Fix some issues in TestRMWebServicesCapacitySchedDynamicConfig.
Contributed by Tamas Domok
---
.../TestRMWebServicesCapacitySchedDynamicConfig.java | 16
1 file changed, 4 insertions(+), 12 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
index df4f18e..0e2ecd0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
@@ -102,8 +102,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
throws Exception {
Configuration config = CSConfigGenerator
.createPercentageConfig();
-config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-YarnConfiguration.MEMORY_CONFIGURATION_STORE);
initResourceManager(config);
@@ -121,8 +119,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
throws Exception {
Configuration config = CSConfigGenerator
.createPercentageConfigLegacyAutoCreation();
-config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-YarnConfiguration.MEMORY_CONFIGURATION_STORE);
initResourceManager(config);
@@ -141,8 +137,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
throws Exception {
Configuration config = CSConfigGenerator
.createAbsoluteConfigLegacyAutoCreation();
-config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-YarnConfiguration.MEMORY_CONFIGURATION_STORE);
initResourceManager(config);
initAutoQueueHandler(8192 * GB);
@@ -157,8 +151,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
throws Exception {
Configuration config = CSConfigGenerator
.createAbsoluteConfig();
-config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-YarnConfiguration.MEMORY_CONFIGURATION_STORE);
initResourceManager(config);
@@ -176,8 +168,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
throws Exception {
Configuration config = CSConfigGenerator
.createWeightConfig();
-config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-YarnConfiguration.MEMORY_CONFIGURATION_STORE);
initResourceManager(config);
@@ -197,8 +187,6 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
throws Exception {
Configuration config = CSConfigGenerator
.createWeightConfigWithAutoQueueCreationEnabled();
-config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-YarnConfiguration.MEMORY_CONFIGURATION_STORE);
config.setInt(CapacitySchedulerConfiguration
.getQueuePrefix("root.autoParent1") +
AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX +
@@ -332,6 +320,10 @@ public class TestRMWebServicesCapacitySchedDynamicConfig
extends
for (Map.Entry entry : configs.entrySet()) {
config.set(entry.getKey(), entry.getValue());
}
+
+ config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
+
return config;
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10894. Follow up YARN-10237: fix the new test case in TestRMWebServicesCapacitySched. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new bb1135c YARN-10894. Follow up YARN-10237: fix the new test case in
TestRMWebServicesCapacitySched. Contributed by Tamas Domok
bb1135c is described below
commit bb1135c77c34ca5dcd8d43fca31c635b65bf3638
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 19:12:51 2022 +0100
YARN-10894. Follow up YARN-10237: fix the new test case in
TestRMWebServicesCapacitySched. Contributed by Tamas Domok
---
.../webapp/TestRMWebServicesCapacitySched.java | 2 +-
.../TestRMWebServicesSchedulerActivities.java | 4 +-
.../scheduler-response-NodeLabelDefaultAPI.xml | 491 -
.../scheduler-response-PerUserResources.json | 487 +++-
.../webapp/scheduler-response-PerUserResources.xml | 490 +++-
.../test/resources/webapp/scheduler-response.json | 487 +++-
.../test/resources/webapp/scheduler-response.xml | 490 +++-
7 files changed, 2443 insertions(+), 8 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 1f73870..2be9b54 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -120,7 +120,7 @@ public class TestRMWebServicesCapacitySched extends
JerseyTestBase {
// Define top-level queues
config.setQueues(CapacitySchedulerConfiguration.ROOT,
-new String[] {"a", "b"});
+new String[] {"a", "b", "c"});
final String a = CapacitySchedulerConfiguration.ROOT + ".a";
config.setCapacity(a, 10.5f);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
index f864794..3fc3c26 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
@@ -156,7 +156,7 @@ public class TestRMWebServicesSchedulerActivities
verifyStateOfAllocations(allocation,
FN_ACT_FINAL_ALLOCATION_STATE, "ALLOCATED");
verifyQueueOrder(allocation,
- "root-root.a-root.b-root.b.b2-root.b.b3-root.b.b1");
+ "root-root.a-root.c-root.b-root.b.b2-root.b.b3-root.b.b1");
} finally {
rm.stop();
}
@@ -380,7 +380,7 @@ public class TestRMWebServicesSchedulerActivities
JSONObject allocations = getFirstSubNodeFromJson(json,
FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS);
verifyQueueOrder(allocations,
- "root-root.a-root.b-root.b.b3-root.b.b1");
+ "root-root.c-root.a-root.b-root.b.b3-root.b.b1");
verifyStateOfAllocations(allocations, FN_ACT_FINAL_ALLOCATION_STATE,
"RESERVED");
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml
index 48b6893..d49491f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml
@@ -10,6 +10,495 @@
2147483647
false
+
+root.c
+0.0
+0.0
+100.0
+0.0
+0.0
+0.0
+-1.0
+0.0
+0
+2147483647
+c
+true
+
[hadoop] branch trunk updated: YARN-11071. AutoCreatedQueueTemplate incorrect wildcard level. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d58c10c YARN-11071. AutoCreatedQueueTemplate incorrect wildcard
level. Contributed by Tamas Domok
d58c10c is described below
commit d58c10c3a5f0d6c923d8a49457ce5a73a0ec7011
Author: Szilard Nemeth
AuthorDate: Thu Feb 17 19:03:44 2022 +0100
YARN-11071. AutoCreatedQueueTemplate incorrect wildcard level. Contributed
by Tamas Domok
---
.../scheduler/capacity/AutoCreatedQueueTemplate.java | 3 +--
.../scheduler/capacity/TestAutoCreatedQueueTemplate.java | 11 +++
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
index 1603b19..8fa1e13 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
@@ -170,8 +170,7 @@ public class AutoCreatedQueueTemplate {
int wildcardLevel = 0;
// root can not be wildcarded
// MAX_WILDCARD_LEVEL will be configurable in the future
-int supportedWildcardLevel = Math.min(queuePathMaxIndex - 1,
-MAX_WILDCARD_LEVEL);
+int supportedWildcardLevel = Math.min(queuePathMaxIndex,
MAX_WILDCARD_LEVEL);
// Allow root to have template properties
if (queuePath.isRoot()) {
supportedWildcardLevel = 0;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
index 5b58feb..e73001b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
@@ -89,6 +89,17 @@ public class TestAutoCreatedQueueTemplate {
}
@Test
+ public void testWildcardAfterRoot() {
+conf.set(getTemplateKey("root.*", "acl_submit_applications"), "user");
+AutoCreatedQueueTemplate template =
+new AutoCreatedQueueTemplate(conf, new QueuePath("root.a"));
+template.setTemplateEntriesForChild(conf, "root.a");
+
+Assert.assertEquals("acl_submit_applications is set", "user",
+template.getTemplateProperties().get("acl_submit_applications"));
+ }
+
+ @Test
public void testTemplatePrecedence() {
conf.set(getTemplateKey("root.a.b", "capacity"), "6w");
conf.set(getTemplateKey("root.a.*", "capacity"), "4w");
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
