This is an automated email from the ASF dual-hosted git repository. sunilg pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 9a2e43e YARN-9937. Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph. 9a2e43e is described below commit 9a2e43e29e14905d98edd8512d24363dec407634 Author: Sunil G <sun...@apache.org> AuthorDate: Thu Oct 31 00:34:27 2019 +0530 YARN-9937. Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph. --- .../webapp/JAXBContextResolver.java | 3 +- .../webapp/dao/CapacitySchedulerInfo.java | 11 +++-- .../webapp/dao/CapacitySchedulerLeafQueueInfo.java | 32 +++++++++---- .../webapp/dao/CapacitySchedulerQueueInfo.java | 55 +++++++++++++++++++++- .../resourcemanager/webapp/dao/QueueAclInfo.java | 48 +++++++++++++++++++ .../resourcemanager/webapp/dao/QueueAclsInfo.java | 47 ++++++++++++++++++ .../webapp/TestRMWebServicesCapacitySched.java | 4 +- 7 files changed, 181 insertions(+), 19 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index e0c6647..f6eb2ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -56,7 +56,8 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> { StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, AppTimeoutInfo.class, ResourceInformationsInfo.class, - ActivitiesInfo.class, AppActivitiesInfo.class}; + ActivitiesInfo.class, AppActivitiesInfo.class, + QueueAclsInfo.class, QueueAclInfo.class}; // these dao classes need root unwrapping final Class[] rootUnwrappedTypes = { NewApplication.class, ApplicationSubmissionContextInfo.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index ca42041..b8a5426 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -61,7 +61,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), parent.getQueueResourceQuotas(), false); - queues = getQueues(parent); + queues = getQueues(cs, parent); health = new CapacitySchedulerHealthInfo(cs); } @@ -89,7 +89,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo { return this.queues; } - protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) { + protected CapacitySchedulerQueueInfoList getQueues( + CapacityScheduler cs, CSQueue parent) { CapacitySchedulerQueueInfoList queuesInfo = new CapacitySchedulerQueueInfoList(); // JAXB marashalling leads to situation where the "type" field injected @@ -112,10 +113,10 @@ public class CapacitySchedulerInfo extends SchedulerInfo { for (CSQueue queue : childQueues) { CapacitySchedulerQueueInfo info; if (queue instanceof LeafQueue) { - info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue); + info = new CapacitySchedulerLeafQueueInfo(cs, (LeafQueue) queue); } else { - info = new CapacitySchedulerQueueInfo(queue); - info.queues = getQueues(queue); + info = new CapacitySchedulerQueueInfo(cs, queue); + info.queues = getQueues(cs, queue); } queuesInfo.addToQueueInfoList(info); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index a53e921..4920f76 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -22,13 +22,13 @@ import java.util.ArrayList; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .AutoCreatedLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; @@ -45,6 +45,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { protected int userLimit; protected UsersInfo users; // To add another level in the XML protected float userLimitFactor; + protected float configuredMaxAMResourceLimit; protected ResourceInfo AMResourceLimit; protected ResourceInfo usedAMResource; protected ResourceInfo userAMResourceLimit; @@ -53,15 +54,14 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { protected String defaultNodeLabelExpression; protected int defaultPriority; protected boolean isAutoCreatedLeafQueue; - - @XmlTransient - protected String orderingPolicyInfo; + protected long maxApplicationLifetime; + protected long defaultApplicationLifetime; CapacitySchedulerLeafQueueInfo() { }; - CapacitySchedulerLeafQueueInfo(LeafQueue q) { - super(q); + CapacitySchedulerLeafQueueInfo(CapacityScheduler cs, LeafQueue q) { + super(cs, q); numActiveApplications = q.getNumActiveApplications(); numPendingApplications = q.getNumPendingApplications(); numContainers = q.getNumContainers(); @@ -70,6 +70,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { userLimit = q.getUserLimit(); users = new UsersInfo(q.getUsersManager().getUsersInfo()); userLimitFactor = q.getUserLimitFactor(); + configuredMaxAMResourceLimit = q.getMaxAMResourcePerQueuePercent(); AMResourceLimit = new ResourceInfo(q.getAMResourceLimit()); usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed()); preemptionDisabled = q.getPreemptionDisabled(); @@ -91,6 +92,8 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { if ( q instanceof AutoCreatedLeafQueue) { isAutoCreatedLeafQueue = true; } + defaultApplicationLifetime = q.getDefaultApplicationLifetime(); + maxApplicationLifetime = q.getMaximumApplicationLifetime(); } @Override @@ -136,6 +139,10 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { public float getUserLimitFactor() { return userLimitFactor; } + + public float getConfiguredMaxAMResourceLimit() { + return configuredMaxAMResourceLimit; + } public ResourceInfo getAMResourceLimit() { return AMResourceLimit; @@ -157,10 +164,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { return intraQueuePreemptionDisabled; } - public String getOrderingPolicyInfo() { - return orderingPolicyInfo; - } - public String getDefaultNodeLabelExpression() { return defaultNodeLabelExpression; } @@ -172,4 +175,13 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { public boolean isAutoCreatedLeafQueue() { return isAutoCreatedLeafQueue; } + + public long getDefaultApplicationLifetime() { + return defaultApplicationLifetime; + } + + public long getMaxApplicationLifetime() { + return maxApplicationLifetime; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index fd256b1..bde1bb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import javax.xml.bind.annotation.XmlAccessType; @@ -27,10 +29,16 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; @@ -65,11 +73,15 @@ public class CapacitySchedulerQueueInfo { protected ResourcesInfo resources; protected ResourceInfo minEffectiveCapacity; protected ResourceInfo maxEffectiveCapacity; + protected ResourceInfo maximumAllocation; + protected QueueAclsInfo queueAcls; + protected int queuePriority; + protected String orderingPolicyInfo; CapacitySchedulerQueueInfo() { }; - CapacitySchedulerQueueInfo(CSQueue q) { + CapacitySchedulerQueueInfo(CapacityScheduler cs, CSQueue q) { queuePath = q.getQueuePath(); capacity = q.getCapacity() * 100; @@ -114,6 +126,31 @@ public class CapacitySchedulerQueueInfo { q.getQueueResourceQuotas().getEffectiveMinResource()); maxEffectiveCapacity = new ResourceInfo( q.getQueueResourceQuotas().getEffectiveMaxResource()); + maximumAllocation = new ResourceInfo(q.getMaximumAllocation()); + + CapacitySchedulerConfiguration conf = cs.getConfiguration(); + queueAcls = new QueueAclsInfo(); + for (Map.Entry<AccessType, AccessControlList> e : conf + .getAcls(queueName).entrySet()) { + QueueAclInfo queueAcl = new QueueAclInfo(e.getKey().toString(), + e.getValue().getAclString()); + queueAcls.add(queueAcl); + } + + String aclApplicationMaxPriority = "acl_" + + StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString()); + String priorityAcls = conf.get(queuePath + aclApplicationMaxPriority, + conf.ALL_ACL); + + QueueAclInfo queueAcl = new QueueAclInfo( + AccessType.APPLICATION_MAX_PRIORITY.toString(), priorityAcls); + queueAcls.add(queueAcl); + + queuePriority = q.getPriority().getPriority(); + if (q instanceof ParentQueue) { + orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() + .getConfigName(); + } } protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { @@ -220,6 +257,22 @@ public class CapacitySchedulerQueueInfo { return maxEffectiveCapacity; } + public ResourceInfo getMaximumAllocation() { + return maximumAllocation; + } + + public QueueAclsInfo getQueueAcls() { + return queueAcls; + } + + public int getPriority() { + return queuePriority; + } + + public String getOrderingPolicyInfo() { + return orderingPolicyInfo; + } + public boolean isLeafQueue() { return getQueues() == null; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java new file mode 100644 index 0000000..6e623ea --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "queueAcl") +@XmlAccessorType(XmlAccessType.FIELD) +public class QueueAclInfo { + + protected String accessType; + protected String accessControlList; + + public QueueAclInfo() { + // JAXB needs this + } + + public QueueAclInfo(String accessType, String accessControlList) { + this.accessType = accessType; + this.accessControlList = accessControlList; + } + + public String getAccessType() { + return accessType; + } + + public String getAccessControlList() { + return accessControlList; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java new file mode 100644 index 0000000..ea07dbf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "queueAcls") +@XmlAccessorType(XmlAccessType.FIELD) +public class QueueAclsInfo { + + protected ArrayList<QueueAclInfo> queueAcl = new ArrayList<QueueAclInfo>(); + + public QueueAclsInfo() { + } // JAXB needs this + + public void add(QueueAclInfo queueAclInfo) { + queueAcl.add(queueAclInfo); + } + + public ArrayList<QueueAclInfo> getQueueAcls() { + return queueAcl; + } + + public void addAll(ArrayList<QueueAclInfo> queueAclsInfo) { + queueAcl.addAll(queueAclsInfo); + } + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index d66130f..25be1eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -364,10 +364,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 20; + int numExpectedElements = 24; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 35; + numExpectedElements = 42; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org