This is an automated email from the ASF dual-hosted git repository. inigoiri pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 9e16f1f883a YARN-11317. [Federation] Refactoring Yarn Router's About Web Page. (#4946) 9e16f1f883a is described below commit 9e16f1f883a053addb786f9642518cf9ecbda068 Author: slfan1989 <55643692+slfan1...@users.noreply.github.com> AuthorDate: Wed Oct 12 04:30:48 2022 +0800 YARN-11317. [Federation] Refactoring Yarn Router's About Web Page. (#4946) --- .../apache/hadoop/yarn/webapp/view/HtmlBlock.java | 24 ++ .../server/resourcemanager/webapp/RMWSConsts.java | 3 + .../resourcemanager/webapp/RMWebServices.java | 11 + .../webapp/dao/SchedulerOverviewInfo.java | 118 ++++++++ .../resourcemanager/webapp/TestRMWebServices.java | 57 ++++ .../webapp/TestRMWebServicesCapacitySched.java | 13 + .../TestRMWebServicesFairScheduler.java | 12 + .../apache/hadoop/yarn/server/router/Router.java | 5 + .../yarn/server/router/webapp/AboutBlock.java | 95 +++---- .../yarn/server/router/webapp/FederationBlock.java | 40 +-- .../server/router/webapp/MetricsOverviewTable.java | 239 ++++++++++++++++ .../yarn/server/router/webapp/RouterBlock.java | 99 +++++++ .../router/webapp/dao/RouterClusterMetrics.java | 310 +++++++++++++++++++++ .../yarn/server/router/webapp/dao/RouterInfo.java | 104 +++++++ .../router/webapp/dao/RouterSchedulerMetrics.java | 109 ++++++++ .../server/router/webapp/dao/package-info.java | 20 ++ .../server/router/webapp/TestFederationWebApp.java | 20 +- 17 files changed, 1191 insertions(+), 88 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java index 6c14b7532f5..c07a76617ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java @@ -97,4 +97,28 @@ public abstract class HtmlBlock extends TextView implements SubView { return callerUGI; } + /** + * Initialize User Help Information Div. + * When the user does not configure the Yarn Federation function, prompt the user. + * + * @param html HTML page. + * @param isEnabled If federation is enabled. + */ + protected void initUserHelpInformationDiv(Block html, boolean isEnabled) { + if (!isEnabled) { + html.style(".alert {padding: 15px; margin-bottom: 20px; " + + " border: 1px solid transparent; border-radius: 4px;}"); + html.style(".alert-dismissable {padding-right: 35px;}"); + html.style(".alert-info {color: #856404;background-color: #fff3cd;border-color: #ffeeba;}"); + + Hamlet.DIV<Hamlet> div = html.div("#div_id").$class("alert alert-dismissable alert-info"); + div.p().$style("color:red").__("Federation is not Enabled.").__() + .p().__() + .p().__("We can refer to the following documents to configure Yarn Federation. ").__() + .p().__() + .a("https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/Federation.html", + "Hadoop: YARN Federation"). + __(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java index 791e34ad3d1..5c7787ce023 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java @@ -57,6 +57,9 @@ public final class RMWSConsts { /** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */ public static final String SCHEDULER_LOGS = "/scheduler/logs"; + /** Path for {@code RMWebServiceProtocol#getSchedulerOverview}. */ + public static final String SCHEDULER_OVERVIEW = "/scheduler-overview"; + /** * Path for {@code RMWebServiceProtocol#validateAndGetSchedulerConfiguration}. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 6a6a2e1de1e..0803af5e76d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -202,6 +202,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesIn import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.webapp.WebServices; @@ -2942,4 +2943,14 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { } return Response.status(Status.OK).build(); } + + @GET + @Path(RMWSConsts.SCHEDULER_OVERVIEW) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public SchedulerOverviewInfo getSchedulerOverview() { + initForReadableEndpoints(); + ResourceScheduler rs = rm.getResourceScheduler(); + return new SchedulerOverviewInfo(rs); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerOverviewInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerOverviewInfo.java new file mode 100644 index 00000000000..06df27dc4c4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerOverviewInfo.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; + +@XmlRootElement(name = "scheduler") +@XmlAccessorType(XmlAccessType.FIELD) +public class SchedulerOverviewInfo { + + private String schedulerType; + private String schedulingResourceType; + private ResourceInfo minimumAllocation; + private ResourceInfo maximumAllocation; + private int applicationPriority; + private int schedulerBusy; + private int rmDispatcherEventQueueSize; + private int schedulerDispatcherEventQueueSize; + + // JAXB needs this + public SchedulerOverviewInfo() { + + } + + public SchedulerOverviewInfo(ResourceScheduler rs) { + // Parse the schedule type + this.schedulerType = getSchedulerName(rs); + + // Parse and allocate resource information + this.minimumAllocation = new ResourceInfo(rs.getMinimumResourceCapability()); + this.maximumAllocation = new ResourceInfo(rs.getMaximumResourceCapability()); + + // Parse App Priority + this.applicationPriority = rs.getMaxClusterLevelAppPriority().getPriority(); + + // Resolving resource types + List<ResourceTypeInfo> resourceTypeInfos = ResourceUtils.getResourcesTypeInfo(); + resourceTypeInfos.sort((o1, o2) -> o1.getName().compareToIgnoreCase(o2.getName())); + this.schedulingResourceType = StringUtils.join(resourceTypeInfos, ","); + + // clusterMetrics + ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(rs); + this.schedulerBusy = clusterMetrics.getRmSchedulerBusyPercent(); + this.rmDispatcherEventQueueSize = clusterMetrics.getRmEventQueueSize(); + this.schedulerDispatcherEventQueueSize = clusterMetrics.getSchedulerEventQueueSize(); + } + + private static String getSchedulerName(ResourceScheduler rs) { + if (rs instanceof CapacityScheduler) { + return "Capacity Scheduler"; + } + if (rs instanceof FairScheduler) { + return "Fair Scheduler"; + } + if (rs instanceof FifoScheduler) { + return "Fifo Scheduler"; + } + return rs.getClass().getSimpleName(); + } + + public String getSchedulerType() { + return schedulerType; + } + + public String getSchedulingResourceType() { + return schedulingResourceType; + } + + public ResourceInfo getMinimumAllocation() { + return minimumAllocation; + } + + public ResourceInfo getMaximumAllocation() { + return maximumAllocation; + } + + public int getApplicationPriority() { + return applicationPriority; + } + + public int getSchedulerBusy() { + return schedulerBusy; + } + + public int getRmDispatcherEventQueueSize() { + return rmDispatcherEventQueueSize; + } + + public int getSchedulerDispatcherEventQueueSize() { + return schedulerDispatcherEventQueueSize; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index ce9de643744..9cf0dda7c39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -1099,4 +1099,61 @@ public class TestRMWebServices extends JerseyTestBase { return webService; } + @Test + public void testClusterSchedulerOverviewFifo() throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler-overview").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject json = response.getEntity(JSONObject.class); + verifyClusterSchedulerOverView(json, "Fifo Scheduler"); + } + + public static void verifyClusterSchedulerOverView( + JSONObject json, String expectedSchedulerType) throws Exception { + + // why json contains 8 elements because we defined 8 fields + assertEquals("incorrect number of elements in: " + json, 8, json.length()); + + // 1.Verify that the schedulerType is as expected + String schedulerType = json.getString("schedulerType"); + assertEquals(expectedSchedulerType, schedulerType); + + // 2.Verify that schedulingResourceType is as expected + String schedulingResourceType = json.getString("schedulingResourceType"); + assertEquals("memory-mb (unit=Mi),vcores", schedulingResourceType); + + // 3.Verify that minimumAllocation is as expected + JSONObject minimumAllocation = json.getJSONObject("minimumAllocation"); + String minMemory = minimumAllocation.getString("memory"); + String minVCores = minimumAllocation.getString("vCores"); + assertEquals("1024", minMemory); + assertEquals("1", minVCores); + + // 4.Verify that maximumAllocation is as expected + JSONObject maximumAllocation = json.getJSONObject("maximumAllocation"); + String maxMemory = maximumAllocation.getString("memory"); + String maxVCores = maximumAllocation.getString("vCores"); + assertEquals("8192", maxMemory); + assertEquals("4", maxVCores); + + // 5.Verify that schedulerBusy is as expected + int schedulerBusy = json.getInt("schedulerBusy"); + assertEquals(-1, schedulerBusy); + + // 6.Verify that rmDispatcherEventQueueSize is as expected + int rmDispatcherEventQueueSize = json.getInt("rmDispatcherEventQueueSize"); + assertEquals(0, rmDispatcherEventQueueSize); + + // 7.Verify that schedulerDispatcherEventQueueSize is as expected + int schedulerDispatcherEventQueueSize = json.getInt("schedulerDispatcherEventQueueSize"); + assertEquals(0, schedulerDispatcherEventQueueSize); + + // 8.Verify that applicationPriority is as expected + int applicationPriority = json.getInt("applicationPriority"); + assertEquals(0, applicationPriority); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 258947af4ef..4454442bf22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Guice; import com.google.inject.servlet.ServletModule; import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.test.framework.WebAppDescriptor; @@ -413,4 +414,16 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); return new MockRM(conf); } + + @Test + public void testClusterSchedulerOverviewCapacity() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler-overview").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject json = response.getEntity(JSONObject.class); + TestRMWebServices.verifyClusterSchedulerOverView(json, "Capacity Scheduler"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java index bf605e9f5f6..cbc6c417859 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServices; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; @@ -157,4 +158,15 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase { assertEquals("root", rootQueue.getString("queueName")); } + @Test + public void testClusterSchedulerOverviewFair() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("scheduler-overview").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject json = response.getEntity(JSONObject.class); + TestRMWebServices.verifyClusterSchedulerOverView(json, "Fair Scheduler"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java index 82a1e1ea6f9..24e9ad23c93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java @@ -79,6 +79,7 @@ public class Router extends CompositeService { private WebApp webApp; @VisibleForTesting protected String webAppAddress; + private static long clusterTimeStamp = System.currentTimeMillis(); /** * Priority of the Router shutdown hook. @@ -237,4 +238,8 @@ public class Router extends CompositeService { } return name; } + + public static long getClusterTimeStamp() { + return clusterTimeStamp; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java index a8a6e6bbac9..878ac75d1c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java @@ -18,15 +18,11 @@ package org.apache.hadoop.yarn.server.router.webapp; -import com.sun.jersey.api.client.Client; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.router.Router; -import org.apache.hadoop.yarn.webapp.util.WebAppUtils; -import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.hadoop.yarn.server.router.webapp.dao.RouterInfo; import org.apache.hadoop.yarn.webapp.view.InfoBlock; import com.google.inject.Inject; @@ -34,62 +30,57 @@ import com.google.inject.Inject; /** * About block for the Router Web UI. */ -public class AboutBlock extends HtmlBlock { - - private static final long BYTES_IN_MB = 1024 * 1024; +public class AboutBlock extends RouterBlock { private final Router router; @Inject AboutBlock(Router router, ViewContext ctx) { - super(ctx); + super(router, ctx); this.router = router; } @Override protected void render(Block html) { - Configuration conf = this.router.getConfig(); - String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); - Client client = RouterWebServiceUtil.createJerseyClient(conf); - ClusterMetricsInfo metrics = RouterWebServiceUtil - .genericForward(webAppAddress, null, ClusterMetricsInfo.class, - HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null, - conf, client); - boolean isEnabled = conf.getBoolean( - YarnConfiguration.FEDERATION_ENABLED, - YarnConfiguration.DEFAULT_FEDERATION_ENABLED); - info("Cluster Status"). - __("Federation Enabled", isEnabled). - __("Applications Submitted", metrics.getAppsSubmitted()). - __("Applications Pending", metrics.getAppsPending()). - __("Applications Running", metrics.getAppsRunning()). - __("Applications Failed", metrics.getAppsFailed()). - __("Applications Killed", metrics.getAppsKilled()). - __("Applications Completed", metrics.getAppsCompleted()). - __("Containers Allocated", metrics.getContainersAllocated()). - __("Containers Reserved", metrics.getReservedContainers()). - __("Containers Pending", metrics.getPendingContainers()). - __("Available Memory", - StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB)). - __("Allocated Memory", - StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB)). - __("Reserved Memory", - StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB)). - __("Total Memory", - StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB)). - __("Available VirtualCores", metrics.getAvailableVirtualCores()). - __("Allocated VirtualCores", metrics.getAllocatedVirtualCores()). - __("Reserved VirtualCores", metrics.getReservedVirtualCores()). - __("Total VirtualCores", metrics.getTotalVirtualCores()). - __("Active Nodes", metrics.getActiveNodes()). - __("Lost Nodes", metrics.getLostNodes()). - __("Available Nodes", metrics.getDecommissionedNodes()). - __("Unhealthy Nodes", metrics.getUnhealthyNodes()). - __("Rebooted Nodes", metrics.getRebootedNodes()). - __("Total Nodes", metrics.getTotalNodes()); + boolean isEnabled = isYarnFederationEnabled(); + + // If Yarn Federation is not enabled, the user needs to be prompted. + initUserHelpInformationDiv(html, isEnabled); + + // Metrics Overview Table + html.__(MetricsOverviewTable.class); + // Init Yarn Router Basic Information + initYarnRouterBasicInformation(isEnabled); + + // InfoBlock html.__(InfoBlock.class); } + + /** + * Init Yarn Router Basic Infomation. + * @param isEnabled true, federation is enabled; false, federation is not enabled. + */ + private void initYarnRouterBasicInformation(boolean isEnabled) { + FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + RouterInfo routerInfo = new RouterInfo(router); + String lastStartTime = + DateFormatUtils.format(routerInfo.getStartedOn(), DATE_PATTERN); + try { + info("Yarn Router Overview"). + __("Federation Enabled:", String.valueOf(isEnabled)). + __("Router ID:", routerInfo.getClusterId()). + __("Router state:", routerInfo.getState()). + __("Router SubCluster Count:", facade.getSubClusters(true).size()). + __("Router RMStateStore:", routerInfo.getRouterStateStore()). + __("Router started on:", lastStartTime). + __("Router version:", routerInfo.getRouterBuildVersion() + + " on " + routerInfo.getRouterVersionBuiltOn()). + __("Hadoop version:", routerInfo.getHadoopBuildVersion() + + " on " + routerInfo.getHadoopVersionBuiltOn()); + } catch (YarnException e) { + LOG.error("initYarnRouterBasicInformation error.", e); + } + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java index 70976d6221d..9e449a46e29 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java @@ -20,50 +20,41 @@ package org.apache.hadoop.yarn.server.router.webapp; import java.io.StringReader; import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.HashMap; import com.google.gson.Gson; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.commons.lang3.time.DateFormatUtils; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; -import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.router.Router; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; -import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import com.google.inject.Inject; import com.sun.jersey.api.json.JSONConfiguration; import com.sun.jersey.api.json.JSONJAXBContext; import com.sun.jersey.api.json.JSONUnmarshaller; -class FederationBlock extends HtmlBlock { +class FederationBlock extends RouterBlock { private final Router router; @Inject FederationBlock(ViewContext ctx, Router router) { - super(ctx); + super(router, ctx); this.router = router; } @Override public void render(Block html) { - Configuration conf = this.router.getConfig(); - boolean isEnabled = conf.getBoolean( - YarnConfiguration.FEDERATION_ENABLED, - YarnConfiguration.DEFAULT_FEDERATION_ENABLED); + boolean isEnabled = isYarnFederationEnabled(); // init Html Page Federation initHtmlPageFederation(html, isEnabled); @@ -122,21 +113,7 @@ class FederationBlock extends HtmlBlock { List<Map<String, String>> lists = new ArrayList<>(); // If Yarn Federation is not enabled, the user needs to be prompted. - if (!isEnabled) { - html.style(".alert {padding: 15px; margin-bottom: 20px; " + - " border: 1px solid transparent; border-radius: 4px;}"); - html.style(".alert-dismissable {padding-right: 35px;}"); - html.style(".alert-info {color: #856404;background-color: #fff3cd;border-color: #ffeeba;}"); - - Hamlet.DIV<Hamlet> div = html.div("#div_id").$class("alert alert-dismissable alert-info"); - div.p().$style("color:red").__("Federation is not Enabled.").__() - .p().__() - .p().__("We can refer to the following documents to configure Yarn Federation. ").__() - .p().__() - .a("https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/Federation.html", - "Hadoop: YARN Federation"). - __(); - } + initUserHelpInformationDiv(html, isEnabled); // Table header TBODY<TABLE<Hamlet>> tbody = @@ -150,16 +127,9 @@ class FederationBlock extends HtmlBlock { .__().__().tbody(); try { - // Binding to the FederationStateStore - FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); - - Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true); // Sort the SubClusters - List<SubClusterInfo> subclusters = new ArrayList<>(); - subclusters.addAll(subClustersInfo.values()); - Comparator<? super SubClusterInfo> cmp = Comparator.comparing(o -> o.getSubClusterId()); - Collections.sort(subclusters, cmp); + List<SubClusterInfo> subclusters = getSubClusterInfoList(); for (SubClusterInfo subcluster : subclusters) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java new file mode 100644 index 00000000000..ba17fa27ff4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.webapp; + +import com.google.inject.Inject; +import com.sun.jersey.api.client.Client; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo; +import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.server.router.webapp.dao.RouterClusterMetrics; +import org.apache.hadoop.yarn.server.router.webapp.dao.RouterSchedulerMetrics; +import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; + +import java.io.IOException; +import java.util.List; + +public class MetricsOverviewTable extends RouterBlock { + + private final Router router; + + @Inject + MetricsOverviewTable(Router router, ViewContext ctx) { + super(router, ctx); + this.router = router; + } + + @Override + protected void render(Block html) { + // Initialize page styles + html.style(".metrics {margin-bottom:5px}"); + + // get routerClusterMetrics Info + ClusterMetricsInfo routerClusterMetricsInfo = getRouterClusterMetricsInfo(); + RouterClusterMetrics routerClusterMetrics = new RouterClusterMetrics(routerClusterMetricsInfo); + + // metrics div + Hamlet.DIV<Hamlet> div = html.div().$class("metrics"); + try { + initFederationClusterAppsMetrics(div, routerClusterMetrics); + initFederationClusterNodesMetrics(div, routerClusterMetrics); + initFederationClusterSchedulersMetrics(div, routerClusterMetrics); + } catch (Exception e) { + LOG.error("MetricsOverviewTable init error.", e); + } + div.__(); + } + + /** + * Init Federation Cluster Apps Metrics. + * Contains App information, resource usage information. + * + * @param div data display div. + * @param metrics data metric information. + */ + private void initFederationClusterAppsMetrics(Hamlet.DIV<Hamlet> div, + RouterClusterMetrics metrics) { + div.h3("Federation Cluster Metrics"). + table("#metricsoverview"). + thead().$class("ui-widget-header"). + // Initialize table header information + tr(). + th().$class("ui-state-default").__("Apps Submitted").__(). + th().$class("ui-state-default").__("Apps Pending").__(). + th().$class("ui-state-default").__("Apps Running").__(). + th().$class("ui-state-default").__("Apps Completed").__(). + th().$class("ui-state-default").__("Containers Running").__(). + th().$class("ui-state-default").__("Used Resources").__(). + th().$class("ui-state-default").__("Total Resources").__(). + th().$class("ui-state-default").__("Reserved Resources").__(). + th().$class("ui-state-default").__("Physical Mem Used %").__(). + th().$class("ui-state-default").__("Physical VCores Used %").__(). + __(). + __(). + // Initialize table data information + tbody().$class("ui-widget-content"). + tr(). + td(metrics.getAppsSubmitted()). + td(metrics.getAppsPending()). + td(String.valueOf(metrics.getAppsRunning())). + td(metrics.getAppsCompleted()). + td(metrics.getAllocatedContainers()). + td(metrics.getUsedResources()). + td(metrics.getTotalResources()). + td(metrics.getReservedResources()). + td(metrics.getUtilizedMBPercent()). + td(metrics.getUtilizedVirtualCoresPercent()). + __(). + __().__(); + } + + /** + * Init Federation Cluster Nodes Metrics. + * + * @param div data display div. + * @param metrics data metric information. + */ + private void initFederationClusterNodesMetrics(Hamlet.DIV<Hamlet> div, + RouterClusterMetrics metrics) { + div.h3("Federation Cluster Nodes Metrics"). + table("#nodemetricsoverview"). + thead().$class("ui-widget-header"). + // Initialize table header information + tr(). + th().$class("ui-state-default").__("Active Nodes").__(). + th().$class("ui-state-default").__("Decommissioning Nodes").__(). + th().$class("ui-state-default").__("Decommissioned Nodes").__(). + th().$class("ui-state-default").__("Lost Nodes").__(). + th().$class("ui-state-default").__("Unhealthy Nodes").__(). + th().$class("ui-state-default").__("Rebooted Nodes").__(). + th().$class("ui-state-default").__("Shutdown Nodes").__(). + __(). + __(). + // Initialize table data information + tbody().$class("ui-widget-content"). + tr(). + td(String.valueOf(metrics.getActiveNodes())). + td(String.valueOf(metrics.getDecommissioningNodes())). + td(String.valueOf(metrics.getDecommissionedNodes())). + td(String.valueOf(metrics.getLostNodes())). + td(String.valueOf(metrics.getUnhealthyNodes())). + td(String.valueOf(metrics.getRebootedNodes())). + td(String.valueOf(metrics.getShutdownNodes())). + __(). + __().__(); + } + + /** + * Init Federation Cluster SchedulersMetrics. + * + * @param div data display div. + * @param metrics data metric information. + * @throws YarnException yarn error. + * @throws IOException io error. + * @throws InterruptedException interrupt error. + */ + private void initFederationClusterSchedulersMetrics(Hamlet.DIV<Hamlet> div, + RouterClusterMetrics metrics) throws YarnException, IOException, InterruptedException { + // Sort the SubClusters. + List<SubClusterInfo> subclusters = getSubClusterInfoList(); + + Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr = + div.h3("Federation Scheduler Metrics"). + table("#schedulermetricsoverview"). + thead().$class("ui-widget-header"). + tr(). + th().$class("ui-state-default").__("SubCluster").__(). + th().$class("ui-state-default").__("Scheduler Type").__(). + th().$class("ui-state-default").__("Scheduling Resource Type").__(). + th().$class("ui-state-default").__("Minimum Allocation").__(). + th().$class("ui-state-default").__("Maximum Allocation").__(). + th().$class("ui-state-default").__("Maximum Cluster Application Priority").__(). + th().$class("ui-state-default").__("Scheduler Busy %").__(). + th().$class("ui-state-default").__("RM Dispatcher EventQueue Size").__(). + th().$class("ui-state-default") + .__("Scheduler Dispatcher EventQueue Size").__(). + __(). + __(). + tbody().$class("ui-widget-content"); + + boolean isEnabled = isYarnFederationEnabled(); + + // If Federation mode is not enabled or there is currently no SubCluster available, + // each column in the list should be displayed as N/A + if (!isEnabled || subclusters == null || subclusters.isEmpty()) { + fsMetricsScheduleTr.tr(). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE). + td(UNAVAILABLE) + .__(); + } else { + initSubClusterOverViewTable(metrics, fsMetricsScheduleTr, subclusters); + } + + fsMetricsScheduleTr.__().__(); + } + + private void initSubClusterOverViewTable(RouterClusterMetrics metrics, + Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr, + List<SubClusterInfo> subclusters) { + + // configuration + Configuration config = this.router.getConfig(); + + Client client = RouterWebServiceUtil.createJerseyClient(config); + + // Traverse all SubClusters to get cluster information. + for (SubClusterInfo subcluster : subclusters) { + + // Call the RM interface to obtain schedule information + String webAppAddress = WebAppUtils.getHttpSchemePrefix(config) + + subcluster.getRMWebServiceAddress(); + + SchedulerOverviewInfo typeInfo = RouterWebServiceUtil + .genericForward(webAppAddress, null, SchedulerOverviewInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_OVERVIEW, null, null, + config, client); + RouterSchedulerMetrics rsMetrics = new RouterSchedulerMetrics(subcluster, metrics, typeInfo); + + // Basic information + fsMetricsScheduleTr.tr(). + td(rsMetrics.getSubCluster()). + td(rsMetrics.getSchedulerType()). + td(rsMetrics.getSchedulingResourceType()). + td(rsMetrics.getMinimumAllocation()). + td(rsMetrics.getMaximumAllocation()). + td(rsMetrics.getApplicationPriority()). + td(rsMetrics.getSchedulerBusy()). + td(rsMetrics.getRmDispatcherEventQueueSize()). + td(rsMetrics.getSchedulerDispatcherEventQueueSize()). + __(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java new file mode 100644 index 00000000000..de5d62edf1f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.webapp; + +import com.sun.jersey.api.client.Client; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; + +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.Collections; +import java.util.Comparator; + +public abstract class RouterBlock extends HtmlBlock { + + private final Router router; + + public RouterBlock(Router router, ViewContext ctx) { + super(ctx); + this.router = router; + } + + /** + * Get RouterClusterMetrics Info. + * + * @return Router ClusterMetricsInfo. + */ + protected ClusterMetricsInfo getRouterClusterMetricsInfo() { + Configuration conf = this.router.getConfig(); + boolean isEnabled = isYarnFederationEnabled(); + if(isEnabled) { + String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); + Client client = RouterWebServiceUtil.createJerseyClient(conf); + ClusterMetricsInfo metrics = RouterWebServiceUtil + .genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null, + conf, client); + return metrics; + } + return null; + } + + /** + * Get a list of subclusters. + * + * @return subcluster List. + * @throws YarnException if the call to the getSubClusters is unsuccessful. + */ + protected List<SubClusterInfo> getSubClusterInfoList() throws YarnException { + FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true); + + // Sort the SubClusters. + List<SubClusterInfo> subclusters = new ArrayList<>(); + subclusters.addAll(subClustersInfo.values()); + Comparator<? super SubClusterInfo> cmp = Comparator.comparing(o -> o.getSubClusterId()); + Collections.sort(subclusters, cmp); + + return subclusters; + } + + /** + * Whether Yarn Federation is enabled. + * + * @return true, enable yarn federation; false, not enable yarn federation; + */ + protected boolean isYarnFederationEnabled() { + Configuration conf = this.router.getConfig(); + boolean isEnabled = conf.getBoolean( + YarnConfiguration.FEDERATION_ENABLED, + YarnConfiguration.DEFAULT_FEDERATION_ENABLED); + return isEnabled; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterClusterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterClusterMetrics.java new file mode 100644 index 00000000000..46e9e89ac18 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterClusterMetrics.java @@ -0,0 +1,310 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.webapp.dao; + +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RouterClusterMetrics { + + protected static final long BYTES_IN_MB = 1024 * 1024; + private static final Logger LOG = LoggerFactory.getLogger(RouterClusterMetrics.class); + + // Application Information. + private String appsSubmitted = "N/A"; + private String appsCompleted = "N/A"; + private String appsPending = "N/A"; + private String appsRunning = "N/A"; + private String appsFailed = "N/A"; + private String appsKilled = "N/A"; + + // Memory Information. + private String totalMemory = "N/A"; + private String reservedMemory = "N/A"; + private String availableMemory = "N/A"; + private String allocatedMemory = "N/A"; + private String pendingMemory = "N/A"; + + // VirtualCores Information. + private String reservedVirtualCores = "N/A"; + private String availableVirtualCores = "N/A"; + private String allocatedVirtualCores = "N/A"; + private String pendingVirtualCores = "N/A"; + private String totalVirtualCores = "N/A"; + + // Resources Information. + private String usedResources = "N/A"; + private String totalResources = "N/A"; + private String reservedResources = "N/A"; + private String allocatedContainers = "N/A"; + + // Resource Percent Information. + private String utilizedMBPercent = "N/A"; + private String utilizedVirtualCoresPercent = "N/A"; + + // Node Information. + private String activeNodes = "N/A"; + private String decommissioningNodes = "N/A"; + private String decommissionedNodes = "N/A"; + private String lostNodes = "N/A"; + private String unhealthyNodes = "N/A"; + private String rebootedNodes = "N/A"; + private String shutdownNodes = "N/A"; + + public RouterClusterMetrics() { + + } + + public RouterClusterMetrics(ClusterMetricsInfo metrics) { + if (metrics != null) { + // Application Information Conversion. + conversionApplicationInformation(metrics); + + // Memory Information Conversion. + conversionMemoryInformation(metrics); + + // Resources Information Conversion. + conversionResourcesInformation(metrics); + + // Percent Information Conversion. + conversionResourcesPercent(metrics); + + // Node Information Conversion. + conversionNodeInformation(metrics); + } + } + + // Get Key Metric Information + public String getAppsSubmitted() { + return appsSubmitted; + } + + public String getAppsCompleted() { + return appsCompleted; + } + + public String getAppsPending() { + return appsPending; + } + + public String getAppsRunning() { + return appsRunning; + } + + public String getAppsFailed() { + return appsFailed; + } + + public String getAppsKilled() { + return appsKilled; + } + + public String getTotalMemory() { + return totalMemory; + } + + public String getReservedMemory() { + return reservedMemory; + } + + public String getAvailableMemory() { + return availableMemory; + } + + public String getAllocatedMemory() { + return allocatedMemory; + } + + public String getPendingMemory() { + return pendingMemory; + } + + public String getReservedVirtualCores() { + return reservedVirtualCores; + } + + public String getAvailableVirtualCores() { + return availableVirtualCores; + } + + public String getAllocatedVirtualCores() { + return allocatedVirtualCores; + } + + public String getPendingVirtualCores() { + return pendingVirtualCores; + } + + public String getTotalVirtualCores() { + return totalVirtualCores; + } + + public String getUsedResources() { + return usedResources; + } + + public String getTotalResources() { + return totalResources; + } + + public String getReservedResources() { + return reservedResources; + } + + public String getAllocatedContainers() { + return allocatedContainers; + } + + public String getUtilizedMBPercent() { + return utilizedMBPercent; + } + + public String getUtilizedVirtualCoresPercent() { + return utilizedVirtualCoresPercent; + } + + public String getActiveNodes() { + return activeNodes; + } + + public String getDecommissioningNodes() { + return decommissioningNodes; + } + + public String getDecommissionedNodes() { + return decommissionedNodes; + } + + public String getLostNodes() { + return lostNodes; + } + + public String getUnhealthyNodes() { + return unhealthyNodes; + } + + public String getRebootedNodes() { + return rebootedNodes; + } + + public String getShutdownNodes() { + return shutdownNodes; + } + + // Metric Information Conversion + public void conversionApplicationInformation(ClusterMetricsInfo metrics) { + try { + // Application Information. + this.appsSubmitted = String.valueOf(metrics.getAppsSubmitted()); + this.appsCompleted = String.valueOf(metrics.getAppsCompleted() + + metrics.getAppsFailed() + metrics.getAppsKilled()); + this.appsPending = String.valueOf(metrics.getAppsPending()); + this.appsRunning = String.valueOf(metrics.getAppsRunning()); + this.appsFailed = String.valueOf(metrics.getAppsFailed()); + this.appsKilled = String.valueOf(metrics.getAppsKilled()); + } catch (Exception e) { + LOG.error("conversionApplicationInformation error.", e); + } + } + + // Metric Memory Information + public void conversionMemoryInformation(ClusterMetricsInfo metrics) { + try { + // Memory Information. + this.totalMemory = StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB); + this.reservedMemory = StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB); + this.availableMemory = StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB); + this.allocatedMemory = StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB); + this.pendingMemory = StringUtils.byteDesc(metrics.getPendingMB() * BYTES_IN_MB); + } catch (Exception e) { + LOG.error("conversionMemoryInformation error.", e); + } + } + + // ResourcesInformation Conversion + public void conversionResourcesInformation(ClusterMetricsInfo metrics) { + try { + // Parse resource information from metrics. + Resource metricUsedResources; + Resource metricTotalResources; + Resource metricReservedResources; + + int metricAllocatedContainers; + if (metrics.getCrossPartitionMetricsAvailable()) { + metricAllocatedContainers = metrics.getTotalAllocatedContainersAcrossPartition(); + metricUsedResources = metrics.getTotalUsedResourcesAcrossPartition().getResource(); + metricTotalResources = metrics.getTotalClusterResourcesAcrossPartition().getResource(); + metricReservedResources = metrics.getTotalReservedResourcesAcrossPartition().getResource(); + // getTotalUsedResourcesAcrossPartition includes reserved resources. + Resources.subtractFrom(metricUsedResources, metricReservedResources); + } else { + metricAllocatedContainers = metrics.getContainersAllocated(); + metricUsedResources = Resource.newInstance(metrics.getAllocatedMB(), + (int) metrics.getAllocatedVirtualCores()); + metricTotalResources = Resource.newInstance(metrics.getTotalMB(), + (int) metrics.getTotalVirtualCores()); + metricReservedResources = Resource.newInstance(metrics.getReservedMB(), + (int) metrics.getReservedVirtualCores()); + } + + // Convert to standard format. + usedResources = metricUsedResources.getFormattedString(); + totalResources = metricTotalResources.getFormattedString(); + reservedResources = metricReservedResources.getFormattedString(); + allocatedContainers = String.valueOf(metricAllocatedContainers); + + } catch (Exception e) { + LOG.error("conversionResourcesInformation error.", e); + } + } + + // ResourcesPercent Conversion + public void conversionResourcesPercent(ClusterMetricsInfo metrics) { + try { + this.utilizedMBPercent = String.valueOf(metrics.getUtilizedMBPercent()); + this.utilizedVirtualCoresPercent = String.valueOf(metrics.getUtilizedVirtualCoresPercent()); + } catch (Exception e) { + LOG.error("conversionResourcesPercent error.", e); + } + } + + // NodeInformation Conversion + public void conversionNodeInformation(ClusterMetricsInfo metrics) { + try { + this.activeNodes = String.valueOf(metrics.getActiveNodes()); + this.decommissioningNodes = String.valueOf(metrics.getDecommissioningNodes()); + this.decommissionedNodes = String.valueOf(metrics.getDecommissionedNodes()); + this.lostNodes = String.valueOf(metrics.getLostNodes()); + this.unhealthyNodes = String.valueOf(metrics.getUnhealthyNodes()); + this.rebootedNodes = String.valueOf(metrics.getRebootedNodes()); + this.shutdownNodes = String.valueOf(metrics.getShutdownNodes()); + } catch (Exception e) { + LOG.error("conversionNodeInformation error.", e); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterInfo.java new file mode 100644 index 00000000000..7cedd0c8e1f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterInfo.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.webapp.dao; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.util.YarnVersionInfo; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RouterInfo { + private long id; + private long startedOn; + private Service.STATE state; + private String routerStateStoreName; + private String routerVersion; + private String routerBuildVersion; + private String routerVersionBuiltOn; + private String hadoopVersion; + private String hadoopBuildVersion; + private String hadoopVersionBuiltOn; + + public RouterInfo() { + } // JAXB needs this + + public RouterInfo(Router router) { + long ts = Router.getClusterTimeStamp(); + this.id = ts; + this.state = router.getServiceState(); + Configuration configuration = router.getConfig(); + this.routerStateStoreName = configuration.get( + YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS, + YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS); + this.routerVersion = YarnVersionInfo.getVersion(); + this.routerBuildVersion = YarnVersionInfo.getBuildVersion(); + this.routerVersionBuiltOn = YarnVersionInfo.getDate(); + this.hadoopVersion = VersionInfo.getVersion(); + this.hadoopBuildVersion = VersionInfo.getBuildVersion(); + this.hadoopVersionBuiltOn = VersionInfo.getDate(); + this.startedOn = ts; + } + + public String getState() { + return this.state.toString(); + } + + public String getRouterStateStore() { + return this.routerStateStoreName; + } + + public String getRouterVersion() { + return this.routerVersion; + } + + public String getRouterBuildVersion() { + return this.routerBuildVersion; + } + + public String getRouterVersionBuiltOn() { + return this.routerVersionBuiltOn; + } + + public String getHadoopVersion() { + return this.hadoopVersion; + } + + public String getHadoopBuildVersion() { + return this.hadoopBuildVersion; + } + + public String getHadoopVersionBuiltOn() { + return this.hadoopVersionBuiltOn; + } + + public long getClusterId() { + return this.id; + } + + public long getStartedOn() { + return this.startedOn; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java new file mode 100644 index 00000000000..4a3af1ba43d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.webapp.dao; + + +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RouterSchedulerMetrics { + + // Metrics Log. + private static final Logger LOG = LoggerFactory.getLogger(RouterSchedulerMetrics.class); + + // Scheduler Information. + private String subCluster = "N/A"; + private String schedulerType = "N/A"; + private String schedulingResourceType = "N/A"; + private String minimumAllocation = "N/A"; + private String maximumAllocation = "N/A"; + private String applicationPriority = "N/A"; + private String schedulerBusy = "N/A"; + private String rmDispatcherEventQueueSize = "N/A"; + private String schedulerDispatcherEventQueueSize = "N/A"; + + public RouterSchedulerMetrics() { + + } + + public RouterSchedulerMetrics(SubClusterInfo subClusterInfo, RouterClusterMetrics metrics, + SchedulerOverviewInfo overview) { + try { + // Parse Scheduler Information. + this.subCluster = subClusterInfo.getSubClusterId().getId(); + this.schedulerType = overview.getSchedulerType(); + this.schedulingResourceType = overview.getSchedulingResourceType(); + this.minimumAllocation = overview.getMinimumAllocation().toString(); + this.maximumAllocation = overview.getMaximumAllocation().toString(); + this.applicationPriority = String.valueOf(overview.getApplicationPriority()); + if (overview.getSchedulerBusy() != -1) { + this.schedulerBusy = String.valueOf(overview.getSchedulerBusy()); + } + this.rmDispatcherEventQueueSize = + String.valueOf(overview.getRmDispatcherEventQueueSize()); + this.schedulerDispatcherEventQueueSize = + String.valueOf(overview.getSchedulerDispatcherEventQueueSize()); + } catch (Exception ex) { + LOG.error("RouterSchedulerMetrics Error.", ex); + } + } + + public String getSubCluster() { + return subCluster; + } + + public String getSchedulerType() { + return schedulerType; + } + + public String getSchedulingResourceType() { + return schedulingResourceType; + } + + public String getMinimumAllocation() { + return minimumAllocation; + } + + public String getMaximumAllocation() { + return maximumAllocation; + } + + public String getApplicationPriority() { + return applicationPriority; + } + + public String getRmDispatcherEventQueueSize() { + return rmDispatcherEventQueueSize; + } + + public String getSchedulerDispatcherEventQueueSize() { + return schedulerDispatcherEventQueueSize; + } + + public String getSchedulerBusy() { + return schedulerBusy; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/package-info.java new file mode 100644 index 00000000000..27f43ad1ff4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Router Web Dao package. **/ +package org.apache.hadoop.yarn.server.router.webapp.dao; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java index e0ea4ccdb39..55f23db72b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java @@ -26,7 +26,7 @@ import org.junit.Test; import java.io.IOException; -public class TestFederationWebApp { +public class TestFederationWebApp extends TestRouterWebServicesREST { @Test public void testFederationWebViewNotEnable() @@ -45,4 +45,22 @@ public class TestFederationWebApp { config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); WebAppTests.testPage(FederationPage.class, Router.class, new MockRouter(config)); } + + @Test + public void testFederationAboutViewEnable() + throws InterruptedException, YarnException, IOException { + // Test Federation Enabled + Configuration config = new YarnConfiguration(); + config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config)); + } + + @Test + public void testFederationAboutViewNotEnable() + throws InterruptedException, YarnException, IOException { + // Test Federation Not Enabled + Configuration config = new YarnConfiguration(); + config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false); + WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config)); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org