Author: vinodkv
Date: Tue Sep 17 00:25:14 2013
New Revision: 1523857
URL: http://svn.apache.org/r1523857
Log:
YARN-1001. Added a web-service to get statistics about per application-type per
state for consumption by downstream projects. Contributed by Zhijie Shen.
svn merge --ignore-ancestry -c 1523855 ../../trunk/
Added:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationStatisticsInfo.java
- copied unchanged from r1523855,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationStatisticsInfo.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/StatisticsItemInfo.java
- copied unchanged from r1523855,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/StatisticsItemInfo.java
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt?rev=1523857&r1=1523856&r2=1523857&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt Tue
Sep 17 00:25:14 2013
@@ -66,6 +66,9 @@ Release 2.1.1-beta - UNRELEASED
YARN-1137. Add support whitelist for system users to Yarn
container-executor.c. (rvs via tucu)
+ YARN-1001. Added a web-service to get statistics about per application-type
+ per state for consumption by downstream projects. (Zhijie Shen via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java?rev=1523857&r1=1523856&r2=1523857&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
Tue Sep 17 00:25:14 2013
@@ -34,12 +34,14 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
+import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
@@ -65,7 +67,8 @@ public class JAXBContextResolver impleme
CapacitySchedulerInfo.class, ClusterMetricsInfo.class,
SchedulerInfo.class, AppsInfo.class, NodesInfo.class,
RemoteExceptionData.class, CapacitySchedulerQueueInfoList.class,
- ResourceInfo.class, UsersInfo.class, UserInfo.class};
+ ResourceInfo.class, UsersInfo.class, UserInfo.class,
+ ApplicationStatisticsInfo.class, StatisticsItemInfo.class};
public JAXBContextResolver() throws Exception {
this.types = new HashSet<Class>(Arrays.asList(cTypes));
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1523857&r1=1523856&r2=1523857&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
Tue Sep 17 00:25:14 2013
@@ -22,7 +22,9 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
@@ -42,12 +44,12 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -58,6 +60,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
@@ -68,6 +71,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
+import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
@@ -80,6 +84,7 @@ import com.google.inject.Singleton;
@Path("/ws/v1/cluster")
public class RMWebServices {
private static final String EMPTY = "";
+ private static final String ANY = "*";
private final ResourceManager rm;
private static RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
@@ -303,53 +308,16 @@ public class RMWebServices {
"finishTimeEnd must be greater than finishTimeBegin");
}
- Set<String> appTypes = new HashSet<String>();
- if (!applicationTypes.isEmpty()) {
- for (String applicationType : applicationTypes) {
- if (applicationType != null && !applicationType.trim().isEmpty()) {
- if (applicationType.indexOf(",") == -1) {
- appTypes.add(applicationType.trim());
- } else {
- String[] types = applicationType.split(",");
- for (String type : types) {
- if (!type.trim().isEmpty()) {
- appTypes.add(type.trim());
- }
- }
- }
- }
- }
- }
+ Set<String> appTypes = parseQueries(applicationTypes, false);
if (!appTypes.isEmpty()) {
checkAppTypes = true;
}
- String allAppStates;
- RMAppState[] stateArray = RMAppState.values();
- allAppStates = Arrays.toString(stateArray);
-
- Set<String> appStates = new HashSet<String>();
// stateQuery is deprecated.
if (stateQuery != null && !stateQuery.isEmpty()) {
statesQuery.add(stateQuery);
}
- if (!statesQuery.isEmpty()) {
- for (String applicationState : statesQuery) {
- if (applicationState != null && !applicationState.isEmpty()) {
- String[] states = applicationState.split(",");
- for (String state : states) {
- try {
- RMAppState.valueOf(state.trim());
- } catch (IllegalArgumentException iae) {
- throw new BadRequestException(
- "Invalid application-state " + state
- + " specified. It should be one of " + allAppStates);
- }
- appStates.add(state.trim().toLowerCase());
- }
- }
- }
- }
+ Set<String> appStates = parseQueries(statesQuery, true);
if (!appStates.isEmpty()) {
checkAppStates = true;
}
@@ -363,8 +331,8 @@ public class RMWebServices {
break;
}
- if (checkAppStates
- && !appStates.contains(rmapp.getState().toString().toLowerCase())) {
+ if (checkAppStates && !appStates.contains(
+ rmapp.createApplicationState().toString().toLowerCase())) {
continue;
}
if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) {
@@ -394,8 +362,8 @@ public class RMWebServices {
continue;
}
}
- if (checkAppTypes
- && !appTypes.contains(rmapp.getApplicationType())) {
+ if (checkAppTypes && !appTypes.contains(
+ rmapp.getApplicationType().trim().toLowerCase())) {
continue;
}
@@ -416,6 +384,122 @@ public class RMWebServices {
}
@GET
+ @Path("/appstatistics")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public ApplicationStatisticsInfo getAppStatistics(
+ @Context HttpServletRequest hsr,
+ @QueryParam("states") Set<String> stateQueries,
+ @QueryParam("applicationTypes") Set<String> typeQueries) {
+ init();
+
+ // parse the params and build the scoreboard
+ // converting state/type name to lowercase
+ Set<String> states = parseQueries(stateQueries, true);
+ Set<String> types = parseQueries(typeQueries, false);
+ // if no types, counts the applications of any types
+ if (types.size() == 0) {
+ types.add(ANY);
+ } else if (types.size() != 1) {
+ throw new BadRequestException("# of applicationTypes = " + types.size()
+ + ", we temporarily support at most one applicationType");
+ }
+ // if no states, returns the counts of all RMAppStates
+ if (states.size() == 0) {
+ for (YarnApplicationState state : YarnApplicationState.values()) {
+ states.add(state.toString().toLowerCase());
+ }
+ }
+ // in case we extend to multiple applicationTypes in the future
+ Map<YarnApplicationState, Map<String, Long>> scoreboard =
+ buildScoreboard(states, types);
+
+ // go through the apps in RM to count the numbers, ignoring the case of
+ // the state/type name
+ ConcurrentMap<ApplicationId, RMApp> apps = rm.getRMContext().getRMApps();
+ for (RMApp rmapp : apps.values()) {
+ YarnApplicationState state = rmapp.createApplicationState();
+ String type = rmapp.getApplicationType().trim().toLowerCase();
+ if (states.contains(state.toString().toLowerCase())) {
+ if (types.contains(ANY)) {
+ countApp(scoreboard, state, ANY);
+ } else if (types.contains(type)) {
+ countApp(scoreboard, state, type);
+ }
+ }
+ }
+
+ // fill the response object
+ ApplicationStatisticsInfo appStatInfo = new ApplicationStatisticsInfo();
+ for (Map.Entry<YarnApplicationState, Map<String, Long>> partScoreboard
+ : scoreboard.entrySet()) {
+ for (Map.Entry<String, Long> statEntry
+ : partScoreboard.getValue().entrySet()) {
+ StatisticsItemInfo statItem = new StatisticsItemInfo(
+ partScoreboard.getKey(), statEntry.getKey(), statEntry.getValue());
+ appStatInfo.add(statItem);
+ }
+ }
+ return appStatInfo;
+ }
+
+ private static Set<String> parseQueries(
+ Set<String> queries, boolean isState) {
+ Set<String> params = new HashSet<String>();
+ if (!queries.isEmpty()) {
+ for (String query : queries) {
+ if (query != null && !query.trim().isEmpty()) {
+ String[] paramStrs = query.split(",");
+ for (String paramStr : paramStrs) {
+ if (paramStr != null && !paramStr.trim().isEmpty()) {
+ if (isState) {
+ try {
+ // enum string is in the uppercase
+ YarnApplicationState.valueOf(paramStr.trim().toUpperCase());
+ } catch (RuntimeException e) {
+ YarnApplicationState[] stateArray =
+ YarnApplicationState.values();
+ String allAppStates = Arrays.toString(stateArray);
+ throw new BadRequestException(
+ "Invalid application-state " + paramStr.trim()
+ + " specified. It should be one of " + allAppStates);
+ }
+ }
+ params.add(paramStr.trim().toLowerCase());
+ }
+ }
+ }
+ }
+ }
+ return params;
+ }
+
+ private static Map<YarnApplicationState, Map<String, Long>> buildScoreboard(
+ Set<String> states, Set<String> types) {
+ Map<YarnApplicationState, Map<String, Long>> scoreboard
+ = new HashMap<YarnApplicationState, Map<String, Long>>();
+ // default states will result in enumerating all YarnApplicationStates
+ assert !states.isEmpty();
+ for (String state : states) {
+ Map<String, Long> partScoreboard = new HashMap<String, Long>();
+ scoreboard.put(
+ YarnApplicationState.valueOf(state.toUpperCase()), partScoreboard);
+ // types is verified no to be empty
+ for (String type : types) {
+ partScoreboard.put(type, 0L);
+ }
+ }
+ return scoreboard;
+ }
+
+ private static void countApp(
+ Map<YarnApplicationState, Map<String, Long>> scoreboard,
+ YarnApplicationState state, String type) {
+ Map<String, Long> partScoreboard = scoreboard.get(state);
+ Long count = partScoreboard.get(type);
+ partScoreboard.put(type, count + 1L);
+ }
+
+ @GET
@Path("/apps/{appid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public AppInfo getApp(@Context HttpServletRequest hsr,
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1523857&r1=1523856&r2=1523857&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
Tue Sep 17 00:25:14 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -227,7 +228,8 @@ public class TestRMWebServicesApps exten
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("apps").queryParam("state", RMAppState.ACCEPTED.toString())
+ .path("apps")
+ .queryParam("state", YarnApplicationState.ACCEPTED.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
@@ -252,7 +254,7 @@ public class TestRMWebServicesApps exten
WebResource r = resource();
MultivaluedMapImpl params = new MultivaluedMapImpl();
- params.add("states", RMAppState.ACCEPTED.toString());
+ params.add("states", YarnApplicationState.ACCEPTED.toString());
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
@@ -268,8 +270,8 @@ public class TestRMWebServicesApps exten
r = resource();
params = new MultivaluedMapImpl();
- params.add("states", RMAppState.ACCEPTED.toString());
- params.add("states", RMAppState.KILLED.toString());
+ params.add("states", YarnApplicationState.ACCEPTED.toString());
+ params.add("states", YarnApplicationState.KILLED.toString());
response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
@@ -301,7 +303,7 @@ public class TestRMWebServicesApps exten
WebResource r = resource();
MultivaluedMapImpl params = new MultivaluedMapImpl();
- params.add("states", RMAppState.ACCEPTED.toString());
+ params.add("states", YarnApplicationState.ACCEPTED.toString());
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
@@ -317,8 +319,8 @@ public class TestRMWebServicesApps exten
r = resource();
params = new MultivaluedMapImpl();
- params.add("states", RMAppState.ACCEPTED.toString() + ","
- + RMAppState.KILLED.toString());
+ params.add("states", YarnApplicationState.ACCEPTED.toString() + ","
+ + YarnApplicationState.KILLED.toString());
response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
@@ -347,7 +349,8 @@ public class TestRMWebServicesApps exten
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("apps").queryParam("states", RMAppState.RUNNING.toString())
+ .path("apps")
+ .queryParam("states", YarnApplicationState.RUNNING.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
@@ -365,7 +368,8 @@ public class TestRMWebServicesApps exten
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("apps").queryParam("state", RMAppState.RUNNING.toString())
+ .path("apps")
+ .queryParam("state", YarnApplicationState.RUNNING.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
@@ -976,6 +980,169 @@ public class TestRMWebServicesApps exten
}
@Test
+ public void testAppStatistics() throws JSONException, Exception {
+ try {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 4096);
+ Thread.sleep(1);
+ RMApp app1 = rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+ .getShortUserName(), null, false, null, 2, null, "MAPREDUCE");
+ amNodeManager.nodeHeartbeat(true);
+ // finish App
+ MockAM am = rm
+ .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId());
+ am.registerAppAttempt();
+ am.unregisterAppAttempt();
+
amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
+ 1, ContainerState.COMPLETE);
+
+ rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+ .getShortUserName(), null, false, null, 2, null, "MAPREDUCE");
+ rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+ .getShortUserName(), null, false, null, 2, null, "OTHER");
+
+ // zero type, zero state
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("appstatistics")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject appsStatInfo = json.getJSONObject("appStatInfo");
+ assertEquals("incorrect number of elements", 1, appsStatInfo.length());
+ JSONArray statItems = appsStatInfo.getJSONArray("statItem");
+ assertEquals("incorrect number of elements",
+ YarnApplicationState.values().length, statItems.length());
+ for (int i = 0; i < YarnApplicationState.values().length; ++i) {
+ assertEquals("*", statItems.getJSONObject(0).getString("type"));
+ if (statItems.getJSONObject(0).getString("state").equals("ACCEPTED")) {
+ assertEquals("2", statItems.getJSONObject(0).getString("count"));
+ } else if (
+ statItems.getJSONObject(0).getString("state").equals("FINISHED")) {
+ assertEquals("1", statItems.getJSONObject(0).getString("count"));
+ } else {
+ assertEquals("0", statItems.getJSONObject(0).getString("count"));
+ }
+ }
+
+ // zero type, one state
+ r = resource();
+ response = r.path("ws").path("v1").path("cluster")
+ .path("appstatistics")
+ .queryParam("states", YarnApplicationState.ACCEPTED.toString())
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ appsStatInfo = json.getJSONObject("appStatInfo");
+ assertEquals("incorrect number of elements", 1, appsStatInfo.length());
+ statItems = appsStatInfo.getJSONArray("statItem");
+ assertEquals("incorrect number of elements", 1, statItems.length());
+ assertEquals("ACCEPTED", statItems.getJSONObject(0).getString("state"));
+ assertEquals("*", statItems.getJSONObject(0).getString("type"));
+ assertEquals("2", statItems.getJSONObject(0).getString("count"));
+
+ // one type, zero state
+ r = resource();
+ response = r.path("ws").path("v1").path("cluster")
+ .path("appstatistics")
+ .queryParam("applicationTypes", "MAPREDUCE")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ appsStatInfo = json.getJSONObject("appStatInfo");
+ assertEquals("incorrect number of elements", 1, appsStatInfo.length());
+ statItems = appsStatInfo.getJSONArray("statItem");
+ assertEquals("incorrect number of elements",
+ YarnApplicationState.values().length, statItems.length());
+ for (int i = 0; i < YarnApplicationState.values().length; ++i) {
+ assertEquals("mapreduce",
statItems.getJSONObject(0).getString("type"));
+ if (statItems.getJSONObject(0).getString("state").equals("ACCEPTED")) {
+ assertEquals("1", statItems.getJSONObject(0).getString("count"));
+ } else if (
+ statItems.getJSONObject(0).getString("state").equals("FINISHED")) {
+ assertEquals("1", statItems.getJSONObject(0).getString("count"));
+ } else {
+ assertEquals("0", statItems.getJSONObject(0).getString("count"));
+ }
+ }
+
+ // two types, zero state
+ r = resource();
+ response = r.path("ws").path("v1").path("cluster")
+ .path("appstatistics")
+ .queryParam("applicationTypes", "MAPREDUCE,OTHER")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject exception = json.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String className = exception.getString("javaClassName");
+ WebServicesTestUtils.checkStringContains("exception message",
+ "we temporarily support at most one applicationType", message);
+ WebServicesTestUtils.checkStringEqual("exception type",
+ "BadRequestException", type);
+ WebServicesTestUtils.checkStringEqual("exception className",
+ "org.apache.hadoop.yarn.webapp.BadRequestException", className);
+
+ // one type, two states
+ r = resource();
+ response = r.path("ws").path("v1").path("cluster")
+ .path("appstatistics")
+ .queryParam("states", YarnApplicationState.FINISHED.toString()
+ + "," + YarnApplicationState.ACCEPTED.toString())
+ .queryParam("applicationTypes", "MAPREDUCE")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ appsStatInfo = json.getJSONObject("appStatInfo");
+ assertEquals("incorrect number of elements", 1, appsStatInfo.length());
+ statItems = appsStatInfo.getJSONArray("statItem");
+ assertEquals("incorrect number of elements", 2, statItems.length());
+ JSONObject statItem1 = statItems.getJSONObject(0);
+ JSONObject statItem2 = statItems.getJSONObject(1);
+ assertTrue((statItem1.getString("state").equals("ACCEPTED") &&
+ statItem2.getString("state").equals("FINISHED")) ||
+ (statItem2.getString("state").equals("ACCEPTED") &&
+ statItem1.getString("state").equals("FINISHED")));
+ assertEquals("mapreduce", statItem1.getString("type"));
+ assertEquals("1", statItem1.getString("count"));
+ assertEquals("mapreduce", statItem2.getString("type"));
+ assertEquals("1", statItem2.getString("count"));
+
+ // invalid state
+ r = resource();
+ response = r.path("ws").path("v1").path("cluster")
+ .path("appstatistics").queryParam("states", "wrong_state")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ exception = json.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ message = exception.getString("message");
+ type = exception.getString("exception");
+ className = exception.getString("javaClassName");
+ WebServicesTestUtils.checkStringContains("exception message",
+ "Invalid application-state wrong_state", message);
+ WebServicesTestUtils.checkStringEqual("exception type",
+ "BadRequestException", type);
+ WebServicesTestUtils.checkStringEqual("exception className",
+ "org.apache.hadoop.yarn.webapp.BadRequestException", className);
+ } finally {
+ rm.stop();
+ }
+ }
+
+ @Test
public void testSingleApp() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1523857&r1=1523856&r2=1523857&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Tue Sep 17 00:25:14 2013
@@ -1269,6 +1269,129 @@ _01_000001</amContainerLogs>
+---+
+* Cluster Application Statistics API
+
+ With the Application Statistics API, you can obtain a collection of triples,
each of which contains the application type, the application state and the
number of applications of this type and this state in ResourceManager context.
Note that with the performance concern, we currently only support at most one
applicationType per query. We may support multiple applicationTypes per query
as well as more statistics in the future. When you run a GET operation on this
resource, you obtain a collection of statItem objects.
+
+** URI
+
+------
+ * http://<rm http address:port>/ws/v1/cluster/appstatistics
+------
+
+** HTTP Operations Supported
+
+------
+ * GET
+------
+
+** Query Parameters Required
+
+ Two paramters can be specified. The parameters are case insensitive.
+
+------
+ * states - states of the applications, specified as a comma-separated list.
If states is not provided, the API will enumerate all application states and
return the counts of them.
+ * applicationTypes - types of the applications, specified as a
comma-separated list. If applicationTypes is not provided, the API will count
the applications of any application type. In this case, the response shows * to
indicate any application type. Note that we only support at most one
applicationType temporarily. Otherwise, users will expect an
BadRequestException.
+------
+
+** Elements of the <appStatInfo> (statItems) object
+
+ When you make a request for the list of statistics items, the information
will be returned as a collection of statItem objects
+
+*-----------+----------------------------------------------------------------------+-------------------------------------+
+|| Item || Data Type
|| Description |
+*-----------+----------------------------------------------------------------------+-------------------------------------+
+| statItem | array of statItem objects(JSON)/zero or more statItem
objects(XML) | The collection of statItem objects |
+*-----------+----------------------------------------------------------------------+-------------------------------------+
+
+** Response Examples
+
+ <<JSON response>>
+
+ HTTP Request:
+
+------
+ GET http://<rm http
address:port>/ws/v1/cluster/appstatistics?states=accepted,running,finished&applicationTypes=mapreduce
+------
+
+ Response Header:
+
++---+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+{
+ "appStatInfo":
+ {
+ "statItem":
+ [
+ {
+ "state" : "accepted",
+ "type" : "mapreduce",
+ "count" : 4
+ },
+ {
+ "state" : "running",
+ "type" : "mapreduce",
+ "count" : 1
+ },
+ {
+ "state" : "finished",
+ "type" : "mapreduce",
+ "count" : 7
+ }
+ ]
+ }
+}
++---+
+
+ <<XML response>>
+
+ HTTP Request:
+
+------
+ GET http://<rm http
address:port>/ws/v1/cluster/appstatistics?states=accepted,running,finished&applicationTypes=mapreduce
+ Accept: application/xml
+------
+
+ Response Header:
+
++---+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 2459
+ Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appStatInfo>
+ <statItem>
+ <state>accepted</state>
+ <type>mapreduce</type>
+ <count>4</count>
+ </statItem>
+ <statItem>
+ <state>running</state>
+ <type>mapreduce</type>
+ <count>1</count>
+ </statItem>
+ <statItem>
+ <state>finished</state>
+ <type>mapreduce</type>
+ <count>7</count>
+ </statItem>
+</appStatInfo>
++---+
+
* Cluster {Application API}
An application resource contains information about a particular application
that was submitted to a cluster.
