YARN-5412. Create a proxy chain for ResourceManager REST API in the Router. (Contributed by Giovanni Matteo Fumarola via curino)
(cherry picked from commit b6240b92abf453affc5fd64e1eedf2d29842aa75) (cherry picked from commit acda6b96a4e92e432bd1d97fa14004a11e70387e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bfd967d3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bfd967d3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bfd967d3 Branch: refs/heads/branch-2 Commit: bfd967d33866d7a3067f0b7cd107d5d45e4adf6e Parents: 049f7c8 Author: Carlo Curino <cur...@apache.org> Authored: Thu Jul 27 14:34:45 2017 -0700 Committer: Carlo Curino <cur...@apache.org> Committed: Thu Sep 21 17:13:28 2017 -0700 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 24 + .../hadoop/yarn/webapp/util/WebAppUtils.java | 14 + .../src/main/resources/yarn-default.xml | 30 + .../resourcemanager/webapp/RMWSConsts.java | 15 + .../resourcemanager/webapp/RMWebAppUtil.java | 29 + .../webapp/RMWebServiceProtocol.java | 133 +- .../resourcemanager/webapp/RMWebServices.java | 4 +- .../webapp/dao/AppAttemptInfo.java | 5 +- .../TestFederationRMStateStoreService.java | 9 +- .../hadoop-yarn-server-router/pom.xml | 34 +- .../hadoop/yarn/server/router/Router.java | 35 + .../webapp/AbstractRESTRequestInterceptor.java | 89 ++ .../webapp/DefaultRequestInterceptorREST.java | 496 +++++++ .../yarn/server/router/webapp/HTTPMethods.java | 34 + .../router/webapp/RESTRequestInterceptor.java | 125 ++ .../yarn/server/router/webapp/RouterWebApp.java | 48 + .../router/webapp/RouterWebServiceUtil.java | 227 +++ .../server/router/webapp/RouterWebServices.java | 876 ++++++++++++ .../yarn/server/router/webapp/package-info.java | 20 + .../webapp/BaseRouterWebServicesTest.java | 601 ++++++++ .../yarn/server/router/webapp/JavaProcess.java | 52 + .../webapp/MockRESTRequestInterceptor.java | 340 +++++ .../PassThroughRESTRequestInterceptor.java | 339 +++++ .../router/webapp/TestRouterWebServices.java | 269 ++++ .../webapp/TestRouterWebServicesREST.java | 1298 ++++++++++++++++++ .../src/test/resources/capacity-scheduler.xml | 111 ++ .../src/test/resources/log4j.properties | 19 + .../src/test/resources/yarn-site.xml | 30 + 28 files changed, 5237 insertions(+), 69 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 7adfdf1..34374cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2629,6 +2629,30 @@ public class YarnConfiguration extends Configuration { ROUTER_PREFIX + "submit.retry"; public static final int DEFAULT_ROUTER_CLIENTRM_SUBMIT_RETRY = 3; + public static final String ROUTER_WEBAPP_PREFIX = ROUTER_PREFIX + "webapp."; + + /** The address of the Router web application. */ + public static final String ROUTER_WEBAPP_ADDRESS = + ROUTER_WEBAPP_PREFIX + "address"; + + public static final int DEFAULT_ROUTER_WEBAPP_PORT = 8089; + public static final String DEFAULT_ROUTER_WEBAPP_ADDRESS = + "0.0.0.0:" + DEFAULT_ROUTER_WEBAPP_PORT; + + /** The https address of the Router web application. */ + public static final String ROUTER_WEBAPP_HTTPS_ADDRESS = + ROUTER_WEBAPP_PREFIX + "https.address"; + + public static final int DEFAULT_ROUTER_WEBAPP_HTTPS_PORT = 8091; + public static final String DEFAULT_ROUTER_WEBAPP_HTTPS_ADDRESS = + "0.0.0.0:" + DEFAULT_ROUTER_WEBAPP_HTTPS_PORT; + + public static final String ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE = + ROUTER_WEBAPP_PREFIX + "interceptor-class.pipeline"; + public static final String DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS = + "org.apache.hadoop.yarn.server.router.webapp." + + "DefaultRequestInterceptorREST"; + //////////////////////////////// // Other Configs //////////////////////////////// http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index a32b2be..d62a810 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -129,6 +129,20 @@ public class WebAppUtils { return getRMWebAppURLWithoutScheme(conf, false); } + public static String getRouterWebAppURLWithScheme(Configuration conf) { + return getHttpSchemePrefix(conf) + getRouterWebAppURLWithoutScheme(conf); + } + + public static String getRouterWebAppURLWithoutScheme(Configuration conf) { + if (YarnConfiguration.useHttps(conf)) { + return conf.get(YarnConfiguration.ROUTER_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_HTTPS_ADDRESS); + } else { + return conf.get(YarnConfiguration.ROUTER_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_ADDRESS); + } + } + public static List<String> getProxyHostsAndPortsForAmFilter( Configuration conf) { List<String> addrs = new ArrayList<String>(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 998e4cb..122b824 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3153,4 +3153,34 @@ <value></value> </property> + <property> + <description> + The comma separated list of class names that implement the + RequestInterceptor interface. This is used by the RouterWebServices + to create the request processing pipeline for users. + </description> + <name>yarn.router.webapp.interceptor-class.pipeline</name> + <value>org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST</value> + </property> + + <property> + <description> + The http address of the Router web application. + If only a host is provided as the value, + the webapp will be served on a random port. + </description> + <name>yarn.router.webapp.address</name> + <value>0.0.0.0:8089</value> + </property> + + <property> + <description> + The https address of the Router web application. + If only a host is provided as the value, + the webapp will be served on a random port. + </description> + <name> yarn.router.webapp.https.address</name> + <value>0.0.0.0:8091</value> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java index 23d4bb1..5a945da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java @@ -168,6 +168,12 @@ public final class RMWSConsts { */ public static final String APPS_TIMEOUT = "/apps/{appid}/timeout"; + /** + * Path for {@code RouterWebServices#getContainer}. + */ + public static final String GET_CONTAINER = + "/apps/{appid}/appattempts/{appattemptid}/containers/{containerid}"; + // ----------------QueryParams for RMWebServiceProtocol---------------- public static final String TIME = "time"; @@ -194,6 +200,15 @@ public final class RMWSConsts { public static final String END_TIME = "end-time"; public static final String INCLUDE_RESOURCE = "include-resource-allocations"; public static final String TYPE = "type"; + public static final String CONTAINERID = "containerid"; + public static final String APPATTEMPTS = "appattempts"; + public static final String TIMEOUTS = "timeouts"; + public static final String PRIORITY = "priority"; + public static final String TIMEOUT = "timeout"; + public static final String ATTEMPTS = "appattempts"; + public static final String GET_LABELS = "get-labels"; + public static final String DESELECTS = "deSelects"; + public static final String CONTAINERS = "containers"; private RMWSConsts() { // not called http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java index f529dc2..ce05456 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java @@ -20,10 +20,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import java.io.IOException; import java.nio.ByteBuffer; +import java.security.Principal; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import javax.servlet.http.HttpServletRequest; + import org.apache.commons.codec.binary.Base64; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -341,4 +344,30 @@ public final class RMWebAppUtil { logAggregationContextInfo.getLogAggregationPolicyClassName(), logAggregationContextInfo.getLogAggregationPolicyParameters()); } + + /** + * Helper method to retrieve the UserGroupInformation from the + * HttpServletRequest. + * + * @param hsr the servlet request + * @param usePrincipal true if we need to use the principal user, remote + * otherwise. + * @return the user group information of the caller. + **/ + public static UserGroupInformation getCallerUserGroupInformation( + HttpServletRequest hsr, boolean usePrincipal) { + + String remoteUser = hsr.getRemoteUser(); + if (usePrincipal) { + Principal princ = hsr.getUserPrincipal(); + remoteUser = princ == null ? null : princ.getName(); + } + + UserGroupInformation callerUGI = null; + if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); + } + + return callerUGI; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java index 250cb95..062ca4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java @@ -108,7 +108,7 @@ public interface RMWebServiceProtocol { * This method dumps the scheduler logs for the time got in input, and it is * reachable by using {@link RMWSConsts#SCHEDULER_LOGS}. * - * @param time the period of time + * @param time the period of time. It is a FormParam. * @param hsr the servlet request * @return the result of the operation * @throws IOException when it cannot create dump log file @@ -121,7 +121,7 @@ public interface RMWebServiceProtocol { * reachable by using {@link RMWSConsts#NODES}. * * @see ApplicationClientProtocol#getClusterNodes - * @param states the states we want to filter + * @param states the states we want to filter. It is a QueryParam. * @return all nodes in the cluster. If the states param is given, returns all * nodes that are in the comma-separated list of states */ @@ -131,7 +131,8 @@ public interface RMWebServiceProtocol { * This method retrieves a specific node information, and it is reachable by * using {@link RMWSConsts#NODES_NODEID}. * - * @param nodeId the node we want to retrieve the information + * @param nodeId the node we want to retrieve the information. It is a + * PathParam. * @return the information about the node in input */ NodeInfo getNode(String nodeId); @@ -142,19 +143,25 @@ public interface RMWebServiceProtocol { * * @see ApplicationClientProtocol#getApplications * @param hsr the servlet request - * @param stateQuery right now the stateQuery is deprecated - * @param statesQuery filter the result by states - * @param finalStatusQuery filter the result by final states - * @param userQuery filter the result by user - * @param queueQuery filter the result by queue - * @param count set a limit of the result - * @param startedBegin filter the result by started begin time - * @param startedEnd filter the result by started end time - * @param finishBegin filter the result by finish begin time - * @param finishEnd filter the result by finish end time - * @param applicationTypes filter the result by types - * @param applicationTags filter the result by tags - * @param unselectedFields De-selected params to avoid from report + * @param stateQuery right now the stateQuery is deprecated. It is a + * QueryParam. + * @param statesQuery filter the result by states. It is a QueryParam. + * @param finalStatusQuery filter the result by final states. It is a + * QueryParam. + * @param userQuery filter the result by user. It is a QueryParam. + * @param queueQuery filter the result by queue. It is a QueryParam. + * @param count set a limit of the result. It is a QueryParam. + * @param startedBegin filter the result by started begin time. It is a + * QueryParam. + * @param startedEnd filter the result by started end time. It is a + * QueryParam. + * @param finishBegin filter the result by finish begin time. It is a + * QueryParam. + * @param finishEnd filter the result by finish end time. It is a QueryParam. + * @param applicationTypes filter the result by types. It is a QueryParam. + * @param applicationTags filter the result by tags. It is a QueryParam. + * @param unselectedFields De-selected params to avoid from report. It is a + * QueryParam. * @return all apps in the cluster */ @SuppressWarnings("checkstyle:parameternumber") @@ -169,7 +176,8 @@ public interface RMWebServiceProtocol { * reachable by using {@link RMWSConsts#SCHEDULER_ACTIVITIES}. * * @param hsr the servlet request - * @param nodeId the node we want to retrieve the activities + * @param nodeId the node we want to retrieve the activities. It is a + * QueryParam. * @return all the activities in the specific node */ ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId); @@ -180,8 +188,10 @@ public interface RMWebServiceProtocol { * {@link RMWSConsts#SCHEDULER_APP_ACTIVITIES}. * * @param hsr the servlet request - * @param appId the applicationId we want to retrieve the activities - * @param time for how long we want to retrieve the activities + * @param appId the applicationId we want to retrieve the activities. It is a + * QueryParam. + * @param time for how long we want to retrieve the activities. It is a + * QueryParam. * @return all the activities about a specific app for a specific time */ AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, @@ -192,8 +202,8 @@ public interface RMWebServiceProtocol { * reachable by using {@link RMWSConsts#APP_STATISTICS}. * * @param hsr the servlet request - * @param stateQueries filter the result by states - * @param typeQueries filter the result by type names + * @param stateQueries filter the result by states. It is a QueryParam. + * @param typeQueries filter the result by type names. It is a QueryParam. * @return the application's statistics for specific states and types */ ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr, @@ -205,8 +215,10 @@ public interface RMWebServiceProtocol { * * @see ApplicationClientProtocol#getApplicationReport * @param hsr the servlet request - * @param appId the Id of the application we want the report - * @param unselectedFields De-selected params to avoid from report + * @param appId the Id of the application we want the report. It is a + * PathParam. + * @param unselectedFields De-selected param list to avoid from report. It is + * a QueryParam. * @return the app report for a specific application */ AppInfo getApp(HttpServletRequest hsr, String appId, @@ -217,7 +229,8 @@ public interface RMWebServiceProtocol { * using {@link RMWSConsts#APPS_APPID_STATE}. * * @param hsr the servlet request - * @param appId the Id of the application we want the state + * @param appId the Id of the application we want the state. It is a + * PathParam. * @return the state for a specific application * @throws AuthorizationException if the user is not authorized */ @@ -228,9 +241,10 @@ public interface RMWebServiceProtocol { * This method updates the state of the app in input, and it is reachable by * using {@link RMWSConsts#APPS_APPID_STATE}. * - * @param targetState the target state for the app + * @param targetState the target state for the app. It is a content param. * @param hsr the servlet request - * @param appId the Id of the application we want to update the state + * @param appId the Id of the application we want to update the state. It is a + * PathParam. * @return Response containing the status code * @throws AuthorizationException if the user is not authorized to invoke this * method @@ -259,7 +273,7 @@ public interface RMWebServiceProtocol { * cluster, and it is reachable by using {@link RMWSConsts#LABEL_MAPPINGS}. * * @see ApplicationClientProtocol#getLabelsToNodes - * @param labels filter the result by node labels + * @param labels filter the result by node labels. It is a QueryParam. * @return all the nodes within multiple node labels * @throws IOException if an IOException happened */ @@ -270,7 +284,7 @@ public interface RMWebServiceProtocol { * reachable by using {@link RMWSConsts#REPLACE_NODE_TO_LABELS}. * * @see ResourceManagerAdministrationProtocol#replaceLabelsOnNode - * @param newNodeToLabels the list of new labels + * @param newNodeToLabels the list of new labels. It is a content param. * @param hsr the servlet request * @return Response containing the status code * @throws Exception if an exception happened @@ -283,9 +297,10 @@ public interface RMWebServiceProtocol { * reachable by using {@link RMWSConsts#NODES_NODEID_REPLACE_LABELS}. * * @see ResourceManagerAdministrationProtocol#replaceLabelsOnNode - * @param newNodeLabelsName the list of new labels + * @param newNodeLabelsName the list of new labels. It is a QueryParam. * @param hsr the servlet request - * @param nodeId the node we want to replace the node labels + * @param nodeId the node we want to replace the node labels. It is a + * PathParam. * @return Response containing the status code * @throws Exception if an exception happened */ @@ -309,7 +324,7 @@ public interface RMWebServiceProtocol { * reachable by using {@link RMWSConsts#ADD_NODE_LABELS}. * * @see ResourceManagerAdministrationProtocol#addToClusterNodeLabels - * @param newNodeLabels the node labels to add + * @param newNodeLabels the node labels to add. It is a content param. * @param hsr the servlet request * @return Response containing the status code * @throws Exception in case of bad request @@ -322,7 +337,7 @@ public interface RMWebServiceProtocol { * reachable by using {@link RMWSConsts#REMOVE_NODE_LABELS}. * * @see ResourceManagerAdministrationProtocol#removeFromClusterNodeLabels - * @param oldNodeLabels the node labels to remove + * @param oldNodeLabels the node labels to remove. It is a QueryParam. * @param hsr the servlet request * @return Response containing the status code * @throws Exception in case of bad request @@ -335,7 +350,8 @@ public interface RMWebServiceProtocol { * reachable by using {@link RMWSConsts#NODES_NODEID_GETLABELS}. * * @param hsr the servlet request - * @param nodeId the node we want to get all the node labels + * @param nodeId the node we want to get all the node labels. It is a + * PathParam. * @return all the labels for a specific node. * @throws IOException if an IOException happened */ @@ -347,7 +363,7 @@ public interface RMWebServiceProtocol { * by using {@link RMWSConsts#APPS_APPID_PRIORITY}. * * @param hsr the servlet request - * @param appId the app we want to get the priority + * @param appId the app we want to get the priority. It is a PathParam. * @return the priority for a specific application * @throws AuthorizationException in case of the user is not authorized */ @@ -358,9 +374,11 @@ public interface RMWebServiceProtocol { * This method updates the priority for a specific application, and it is * reachable by using {@link RMWSConsts#APPS_APPID_PRIORITY}. * - * @param targetPriority the priority we want to set for the app + * @param targetPriority the priority we want to set for the app. It is a + * content param. * @param hsr the servlet request - * @param appId the application we want to update its priority + * @param appId the application we want to update its priority. It is a + * PathParam. * @return Response containing the status code * @throws AuthorizationException if the user is not authenticated * @throws YarnException if the target is null @@ -376,7 +394,8 @@ public interface RMWebServiceProtocol { * using {@link RMWSConsts#APPS_APPID_QUEUE}. * * @param hsr the servlet request - * @param appId the application we want to retrieve its queue + * @param appId the application we want to retrieve its queue. It is a + * PathParam. * @return the Queue for a specific application. * @throws AuthorizationException if the user is not authenticated */ @@ -387,9 +406,10 @@ public interface RMWebServiceProtocol { * This method updates the queue for a specific application, and it is * reachable by using {@link RMWSConsts#APPS_APPID_QUEUE}. * - * @param targetQueue the queue we want to set + * @param targetQueue the queue we want to set. It is a content param. * @param hsr the servlet request - * @param appId the application we want to change its queue + * @param appId the application we want to change its queue. It is a + * PathParam. * @return Response containing the status code * @throws AuthorizationException if the user is not authenticated * @throws YarnException if the app is not found @@ -424,7 +444,7 @@ public interface RMWebServiceProtocol { * @see ApplicationClientProtocol#submitApplication * * @param newApp structure containing information to construct the - * ApplicationSubmissionContext + * ApplicationSubmissionContext. It is a content param. * @param hsr the servlet request * @return Response containing the status code * @throws AuthorizationException if the user is not authorized to invoke this @@ -441,7 +461,7 @@ public interface RMWebServiceProtocol { * by using {@link RMWSConsts#DELEGATION_TOKEN}. * * @see ApplicationBaseProtocol#getDelegationToken - * @param tokenData the token to delegate + * @param tokenData the token to delegate. It is a content param. * @param hsr the servlet request * @return Response containing the status code * @throws AuthorizationException if Kerberos auth failed @@ -508,7 +528,7 @@ public interface RMWebServiceProtocol { * @see ApplicationClientProtocol#submitReservation * * @param resContext provides information to construct the - * ReservationSubmissionRequest + * ReservationSubmissionRequest. It is a content param. * @param hsr the servlet request * @return Response containing the status code * @throws AuthorizationException if the user is not authorized to invoke this @@ -527,7 +547,7 @@ public interface RMWebServiceProtocol { * @see ApplicationClientProtocol#updateReservation * * @param resContext provides information to construct the - * ReservationUpdateRequest + * ReservationUpdateRequest. It is a content param. * @param hsr the servlet request * @return Response containing the status code * @throws AuthorizationException if the user is not authorized to invoke this @@ -546,7 +566,7 @@ public interface RMWebServiceProtocol { * @see ApplicationClientProtocol#deleteReservation * * @param resContext provides information to construct the - * ReservationDeleteRequest + * ReservationDeleteRequest. It is a content param. * @param hsr the servlet request * @return Response containing the status code * @throws AuthorizationException when the user group information cannot be @@ -566,12 +586,13 @@ public interface RMWebServiceProtocol { * reachable by using {@link RMWSConsts#RESERVATION_LIST}. * * @see ApplicationClientProtocol#listReservations - * @param queue filter the result by queue - * @param reservationId filter the result by reservationId - * @param startTime filter the result by start time - * @param endTime filter the result by end time + * @param queue filter the result by queue. It is a QueryParam. + * @param reservationId filter the result by reservationId. It is a + * QueryParam. + * @param startTime filter the result by start time. It is a QueryParam. + * @param endTime filter the result by end time. It is a QueryParam. * @param includeResourceAllocations true if the resource allocation should be - * in the result, false otherwise + * in the result, false otherwise. It is a QueryParam. * @param hsr the servlet request * @return Response containing the status code * @throws Exception in case of bad request @@ -586,8 +607,8 @@ public interface RMWebServiceProtocol { * {@link RMWSConsts#APPS_TIMEOUTS_TYPE}. * * @param hsr the servlet request - * @param appId the application we want to get the timeout - * @param type the type of the timeouts + * @param appId the application we want to get the timeout. It is a PathParam. + * @param type the type of the timeouts. It is a PathParam. * @return the timeout for a specific application with a specific type. * @throws AuthorizationException if the user is not authorized */ @@ -599,7 +620,8 @@ public interface RMWebServiceProtocol { * reachable by using {@link RMWSConsts#APPS_TIMEOUTS}. * * @param hsr the servlet request - * @param appId the application we want to get the timeouts + * @param appId the application we want to get the timeouts. It is a + * PathParam. * @return the timeouts for a specific application * @throws AuthorizationException if the user is not authorized */ @@ -611,9 +633,9 @@ public interface RMWebServiceProtocol { * reachable by using {@link RMWSConsts#APPS_TIMEOUT}. * * @see ApplicationClientProtocol#updateApplicationTimeouts - * @param appTimeout the appTimeoutInfo + * @param appTimeout the appTimeoutInfo. It is a content param. * @param hsr the servlet request - * @param appId the application we want to update + * @param appId the application we want to update. It is a PathParam. * @return Response containing the status code * @throws AuthorizationException if the user is not authorized to invoke this * method @@ -631,7 +653,8 @@ public interface RMWebServiceProtocol { * * @see ApplicationBaseProtocol#getApplicationAttempts * @param hsr the servlet request - * @param appId the application we want to get the attempts + * @param appId the application we want to get the attempts. It is a + * PathParam. * @return all the attempts info for a specific application */ AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId); http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index b8bd9fb..166a4c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -435,7 +435,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { @QueryParam(RMWSConsts.FINISHED_TIME_END) String finishEnd, @QueryParam(RMWSConsts.APPLICATION_TYPES) Set<String> applicationTypes, @QueryParam(RMWSConsts.APPLICATION_TAGS) Set<String> applicationTags, - @QueryParam("deSelects") Set<String> unselectedFields) { + @QueryParam(RMWSConsts.DESELECTS) Set<String> unselectedFields) { boolean checkCount = false; boolean checkStart = false; boolean checkEnd = false; @@ -818,7 +818,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { @Override public AppInfo getApp(@Context HttpServletRequest hsr, @PathParam(RMWSConsts.APPID) String appId, - @QueryParam("deSelects") Set<String> unselectedFields) { + @QueryParam(RMWSConsts.DESELECTS) Set<String> unselectedFields) { init(); ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); RMApp app = rm.getRMContext().getRMApps().get(id); http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java index 55bf999..82a946e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java @@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @XmlRootElement(name = "appAttempt") @@ -106,4 +105,8 @@ public class AppAttemptInfo { public String getLogsLink() { return this.logsLink; } + + public String getAppAttemptId() { + return this.appAttemptId; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java index d92a793..e5e156d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; @@ -86,12 +87,8 @@ public class TestFederationRMStateStoreService { // Initially there should be no entry for the sub-cluster rm.init(conf); stateStore = rm.getFederationStateStoreService().getStateStoreClient(); - try { - stateStore.getSubCluster(request); - Assert.fail("There should be no entry for the sub-cluster."); - } catch (YarnException e) { - Assert.assertTrue(e.getMessage().endsWith("does not exist")); - } + GetSubClusterInfoResponse response = stateStore.getSubCluster(request); + Assert.assertNull(response); // Validate if sub-cluster is registered rm.start(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml index 4eea9a6..e8b4d56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml @@ -50,6 +50,25 @@ <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-common</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-common</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + + <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <type>test-jar</type> <scope>test</scope> @@ -57,21 +76,26 @@ <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-common</artifactId> + <artifactId>hadoop-yarn-server-resourcemanager</artifactId> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-common</artifactId> - <type>test-jar</type> + <artifactId>hadoop-yarn-server-nodemanager</artifactId> <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> <scope>test</scope> </dependency> + + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + </dependency> + </dependencies> <build> http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java index d2eee5a..121e534 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.router; import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.ShutdownHookManager; @@ -28,11 +29,19 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil; import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService; import org.apache.hadoop.yarn.server.router.rmadmin.RouterRMAdminService; +import org.apache.hadoop.yarn.server.router.webapp.RouterWebApp; +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.WebApps; +import org.apache.hadoop.yarn.webapp.WebApps.Builder; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; + /** * The router is a stateless YARN component which is the entry point to the * cluster. It can be deployed on multiple nodes behind a Virtual IP (VIP) with @@ -56,6 +65,9 @@ public class Router extends CompositeService { private AtomicBoolean isStopping = new AtomicBoolean(false); private RouterClientRMService clientRMProxyService; private RouterRMAdminService rmAdminProxyService; + private WebApp webApp; + @VisibleForTesting + protected String webAppAddress; /** * Priority of the Router shutdown hook. @@ -79,6 +91,10 @@ public class Router extends CompositeService { // RMAdmin Proxy rmAdminProxyService = createRMAdminProxyService(); addService(rmAdminProxyService); + // WebService + webAppAddress = WebAppUtils.getWebAppBindURL(this.conf, + YarnConfiguration.ROUTER_BIND_HOST, + WebAppUtils.getRouterWebAppURLWithoutScheme(this.conf)); super.serviceInit(conf); } @@ -89,11 +105,15 @@ public class Router extends CompositeService { } catch (IOException e) { throw new YarnRuntimeException("Failed Router login", e); } + startWepApp(); super.serviceStart(); } @Override protected void serviceStop() throws Exception { + if (webApp != null) { + webApp.stop(); + } if (isStopping.getAndSet(true)) { return; } @@ -117,6 +137,21 @@ public class Router extends CompositeService { return new RouterRMAdminService(); } + @Private + public WebApp getWebapp() { + return this.webApp; + } + + @VisibleForTesting + public void startWepApp() { + + RMWebAppUtil.setupSecurityAndFilters(conf, null); + + Builder<Object> builder = + WebApps.$for("cluster", null, null, "ws").with(conf).at(webAppAddress); + webApp = builder.start(new RouterWebApp(this)); + } + public static void main(String[] argv) { Configuration conf = new YarnConfiguration(); Thread http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AbstractRESTRequestInterceptor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AbstractRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AbstractRESTRequestInterceptor.java new file mode 100644 index 0000000..a2d78a4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AbstractRESTRequestInterceptor.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.router.webapp; + +import org.apache.hadoop.conf.Configuration; + +/** + * Extends the RequestInterceptor class and provides common functionality which + * can be used and/or extended by other concrete intercepter classes. + */ +public abstract class AbstractRESTRequestInterceptor + implements RESTRequestInterceptor { + + private Configuration conf; + private RESTRequestInterceptor nextInterceptor; + + /** + * Sets the {@link RESTRequestInterceptor} in the chain. + */ + @Override + public void setNextInterceptor(RESTRequestInterceptor nextInterceptor) { + this.nextInterceptor = nextInterceptor; + } + + /** + * Sets the {@link Configuration}. + */ + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + if (this.nextInterceptor != null) { + this.nextInterceptor.setConf(conf); + } + } + + /** + * Gets the {@link Configuration}. + */ + @Override + public Configuration getConf() { + return this.conf; + } + + /** + * Initializes the {@link RESTRequestInterceptor}. + */ + @Override + public void init(String user) { + if (this.nextInterceptor != null) { + this.nextInterceptor.init(user); + } + } + + /** + * Disposes the {@link RESTRequestInterceptor}. + */ + @Override + public void shutdown() { + if (this.nextInterceptor != null) { + this.nextInterceptor.shutdown(); + } + } + + /** + * Gets the next {@link RESTRequestInterceptor} in the chain. + */ + @Override + public RESTRequestInterceptor getNextInterceptor() { + return this.nextInterceptor; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java new file mode 100644 index 0000000..aa8e3eb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java @@ -0,0 +1,496 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.router.webapp; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.Response; + +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntryList; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; +import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; +import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; +import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; + +/** + * Extends the AbstractRequestInterceptorClient class and provides an + * implementation that simply forwards the client requests to the resource + * manager. + */ +public final class DefaultRequestInterceptorREST + extends AbstractRESTRequestInterceptor { + + private String webAppAddress; + + @Override + public void init(String user) { + webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(getConf()); + } + + @Override + public ClusterInfo get() { + return getClusterInfo(); + } + + @Override + public ClusterInfo getClusterInfo() { + return RouterWebServiceUtil.genericForward(webAppAddress, null, + ClusterInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.INFO, null, null); + } + + @Override + public ClusterMetricsInfo getClusterMetricsInfo() { + return RouterWebServiceUtil.genericForward(webAppAddress, null, + ClusterMetricsInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null); + } + + @Override + public SchedulerTypeInfo getSchedulerInfo() { + return RouterWebServiceUtil.genericForward(webAppAddress, null, + SchedulerTypeInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER, null, null); + } + + @Override + public String dumpSchedulerLogs(String time, HttpServletRequest hsr) + throws IOException { + // time is specified inside hsr + return RouterWebServiceUtil.genericForward(webAppAddress, null, + String.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_LOGS, null, null); + } + + @Override + public NodesInfo getNodes(String states) { + // states will be part of additionalParam + Map<String, String[]> additionalParam = new HashMap<String, String[]>(); + additionalParam.put(RMWSConsts.STATES, new String[] {states}); + return RouterWebServiceUtil.genericForward(webAppAddress, null, + NodesInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, + additionalParam); + } + + @Override + public NodeInfo getNode(String nodeId) { + return RouterWebServiceUtil.genericForward(webAppAddress, null, + NodeInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES + "/" + nodeId, null, + null); + } + + @Override + public AppsInfo getApps(HttpServletRequest hsr, String stateQuery, + Set<String> statesQuery, String finalStatusQuery, String userQuery, + String queueQuery, String count, String startedBegin, String startedEnd, + String finishBegin, String finishEnd, Set<String> applicationTypes, + Set<String> applicationTags, Set<String> unselectedFields) { + // all the params are specified inside hsr + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + AppsInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null); + } + + @Override + public ActivitiesInfo getActivities(HttpServletRequest hsr, String nodeId) { + // nodeId is specified inside hsr + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + ActivitiesInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_ACTIVITIES, null, + null); + } + + @Override + public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, + String appId, String time) { + // time and appId are specified inside hsr + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + AppActivitiesInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_APP_ACTIVITIES, + null, null); + } + + @Override + public ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr, + Set<String> stateQueries, Set<String> typeQueries) { + // stateQueries and typeQueries are specified inside hsr + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + ApplicationStatisticsInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APP_STATISTICS, null, null); + } + + @Override + public AppInfo getApp(HttpServletRequest hsr, String appId, + Set<String> unselectedFields) { + // unselectedFields is specified inside hsr + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + AppInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId, null, + null); + } + + @Override + public AppState getAppState(HttpServletRequest hsr, String appId) + throws AuthorizationException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + AppState.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.STATE, + null, null); + } + + @Override + public Response updateAppState(AppState targetState, HttpServletRequest hsr, + String appId) throws AuthorizationException, YarnException, + InterruptedException, IOException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH + + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.STATE, + targetState, null); + } + + @Override + public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr) + throws IOException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + NodeToLabelsInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.GET_NODE_TO_LABELS, null, + null); + } + + @Override + public LabelsToNodesInfo getLabelsToNodes(Set<String> labels) + throws IOException { + // labels will be part of additionalParam + Map<String, String[]> additionalParam = new HashMap<String, String[]>(); + additionalParam.put(RMWSConsts.LABELS, + labels.toArray(new String[labels.size()])); + return RouterWebServiceUtil.genericForward(webAppAddress, null, + LabelsToNodesInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.LABEL_MAPPINGS, null, + additionalParam); + } + + @Override + public Response replaceLabelsOnNodes(NodeToLabelsEntryList newNodeToLabels, + HttpServletRequest hsr) throws IOException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.POST, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.REPLACE_NODE_TO_LABELS, + newNodeToLabels, null); + } + + @Override + public Response replaceLabelsOnNode(Set<String> newNodeLabelsName, + HttpServletRequest hsr, String nodeId) throws Exception { + // newNodeLabelsName is specified inside hsr + return RouterWebServiceUtil + .genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.POST, RMWSConsts.RM_WEB_SERVICE_PATH + + RMWSConsts.NODES + "/" + nodeId + "/replace-labels", + null, null); + } + + @Override + public NodeLabelsInfo getClusterNodeLabels(HttpServletRequest hsr) + throws IOException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + NodeLabelsInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.GET_NODE_LABELS, null, + null); + } + + @Override + public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels, + HttpServletRequest hsr) throws Exception { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.POST, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.ADD_NODE_LABELS, + newNodeLabels, null); + } + + @Override + public Response removeFromCluserNodeLabels(Set<String> oldNodeLabels, + HttpServletRequest hsr) throws Exception { + // oldNodeLabels is specified inside hsr + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.POST, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.REMOVE_NODE_LABELS, null, + null); + } + + @Override + public NodeLabelsInfo getLabelsOnNode(HttpServletRequest hsr, String nodeId) + throws IOException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + NodeLabelsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + + RMWSConsts.NODES + "/" + nodeId + "/get-labels", + null, null); + } + + @Override + public AppPriority getAppPriority(HttpServletRequest hsr, String appId) + throws AuthorizationException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + AppPriority.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.PRIORITY, + null, null); + } + + @Override + public Response updateApplicationPriority(AppPriority targetPriority, + HttpServletRequest hsr, String appId) throws AuthorizationException, + YarnException, InterruptedException, IOException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH + + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.PRIORITY, + targetPriority, null); + } + + @Override + public AppQueue getAppQueue(HttpServletRequest hsr, String appId) + throws AuthorizationException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + AppQueue.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.QUEUE, + null, null); + } + + @Override + public Response updateAppQueue(AppQueue targetQueue, HttpServletRequest hsr, + String appId) throws AuthorizationException, YarnException, + InterruptedException, IOException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH + + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.QUEUE, + targetQueue, null); + } + + @Override + public Response createNewApplication(HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.POST, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS_NEW_APPLICATION, null, + null); + } + + @Override + public Response submitApplication(ApplicationSubmissionContextInfo newApp, + HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.POST, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, newApp, null); + } + + @Override + public Response postDelegationToken(DelegationToken tokenData, + HttpServletRequest hsr) throws AuthorizationException, IOException, + InterruptedException, Exception { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.POST, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.DELEGATION_TOKEN, tokenData, + null); + } + + @Override + public Response postDelegationTokenExpiration(HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException, + Exception { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.POST, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.DELEGATION_TOKEN_EXPIRATION, + null, null); + } + + @Override + public Response cancelDelegationToken(HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException, + Exception { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.DELETE, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.DELEGATION_TOKEN, null, + null); + } + + @Override + public Response createNewReservation(HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.POST, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_NEW, null, + null); + } + + @Override + public Response submitReservation(ReservationSubmissionRequestInfo resContext, + HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.POST, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_SUBMIT, + resContext, null); + } + + @Override + public Response updateReservation(ReservationUpdateRequestInfo resContext, + HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.POST, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_UPDATE, + resContext, null); + } + + @Override + public Response deleteReservation(ReservationDeleteRequestInfo resContext, + HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.POST, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_DELETE, + resContext, null); + } + + @Override + public Response listReservation(String queue, String reservationId, + long startTime, long endTime, boolean includeResourceAllocations, + HttpServletRequest hsr) throws Exception { + // queue, reservationId, startTime, endTime, includeResourceAllocations are + // specified inside hsr + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.RESERVATION_LIST, null, + null); + } + + @Override + public AppTimeoutInfo getAppTimeout(HttpServletRequest hsr, String appId, + String type) throws AuthorizationException { + return RouterWebServiceUtil + .genericForward(webAppAddress, hsr, AppTimeoutInfo.class, + HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + + "/" + appId + "/" + RMWSConsts.TIMEOUTS + "/" + type, + null, null); + } + + @Override + public AppTimeoutsInfo getAppTimeouts(HttpServletRequest hsr, String appId) + throws AuthorizationException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + AppTimeoutsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.TIMEOUTS, + null, null); + } + + @Override + public Response updateApplicationTimeout(AppTimeoutInfo appTimeout, + HttpServletRequest hsr, String appId) throws AuthorizationException, + YarnException, InterruptedException, IOException { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + Response.class, HTTPMethods.PUT, RMWSConsts.RM_WEB_SERVICE_PATH + + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.TIMEOUT, + appTimeout, null); + } + + @Override + public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) { + return RouterWebServiceUtil.genericForward(webAppAddress, hsr, + AppAttemptsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + + RMWSConsts.APPS + "/" + appId + "/" + RMWSConsts.APPATTEMPTS, + null, null); + } + + @Override + public AppAttemptInfo getAppAttempt(HttpServletRequest req, + HttpServletResponse res, String appId, String appAttemptId) { + return RouterWebServiceUtil.genericForward(webAppAddress, req, + AppAttemptInfo.class, + HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + + appId + "/" + RMWSConsts.APPATTEMPTS + "/" + appAttemptId, + null, null); + } + + @Override + public ContainersInfo getContainers(HttpServletRequest req, + HttpServletResponse res, String appId, String appAttemptId) { + return RouterWebServiceUtil.genericForward(webAppAddress, req, + ContainersInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + + RMWSConsts.APPATTEMPTS + "/" + appAttemptId + "/" + + RMWSConsts.CONTAINERS, + null, null); + } + + @Override + public ContainerInfo getContainer(HttpServletRequest req, + HttpServletResponse res, String appId, String appAttemptId, + String containerId) { + return RouterWebServiceUtil.genericForward(webAppAddress, req, + ContainerInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS + "/" + appId + "/" + + RMWSConsts.APPATTEMPTS + "/" + appAttemptId + "/" + + RMWSConsts.CONTAINERS + "/" + containerId, + null, null); + } + + @Override + public void setNextInterceptor(RESTRequestInterceptor next) { + throw new YarnRuntimeException("setNextInterceptor is being called on " + + "DefaultRequestInterceptorREST, which should be the last one " + + "in the chain. Check if the interceptor pipeline configuration " + + "is correct"); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/HTTPMethods.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/HTTPMethods.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/HTTPMethods.java new file mode 100644 index 0000000..45056ca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/HTTPMethods.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.router.webapp; + +/** + * HTTP verbs. + **/ +public enum HTTPMethods { + + /* to retrieve resource representation/information */ + GET, + /* to update existing resource */ + PUT, + /* to delete resources */ + DELETE, + /* to create new subordinate resources */ + POST +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RESTRequestInterceptor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RESTRequestInterceptor.java new file mode 100644 index 0000000..06f39b5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RESTRequestInterceptor.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.router.webapp; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServiceProtocol; +import org.apache.hadoop.yarn.server.webapp.WebServices; +import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; +import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; +import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; + +/** + * Defines the contract to be implemented by the request intercepter classes, + * that can be used to intercept and inspect messages sent from the client to + * the resource manager server. + * + * This class includes 4 methods getAppAttempts, getAppAttempt, getContainers + * and getContainer that belong to {@link WebServices}. They are in this class + * to make sure that RouterWebServices implements the same REST methods of + * {@code RMWebServices}. + */ +public interface RESTRequestInterceptor + extends RMWebServiceProtocol, Configurable { + + /** + * This method is called for initializing the intercepter. This is guaranteed + * to be called only once in the lifetime of this instance. + * + * @param user the name of the client + */ + void init(String user); + + /** + * This method is called to release the resources held by the intercepter. + * This will be called when the application pipeline is being destroyed. The + * concrete implementations should dispose the resources and forward the + * request to the next intercepter, if any. + */ + void shutdown(); + + /** + * Sets the next intercepter in the pipeline. The concrete implementation of + * this interface should always pass the request to the nextInterceptor after + * inspecting the message. The last intercepter in the chain is responsible to + * send the messages to the resource manager service and so the last + * intercepter will not receive this method call. + * + * @param nextInterceptor the RESTRequestInterceptor to set in the pipeline + */ + void setNextInterceptor(RESTRequestInterceptor nextInterceptor); + + /** + * Returns the next intercepter in the chain. + * + * @return the next intercepter in the chain + */ + RESTRequestInterceptor getNextInterceptor(); + + /** + * + * @see WebServices#getAppAttempt(HttpServletRequest, HttpServletResponse, + * String, String) + * @param req the servlet request + * @param res the servlet response + * @param appId the application we want to get the appAttempt. It is a + * PathParam. + * @param appAttemptId the AppAttempt we want to get the info. It is a + * PathParam. + * @return AppAttemptInfo of the specific AppAttempt + */ + AppAttemptInfo getAppAttempt(HttpServletRequest req, HttpServletResponse res, + String appId, String appAttemptId); + + /** + * + * @see WebServices#getContainers(HttpServletRequest, HttpServletResponse, + * String, String) + * @param req the servlet request + * @param res the servlet response + * @param appId the application we want to get the containers info. It is a + * PathParam. + * @param appAttemptId the AppAttempt we want to get the info. It is a + * PathParam. + * @return ContainersInfo of all the containers that belong to the specific + * AppAttempt + */ + ContainersInfo getContainers(HttpServletRequest req, HttpServletResponse res, + String appId, String appAttemptId); + + /** + * + * @see WebServices#getContainer(HttpServletRequest, HttpServletResponse, + * String, String, String) + * @param req the servlet request + * @param res the servlet response + * @param appId the application we want to get the containers info. It is a + * PathParam. + * @param appAttemptId the AppAttempt we want to get the info. It is a + * PathParam. + * @param containerId the container we want to get the info. It is a + * PathParam. + * @return ContainerInfo of the specific ContainerId + */ + ContainerInfo getContainer(HttpServletRequest req, HttpServletResponse res, + String appId, String appAttemptId, String containerId); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java new file mode 100644 index 0000000..5436bad --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.router.webapp; + +import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; +import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.YarnWebParams; + +/** + * The Router webapp. + */ +public class RouterWebApp extends WebApp implements YarnWebParams { + private Router router; + + public RouterWebApp(Router router) { + this.router = router; + } + + @Override + public void setup() { + bind(JAXBContextResolver.class); + bind(RouterWebServices.class); + bind(GenericExceptionHandler.class); + bind(RouterWebApp.class).toInstance(this); + + if (router != null) { + bind(Router.class).toInstance(router); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd967d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java new file mode 100644 index 0000000..18618ee --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java @@ -0,0 +1,227 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.router.webapp; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.ResponseBuilder; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil; +import org.apache.hadoop.yarn.webapp.BadRequestException; +import org.apache.hadoop.yarn.webapp.ForbiddenException; +import org.apache.hadoop.yarn.webapp.NotFoundException; + +import com.sun.jersey.api.ConflictException; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.WebResource.Builder; +import com.sun.jersey.core.util.MultivaluedMapImpl; + +/** + * The Router webservice util class. + */ +public final class RouterWebServiceUtil { + + private static String user = "YarnRouter"; + + private static final Log LOG = + LogFactory.getLog(RouterWebServiceUtil.class.getName()); + + /** Disable constructor. */ + private RouterWebServiceUtil() { + } + + /** + * Creates and performs a REST call to a specific WebService. + * + * @param webApp the address of the remote webap + * @param hsr the servlet request + * @param returnType the return type of the REST call + * @param <T> Type of return object. + * @param method the HTTP method of the REST call + * @param targetPath additional path to add to the webapp address + * @param formParam the form parameters as input for a specific REST call + * @param additionalParam the query parameters as input for a specific REST + * call in case the call has no servlet request + * @return the retrieved entity from the REST call + */ + protected static <T> T genericForward(String webApp, HttpServletRequest hsr, + final Class<T> returnType, HTTPMethods method, String targetPath, + Object formParam, Map<String, String[]> additionalParam) { + + UserGroupInformation callerUGI = null; + + if (hsr != null) { + callerUGI = RMWebAppUtil.getCallerUserGroupInformation(hsr, true); + } else { + // user not required + callerUGI = UserGroupInformation.createRemoteUser(user); + + } + + if (callerUGI == null) { + LOG.error("Unable to obtain user name, user not authenticated"); + return null; + } + + try { + return callerUGI.doAs(new PrivilegedExceptionAction<T>() { + @SuppressWarnings("unchecked") + @Override + public T run() { + + Map<String, String[]> paramMap = null; + + // We can have hsr or additionalParam. There are no case with both. + if (hsr != null) { + paramMap = hsr.getParameterMap(); + } else if (additionalParam != null) { + paramMap = additionalParam; + } + + ClientResponse response = RouterWebServiceUtil.invokeRMWebService( + webApp, targetPath, method, + (hsr == null) ? null : hsr.getPathInfo(), paramMap, formParam); + if (Response.class.equals(returnType)) { + return (T) RouterWebServiceUtil.clientResponseToResponse(response); + } + // YARN RM can answer with Status.OK or it throws an exception + if (response.getStatus() == 200) { + return response.getEntity(returnType); + } + RouterWebServiceUtil.retrieveException(response); + return null; + } + }); + } catch (InterruptedException e) { + return null; + } catch (IOException e) { + return null; + } + } + + /** + * Performs an invocation of a REST call on a remote RMWebService. + * + * @param additionalParam + */ + private static ClientResponse invokeRMWebService(String webApp, String path, + HTTPMethods method, String additionalPath, + Map<String, String[]> queryParams, Object formParam) { + Client client = Client.create(); + + WebResource webResource = client.resource(webApp).path(path); + + if (additionalPath != null && !additionalPath.isEmpty()) { + webResource = webResource.path(additionalPath); + } + + if (queryParams != null && !queryParams.isEmpty()) { + MultivaluedMap<String, String> paramMap = new MultivaluedMapImpl(); + + for (Entry<String, String[]> param : queryParams.entrySet()) { + String[] values = param.getValue(); + for (int i = 0; i < values.length; i++) { + paramMap.add(param.getKey(), values[i]); + } + } + webResource = webResource.queryParams(paramMap); + } + + // I can forward the call in JSON or XML since the Router will convert it + // again in Object before send it back to the client + Builder builder = null; + if (formParam != null) { + builder = webResource.entity(formParam, MediaType.APPLICATION_XML); + builder = builder.accept(MediaType.APPLICATION_XML); + } else { + builder = webResource.accept(MediaType.APPLICATION_XML); + } + + ClientResponse response = null; + + switch (method) { + case DELETE: + response = builder.delete(ClientResponse.class); + break; + case GET: + response = builder.get(ClientResponse.class); + break; + case POST: + response = builder.post(ClientResponse.class); + break; + case PUT: + response = builder.put(ClientResponse.class); + break; + default: + break; + } + + return response; + } + + public static Response clientResponseToResponse(ClientResponse r) { + if (r == null) { + return null; + } + // copy the status code + ResponseBuilder rb = Response.status(r.getStatus()); + // copy all the headers + for (Entry<String, List<String>> entry : r.getHeaders().entrySet()) { + for (String value : entry.getValue()) { + rb.header(entry.getKey(), value); + } + } + // copy the entity + rb.entity(r.getEntityInputStream()); + // return the response + return rb.build(); + } + + public static void retrieveException(ClientResponse response) { + String serverErrorMsg = response.getEntity(String.class); + int status = response.getStatus(); + if (status == 400) { + throw new BadRequestException(serverErrorMsg); + } + if (status == 403) { + throw new ForbiddenException(serverErrorMsg); + } + if (status == 404) { + throw new NotFoundException(serverErrorMsg); + } + if (status == 409) { + throw new ConflictException(serverErrorMsg); + } + + } + +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org