[24/50] [abbrv] hadoop git commit: YARN-5411. Create a proxy chain for ApplicationClientProtocol in the Router. (Giovanni Matteo Fumarola via Subru).

2017-09-21 Thread curino
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc0a2e6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterClientRMService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterClientRMService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterClientRMService.java
new file mode 100644
index 000..a9c3729
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterClientRMService.java
@@ -0,0 +1,210 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.clientrm;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
+import 
org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
+import 
org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import 
org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService.RequestInterceptorChainWrapper;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class to validate the ClientRM Service inside the Router.
+ */
+public class TestRouterClientRMService extends BaseRouterClientRMTest {
+
+  private static final Logger LOG =
+  LoggerFactory.getLogger(TestRouterClientRMService.class);
+
+  /**
+   * Tests if the pipeline is created properly.
+   */
+  @Test
+  public void testRequestInterceptorChainCreation() throws Exception {
+ClientRequestInterceptor root =
+super.getRouterClientRMService().createRequestInterceptorChain();
+int index = 0;
+while (root != null) {
+  // The current pipeline is:
+  // PassThroughClientRequestInterceptor - index = 0
+  // PassThroughClientRequestInterceptor - index = 1
+  // PassThroughClientRequestInterceptor - index = 2
+  // MockClientRequestInterceptor - index = 3
+  switch (index) {
+  case 0: // Fall to the next case
+  case 1: // Fall to the next case
+  case 2:
+// If index is equal to 0,1 or 2 we fall in this check
+
Assert.assertEquals(PassThroughClientRequestInterceptor.class.getName(),
+root.getClass().getName());
+break;
+  case 3:
+Assert.assertEquals(MockClientRequestInterceptor.class.getName(),
+root.getClass().getName());
+break;
+  default:
+Assert.fail();
+  }
+  root = root.getNextInterceptor();
+  index++;
+}
+Assert.assertEquals("The number of interceptors in chain does not match", 
4,
+index);
+  }
+
+  /**
+   * Test if the RouterClientRM forwards all the requests to the MockRM and get
+   * back the responses.
+   */
+  @Test
+  public void testRouterClientRMServiceE2E() throws Exception {
+
+String user = "test1";
+
+LOG.info("testRouterClientRMServiceE2E - Get New Application");
+
+GetNewApplicationResponse responseGetNewApp = getNewAppl

[24/50] [abbrv] hadoop git commit: YARN-5411. Create a proxy chain for ApplicationClientProtocol in the Router. (Giovanni Matteo Fumarola via Subru).

2017-08-01 Thread curino
YARN-5411. Create a proxy chain for ApplicationClientProtocol in the Router. 
(Giovanni Matteo Fumarola via Subru).

(cherry picked from commit 4846069061b6baa06da3b524b9e36567dd368388)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80e19040
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80e19040
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80e19040

Branch: refs/heads/trunk
Commit: 80e1904000923819c98ae68456cb0452c255d5f3
Parents: 5272af8
Author: Subru Krishnan 
Authored: Wed May 3 18:26:15 2017 -0700
Committer: Carlo Curino 
Committed: Tue Aug 1 17:28:24 2017 -0700

--
 hadoop-project/pom.xml  |   7 +
 .../hadoop/yarn/conf/YarnConfiguration.java |  21 +
 .../hadoop/yarn/util/LRUCacheHashMap.java   |  49 ++
 .../src/main/resources/yarn-default.xml |  18 +
 .../hadoop/yarn/util/TestLRUCacheHashMap.java   |  74 +++
 .../hadoop-yarn-server-common/pom.xml   |  11 +
 .../yarn/server/MockResourceManagerFacade.java  | 511 +
 .../hadoop-yarn-server-nodemanager/pom.xml  |   7 +
 .../amrmproxy/MockRequestInterceptor.java   |  14 +-
 .../amrmproxy/MockResourceManagerFacade.java| 514 -
 .../hadoop-yarn-server-router/pom.xml   |  19 +
 .../hadoop/yarn/server/router/Router.java   |  98 +++-
 .../AbstractClientRequestInterceptor.java   |  89 +++
 .../clientrm/ClientRequestInterceptor.java  |  65 +++
 .../DefaultClientRequestInterceptor.java| 334 +++
 .../router/clientrm/RouterClientRMService.java  | 544 ++
 .../server/router/clientrm/package-info.java|  20 +
 .../hadoop/yarn/server/router/TestRouter.java   |  26 -
 .../router/clientrm/BaseRouterClientRMTest.java | 574 +++
 .../clientrm/MockClientRequestInterceptor.java  |  36 ++
 .../PassThroughClientRequestInterceptor.java| 267 +
 .../clientrm/TestRouterClientRMService.java | 210 +++
 22 files changed, 2960 insertions(+), 548 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/80e19040/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 15bd1fa..7301e90 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -324,6 +324,13 @@
 
   
 org.apache.hadoop
+hadoop-yarn-server-common
+${project.version}
+test-jar
+  
+
+  
+org.apache.hadoop
  hadoop-yarn-server-tests
 ${project.version}
 test-jar

http://git-wip-us.apache.org/repos/asf/hadoop/blob/80e19040/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7bcb123..cf9c237 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2637,6 +2637,27 @@ public class YarnConfiguration extends Configuration {
 
   public static final int DEFAULT_FEDERATION_STATESTORE_SQL_MAXCONNECTIONS = 1;
 
+  public static final String ROUTER_PREFIX = YARN_PREFIX + "router.";
+
+  public static final String ROUTER_CLIENTRM_PREFIX =
+  ROUTER_PREFIX + "clientrm.";
+
+  public static final String ROUTER_CLIENTRM_ADDRESS =
+  ROUTER_CLIENTRM_PREFIX + ".address";
+  public static final int DEFAULT_ROUTER_CLIENTRM_PORT = 8050;
+  public static final String DEFAULT_ROUTER_CLIENTRM_ADDRESS =
+  "0.0.0.0:" + DEFAULT_ROUTER_CLIENTRM_PORT;
+
+  public static final String ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE =
+  ROUTER_CLIENTRM_PREFIX + "interceptor-class.pipeline";
+  public static final String DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS =
+  "org.apache.hadoop.yarn.server.router.clientrm."
+  + "DefaultClientRequestInterceptor";
+
+  public static final String ROUTER_CLIENTRM_PIPELINE_CACHE_MAX_SIZE =
+  ROUTER_CLIENTRM_PREFIX + "cache-max-size";
+  public static final int DEFAULT_ROUTER_CLIENTRM_PIPELINE_CACHE_MAX_SIZE = 25;
+
   
   // Other Configs
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/80e19040/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LRUCacheHashMap.java