Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1513258&r1=1513257&r2=1513258&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original) +++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Mon Aug 12 21:25:49 2013 @@ -638,6 +638,191 @@ public class TestRMWebServicesApps exten } @Test + public void testAppsQueryAppTypes() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + Thread.sleep(1); + RMApp app1 = rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + // finish App + MockAM am = rm + .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); + am.registerAppAttempt(); + am.unregisterAppAttempt(); + amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), + 1, ContainerState.COMPLETE); + + rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + .getShortUserName(), null, false, null, 2, null, "MAPREDUCE"); + rm.submitApp(1024, "", UserGroupInformation.getCurrentUser() + .getShortUserName(), null, false, null, 2, null, "NON-YARN"); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("applicationTypes", "MAPREDUCE") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + assertEquals("MAPREDUCE", + array.getJSONObject(0).getString("applicationType")); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", "YARN") + .queryParam("applicationTypes", "MAPREDUCE") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + assertTrue((array.getJSONObject(0).getString("applicationType") + .equals("YARN") && array.getJSONObject(1).getString("applicationType") + .equals("MAPREDUCE")) || + (array.getJSONObject(1).getString("applicationType").equals("YARN") + && array.getJSONObject(0).getString("applicationType") + .equals("MAPREDUCE"))); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", "YARN,NON-YARN") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + assertTrue((array.getJSONObject(0).getString("applicationType") + .equals("YARN") && array.getJSONObject(1).getString("applicationType") + .equals("NON-YARN")) || + (array.getJSONObject(1).getString("applicationType").equals("YARN") + && array.getJSONObject(0).getString("applicationType") + .equals("NON-YARN"))); + + r = resource(); + response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("applicationTypes", "") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 3, array.length()); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", "YARN,NON-YARN") + .queryParam("applicationTypes", "MAPREDUCE") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 3, array.length()); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", "YARN") + .queryParam("applicationTypes", "") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + assertEquals("YARN", + array.getJSONObject(0).getString("applicationType")); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", ",,, ,, YARN ,, ,") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + assertEquals("YARN", + array.getJSONObject(0).getString("applicationType")); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", ",,, ,, ,, ,") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 3, array.length()); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", "YARN, ,NON-YARN, ,,") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + assertTrue((array.getJSONObject(0).getString("applicationType") + .equals("YARN") && array.getJSONObject(1).getString("applicationType") + .equals("NON-YARN")) || + (array.getJSONObject(1).getString("applicationType").equals("YARN") + && array.getJSONObject(0).getString("applicationType") + .equals("NON-YARN"))); + + r = resource(); + response = + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("applicationTypes", " YARN, , ,,,") + .queryParam("applicationTypes", "MAPREDUCE , ,, ,") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + assertTrue((array.getJSONObject(0).getString("applicationType") + .equals("YARN") && array.getJSONObject(1).getString("applicationType") + .equals("MAPREDUCE")) || + (array.getJSONObject(1).getString("applicationType").equals("YARN") + && array.getJSONObject(0).getString("applicationType") + .equals("MAPREDUCE"))); + + rm.stop(); + } + + @Test public void testSingleApp() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml (original) +++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml Mon Aug 12 21:25:49 2013 @@ -19,11 +19,11 @@ <parent> <artifactId>hadoop-yarn-server</artifactId> <groupId>org.apache.hadoop</groupId> - <version>2.2.0-SNAPSHOT</version> + <version>2.3.0-SNAPSHOT</version> </parent> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-tests</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>2.3.0-SNAPSHOT</version> <name>hadoop-yarn-server-tests</name> <properties> Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1513258&r1=1513257&r2=1513258&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original) +++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Mon Aug 12 21:25:49 2013 @@ -304,6 +304,16 @@ public class MiniYARNCluster extends Com MiniYARNCluster.getHostname() + ":0"); getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS, MiniYARNCluster.getHostname() + ":0"); + + // Disable resource checks by default + if (!getConfig().getBoolean( + YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, + YarnConfiguration. + DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) { + getConfig().setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false); + getConfig().setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false); + } + LOG.info("Starting NM: " + index); nodeManagers[index].init(getConfig()); new Thread() { @@ -390,6 +400,11 @@ public class MiniYARNCluster extends Com } }; }; + + @Override + protected void stopRMProxy() { + return; + } }; }; } Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1513258&r1=1513257&r2=1513258&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original) +++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Mon Aug 12 21:25:49 2013 @@ -23,6 +23,10 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.net.InetSocketAddress; import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; import junit.framework.Assert; @@ -32,22 +36,29 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; @@ -56,7 +67,11 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +@RunWith(Parameterized.class) public class TestContainerManagerSecurity { static Log LOG = LogFactory.getLog(TestContainerManagerSecurity.class); @@ -64,28 +79,33 @@ public class TestContainerManagerSecurit .getRecordFactory(null); private static MiniYARNCluster yarnCluster; - static final Configuration conf = new Configuration(); + private Configuration conf; - @Test (timeout = 1000000) - public void testContainerManagerWithSecurityEnabled() throws Exception { - conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + @Parameters + public static Collection<Object[]> configs() { + Configuration configurationWithoutSecurity = new Configuration(); + configurationWithoutSecurity.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "simple"); + + Configuration configurationWithSecurity = new Configuration(); + configurationWithSecurity.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - testContainerManager(); + return Arrays.asList(new Object[][] { { configurationWithoutSecurity }, + { configurationWithSecurity } }); } - @Test (timeout=1000000) - public void testContainerManagerWithSecurityDisabled() throws Exception { - conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, - "simple"); - testContainerManager(); + public TestContainerManagerSecurity(Configuration conf) { + conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L); + UserGroupInformation.setConfiguration(conf); + this.conf = conf; } - private void testContainerManager() throws Exception { + @Test (timeout = 1000000) + public void testContainerManager() throws Exception { try { yarnCluster = new MiniYARNCluster(TestContainerManagerSecurity.class .getName(), 1, 1, 1); - conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L); - UserGroupInformation.setConfiguration(conf); yarnCluster.init(conf); yarnCluster.start(); @@ -177,6 +197,18 @@ public class TestContainerManagerSecurit } while (tempManager.getCurrentKey().getKeyId() == nmTokenSecretManagerRM .getCurrentKey().getKeyId()); + // Testing that NM rejects the requests when we don't send any token. + if (UserGroupInformation.isSecurityEnabled()) { + sb = new StringBuilder("Client cannot authenticate via:[TOKEN]"); + } else { + sb = + new StringBuilder( + "SIMPLE authentication is not enabled. Available:[TOKEN]"); + } + String errorMsg = testStartContainer(rpc, validAppAttemptId, validNode, + validContainerToken, null, true); + Assert.assertTrue(errorMsg.contains(sb.toString())); + org.apache.hadoop.yarn.api.records.Token invalidNMToken = tempManager.createNMToken(validAppAttemptId, validNode, user); sb = new StringBuilder("Given NMToken for application : "); @@ -211,12 +243,30 @@ public class TestContainerManagerSecurit Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode, validContainerToken, invalidNMToken, true).contains(sb.toString())); - // using correct tokens. nmtoken for appattempt should get saved. + // using correct tokens. nmtoken for app attempt should get saved. + conf.setInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, + 4 * 60 * 1000); + validContainerToken = + containerTokenSecretManager.createContainerToken(validContainerId, + validNode, user, r); + testStartContainer(rpc, validAppAttemptId, validNode, validContainerToken, validNMToken, false); Assert.assertTrue(nmTokenSecretManagerNM .isAppAttemptNMTokenKeyPresent(validAppAttemptId)); + //Now lets wait till container finishes and is removed from node manager. + waitForContainerToFinishOnNM(validContainerId); + sb = new StringBuilder("Attempt to relaunch the same container with id "); + sb.append(validContainerId); + Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode, + validContainerToken, validNMToken, true).contains(sb.toString())); + + // Container is removed from node manager's memory by this time. + // trying to stop the container. It should not throw any exception. + testStopContainer(rpc, validAppAttemptId, validNode, validContainerId, + validNMToken, false); + // Rolling over master key twice so that we can check whether older keys // are used for authentication. rollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM); @@ -224,13 +274,38 @@ public class TestContainerManagerSecurit rollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM); // trying get container status. Now saved nmToken should be used for - // authentication. + // authentication... It should complain saying container was recently + // stopped. + sb = new StringBuilder("Container "); + sb.append(validContainerId); + sb.append(" was recently stopped on node manager"); + Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode, + validContainerId, validNMToken, true).contains(sb.toString())); + + // Now lets remove the container from nm-memory + nm.getNodeStatusUpdater().clearFinishedContainersFromCache(); + + // This should fail as container is removed from recently tracked finished + // containers. sb = new StringBuilder("Container "); sb.append(validContainerId.toString()); sb.append(" is not handled by this NodeManager"); Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode, validContainerId, validNMToken, false).contains(sb.toString())); - + + } + + private void waitForContainerToFinishOnNM(ContainerId containerId) { + Context nmContet = yarnCluster.getNodeManager(0).getNMContext(); + int interval = 4 * 60; // Max time for container token to expire. + while ((interval-- > 0) + && nmContet.getContainers().containsKey(containerId)) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + Assert.assertFalse(nmContet.getContainers().containsKey(containerId)); } protected void waitForNMToReceiveNMTokenKey( @@ -259,6 +334,23 @@ public class TestContainerManagerSecurit Assert.assertTrue((nmTokenSecretManagerNM.getCurrentKey().getKeyId() == nmTokenSecretManagerRM.getCurrentKey().getKeyId())); } + + private String testStopContainer(YarnRPC rpc, + ApplicationAttemptId appAttemptId, NodeId nodeId, + ContainerId containerId, Token nmToken, boolean isExceptionExpected) { + try { + stopContainer(rpc, nmToken, + Arrays.asList(new ContainerId[] { containerId }), appAttemptId, + nodeId); + if (isExceptionExpected) { + fail("Exception was expected!!"); + } + return ""; + } catch (Exception e) { + e.printStackTrace(); + return e.getMessage(); + } + } private String testGetContainer(YarnRPC rpc, ApplicationAttemptId appAttemptId, NodeId nodeId, @@ -278,7 +370,7 @@ public class TestContainerManagerSecurit } } - protected String testStartContainer(YarnRPC rpc, + private String testStartContainer(YarnRPC rpc, ApplicationAttemptId appAttemptId, NodeId nodeId, org.apache.hadoop.yarn.api.records.Token containerToken, org.apache.hadoop.yarn.api.records.Token nmToken, @@ -296,24 +388,50 @@ public class TestContainerManagerSecurit } } + private void stopContainer(YarnRPC rpc, Token nmToken, + List<ContainerId> containerId, ApplicationAttemptId appAttemptId, + NodeId nodeId) throws Exception { + StopContainersRequest request = + StopContainersRequest.newInstance(containerId); + ContainerManagementProtocol proxy = null; + try { + proxy = + getContainerManagementProtocolProxy(rpc, nmToken, nodeId, + appAttemptId.toString()); + StopContainersResponse response = proxy.stopContainers(request); + if (response.getFailedRequests() != null && + response.getFailedRequests().containsKey(containerId)) { + parseAndThrowException(response.getFailedRequests().get(containerId) + .deSerialize()); + } + } catch (Exception e) { + if (proxy != null) { + rpc.stopProxy(proxy, conf); + } + } + } + private void getContainerStatus(YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken, ContainerId containerId, ApplicationAttemptId appAttemptId, NodeId nodeId, boolean isExceptionExpected) throws Exception { - GetContainerStatusRequest request = - Records.newRecord(GetContainerStatusRequest.class); - request.setContainerId(containerId); - + List<ContainerId> containerIds = new ArrayList<ContainerId>(); + containerIds.add(containerId); + GetContainerStatusesRequest request = + GetContainerStatusesRequest.newInstance(containerIds); ContainerManagementProtocol proxy = null; - try { proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.toString()); - proxy.getContainerStatus(request); - + GetContainerStatusesResponse statuses = proxy.getContainerStatuses(request); + if (statuses.getFailedRequests() != null + && statuses.getFailedRequests().containsKey(containerId)) { + parseAndThrowException(statuses.getFailedRequests().get(containerId) + .deSerialize()); + } } finally { if (proxy != null) { rpc.stopProxy(proxy, conf); @@ -326,17 +444,21 @@ public class TestContainerManagerSecurit org.apache.hadoop.yarn.api.records.Token containerToken, NodeId nodeId, String user) throws Exception { - StartContainerRequest request = - Records.newRecord(StartContainerRequest.class); - request.setContainerToken(containerToken); ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); - request.setContainerLaunchContext(context); - + StartContainerRequest scRequest = + StartContainerRequest.newInstance(context,containerToken); + List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); ContainerManagementProtocol proxy = null; try { proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user); - proxy.startContainer(request); + StartContainersResponse response = proxy.startContainers(allRequests); + for(SerializedException ex : response.getFailedRequests().values()){ + parseAndThrowException(ex.deSerialize()); + } } finally { if (proxy != null) { rpc.stopProxy(proxy, conf); @@ -344,6 +466,17 @@ public class TestContainerManagerSecurit } } + private void parseAndThrowException(Throwable t) throws YarnException, + IOException { + if (t instanceof YarnException) { + throw (YarnException) t; + } else if (t instanceof InvalidToken) { + throw (InvalidToken) t; + } else { + throw (IOException) t; + } + } + protected ContainerManagementProtocol getContainerManagementProtocolProxy( final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken, NodeId nodeId, String user) { @@ -351,7 +484,9 @@ public class TestContainerManagerSecurit UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); final InetSocketAddress addr = NetUtils.createSocketAddr(nodeId.getHost(), nodeId.getPort()); - ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr)); + if (nmToken != null) { + ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr)); + } proxy = ugi .doAs(new PrivilegedAction<ContainerManagementProtocol>() { Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml (original) +++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml Mon Aug 12 21:25:49 2013 @@ -19,12 +19,12 @@ <parent> <artifactId>hadoop-yarn-server</artifactId> <groupId>org.apache.hadoop</groupId> - <version>2.2.0-SNAPSHOT</version> + <version>2.3.0-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-web-proxy</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>2.3.0-SNAPSHOT</version> <name>hadoop-yarn-server-web-proxy</name> <properties> Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml (original) +++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml Mon Aug 12 21:25:49 2013 @@ -19,12 +19,12 @@ <parent> <artifactId>hadoop-yarn</artifactId> <groupId>org.apache.hadoop</groupId> - <version>2.2.0-SNAPSHOT</version> + <version>2.3.0-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>2.3.0-SNAPSHOT</version> <name>hadoop-yarn-server</name> <packaging>pom</packaging> Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml (original) +++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml Mon Aug 12 21:25:49 2013 @@ -19,12 +19,12 @@ <parent> <artifactId>hadoop-yarn</artifactId> <groupId>org.apache.hadoop</groupId> - <version>2.2.0-SNAPSHOT</version> + <version>2.3.0-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-site</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>2.3.0-SNAPSHOT</version> <name>hadoop-yarn-site</name> <properties> Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1513258&r1=1513257&r2=1513258&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original) +++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Mon Aug 12 21:25:49 2013 @@ -1119,6 +1119,7 @@ ResourceManager REST API's. * startedTimeEnd - applications with start time ending with this time, specified in ms since epoch * finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch * finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch + * applicationTypes - applications matching the given application types, specified as a comma-separated list. ------ ** Elements of the <apps> (Applications) object Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/pom.xml (original) +++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/pom.xml Mon Aug 12 21:25:49 2013 @@ -16,12 +16,12 @@ <parent> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-project</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>2.3.0-SNAPSHOT</version> <relativePath>../../hadoop-project</relativePath> </parent> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>2.3.0-SNAPSHOT</version> <packaging>pom</packaging> <name>hadoop-yarn</name> Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-yarn-project/pom.xml (original) +++ hadoop/common/branches/YARN-321/hadoop-yarn-project/pom.xml Mon Aug 12 21:25:49 2013 @@ -18,12 +18,12 @@ <parent> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-project</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>2.3.0-SNAPSHOT</version> <relativePath>../hadoop-project</relativePath> </parent> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-project</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>2.3.0-SNAPSHOT</version> <packaging>pom</packaging> <name>hadoop-yarn-project</name> <url>http://hadoop.apache.org/yarn/</url>