Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java?rev=1506393&r1=1506392&r2=1506393&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java Wed Jul 24 03:48:19 2013 @@ -40,8 +40,9 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -227,9 +228,6 @@ public class TestContainersMonitor exten commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); ContainerTokenIdentifier containerIdentifier = new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user, r, System.currentTimeMillis() + 120000, 123, DUMMY_RM_IDENTIFIER); @@ -237,8 +235,14 @@ public class TestContainersMonitor exten BuilderUtils.newContainerToken(context.getNodeId(), containerManager.getContext().getContainerTokenSecretManager() .createPassword(containerIdentifier), containerIdentifier); - startRequest.setContainerToken(containerToken); - containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); int timeoutSecs = 0; while (!processStartFile.exists() && timeoutSecs++ < 20) { @@ -260,11 +264,12 @@ public class TestContainersMonitor exten BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE, 60); - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(cId); + List<ContainerId> containerIds = new ArrayList<ContainerId>(); + containerIds.add(cId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = - containerManager.getContainerStatus(gcsRequest).getStatus(); + containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0); Assert.assertEquals(ExitCode.TERMINATED.getExitCode(), containerStatus.getExitStatus()); String expectedMsgPattern =
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1506393&r1=1506392&r2=1506393&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Wed Jul 24 03:48:19 2013 @@ -22,6 +22,8 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -33,11 +35,15 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; @@ -46,8 +52,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -68,8 +72,6 @@ public class AMLauncher implements Runna private final RMAppAttempt application; private final Configuration conf; - private final RecordFactory recordFactory = - RecordFactoryProvider.getRecordFactory(null); private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; @@ -102,22 +104,42 @@ public class AMLauncher implements Runna + " for AM " + application.getAppAttemptId()); ContainerLaunchContext launchContext = createAMContainerLaunchContext(applicationContext, masterContainerID); - StartContainerRequest request = - recordFactory.newRecordInstance(StartContainerRequest.class); - request.setContainerLaunchContext(launchContext); - request.setContainerToken(masterContainer.getContainerToken()); - containerMgrProxy.startContainer(request); - LOG.info("Done launching container " + masterContainer - + " for AM " + application.getAppAttemptId()); + + StartContainerRequest scRequest = + StartContainerRequest.newInstance(launchContext, + masterContainer.getContainerToken()); + List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + + StartContainersResponse response = + containerMgrProxy.startContainers(allRequests); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(masterContainerID)) { + Throwable t = + response.getFailedRequests().get(masterContainerID).deSerialize(); + parseAndThrowException(t); + } else { + LOG.info("Done launching container " + masterContainer + " for AM " + + application.getAppAttemptId()); + } } private void cleanup() throws IOException, YarnException { connect(); ContainerId containerId = masterContainer.getId(); - StopContainerRequest stopRequest = - recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(containerId); - containerMgrProxy.stopContainer(stopRequest); + List<ContainerId> containerIds = new ArrayList<ContainerId>(); + containerIds.add(containerId); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + StopContainersResponse response = + containerMgrProxy.stopContainers(stopRequest); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(containerId)) { + Throwable t = response.getFailedRequests().get(containerId).deSerialize(); + parseAndThrowException(t); + } } // Protected. For tests. @@ -254,4 +276,15 @@ public class AMLauncher implements Runna break; } } + + private void parseAndThrowException(Throwable t) throws YarnException, + IOException { + if (t instanceof YarnException) { + throw (YarnException) t; + } else if (t instanceof InvalidToken) { + throw (InvalidToken) t; + } else { + throw (IOException) t; + } + } } Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java?rev=1506393&r1=1506392&r2=1506393&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java Wed Jul 24 03:48:19 2013 @@ -35,7 +35,8 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -212,9 +213,11 @@ public class Application { NodeManager nodeManager = task.getNodeManager(); ContainerId containerId = task.getContainerId(); task.stop(); - StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(containerId); - nodeManager.stopContainer(stopRequest); + List<ContainerId> containerIds = new ArrayList<ContainerId>(); + containerIds.add(containerId); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + nodeManager.stopContainers(stopRequest); Resources.subtractFrom(used, requestSpec.get(task.getPriority())); @@ -339,10 +342,15 @@ public class Application { updateResourceRequests(requests.get(priority), type, task); // Launch the container - StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(createCLC()); - startRequest.setContainerToken(container.getContainerToken()); - nodeManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(createCLC(), + container.getContainerToken()); + List<StartContainerRequest> list = + new ArrayList<StartContainerRequest>(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + nodeManager.startContainers(allRequests); break; } } Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1506393&r1=1506392&r2=1506393&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java Wed Jul 24 03:48:19 2013 @@ -31,12 +31,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -158,61 +159,60 @@ public class NodeManager implements Cont } @Override - synchronized public StartContainerResponse startContainer( - StartContainerRequest request) + synchronized public StartContainersResponse startContainers( + StartContainersRequest requests) throws YarnException { - Token containerToken = request.getContainerToken(); - ContainerTokenIdentifier tokenId = null; + for (StartContainerRequest request : requests.getStartContainerRequests()) { + Token containerToken = request.getContainerToken(); + ContainerTokenIdentifier tokenId = null; + + try { + tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); + } - try { - tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); - } catch (IOException e) { - throw RPCUtil.getRemoteException(e); - } + ContainerId containerID = tokenId.getContainerID(); + ApplicationId applicationId = + containerID.getApplicationAttemptId().getApplicationId(); + + List<Container> applicationContainers = containers.get(applicationId); + if (applicationContainers == null) { + applicationContainers = new ArrayList<Container>(); + containers.put(applicationId, applicationContainers); + } - ContainerId containerID = tokenId.getContainerID(); - ApplicationId applicationId = - containerID.getApplicationAttemptId().getApplicationId(); - - List<Container> applicationContainers = containers.get(applicationId); - if (applicationContainers == null) { - applicationContainers = new ArrayList<Container>(); - containers.put(applicationId, applicationContainers); - } - - // Sanity check - for (Container container : applicationContainers) { - if (container.getId().compareTo(containerID) - == 0) { - throw new IllegalStateException( - "Container " + containerID + - " already setup on node " + containerManagerAddress); + // Sanity check + for (Container container : applicationContainers) { + if (container.getId().compareTo(containerID) == 0) { + throw new IllegalStateException("Container " + containerID + + " already setup on node " + containerManagerAddress); + } } - } - Container container = - BuilderUtils.newContainer(containerID, - this.nodeId, nodeHttpAddress, - tokenId.getResource(), - null, null // DKDC - Doesn't matter + Container container = + BuilderUtils.newContainer(containerID, this.nodeId, nodeHttpAddress, + tokenId.getResource(), null, null // DKDC - Doesn't matter ); - ContainerStatus containerStatus = - BuilderUtils.newContainerStatus(container.getId(), ContainerState.NEW, - "", -1000); - applicationContainers.add(container); - containerStatusMap.put(container, containerStatus); - Resources.subtractFrom(available, tokenId.getResource()); - Resources.addTo(used, tokenId.getResource()); - - if(LOG.isDebugEnabled()) { - LOG.debug("startContainer:" + " node=" + containerManagerAddress - + " application=" + applicationId + " container=" + container - + " available=" + available + " used=" + used); - } + ContainerStatus containerStatus = + BuilderUtils.newContainerStatus(container.getId(), + ContainerState.NEW, "", -1000); + applicationContainers.add(container); + containerStatusMap.put(container, containerStatus); + Resources.subtractFrom(available, tokenId.getResource()); + Resources.addTo(used, tokenId.getResource()); + + if (LOG.isDebugEnabled()) { + LOG.debug("startContainer:" + " node=" + containerManagerAddress + + " application=" + applicationId + " container=" + container + + " available=" + available + " used=" + used); + } - StartContainerResponse response = recordFactory.newRecordInstance(StartContainerResponse.class); + } + StartContainersResponse response = + StartContainersResponse.newInstance(null, null, null); return response; } @@ -225,76 +225,79 @@ public class NodeManager implements Cont } @Override - synchronized public StopContainerResponse stopContainer(StopContainerRequest request) + synchronized public StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException { - ContainerId containerID = request.getContainerId(); - String applicationId = String.valueOf( - containerID.getApplicationAttemptId().getApplicationId().getId()); - - // Mark the container as COMPLETE - List<Container> applicationContainers = containers.get(applicationId); - for (Container c : applicationContainers) { - if (c.getId().compareTo(containerID) == 0) { - ContainerStatus containerStatus = containerStatusMap.get(c); - containerStatus.setState(ContainerState.COMPLETE); - containerStatusMap.put(c, containerStatus); + for (ContainerId containerID : request.getContainerIds()) { + String applicationId = + String.valueOf(containerID.getApplicationAttemptId() + .getApplicationId().getId()); + + // Mark the container as COMPLETE + List<Container> applicationContainers = containers.get(applicationId); + for (Container c : applicationContainers) { + if (c.getId().compareTo(containerID) == 0) { + ContainerStatus containerStatus = containerStatusMap.get(c); + containerStatus.setState(ContainerState.COMPLETE); + containerStatusMap.put(c, containerStatus); + } } - } - - // Send a heartbeat - try { - heartbeat(); - } catch (IOException ioe) { - throw RPCUtil.getRemoteException(ioe); - } - - // Remove container and update status - int ctr = 0; - Container container = null; - for (Iterator<Container> i=applicationContainers.iterator(); i.hasNext();) { - container = i.next(); - if (container.getId().compareTo(containerID) == 0) { - i.remove(); - ++ctr; + + // Send a heartbeat + try { + heartbeat(); + } catch (IOException ioe) { + throw RPCUtil.getRemoteException(ioe); } - } - - if (ctr != 1) { - throw new IllegalStateException("Container " + containerID + - " stopped " + ctr + " times!"); - } - - Resources.addTo(available, container.getResource()); - Resources.subtractFrom(used, container.getResource()); - - if(LOG.isDebugEnabled()) { - LOG.debug("stopContainer:" + " node=" + containerManagerAddress - + " application=" + applicationId + " container=" + containerID - + " available=" + available + " used=" + used); - } - StopContainerResponse response = recordFactory.newRecordInstance(StopContainerResponse.class); - return response; + // Remove container and update status + int ctr = 0; + Container container = null; + for (Iterator<Container> i = applicationContainers.iterator(); i + .hasNext();) { + container = i.next(); + if (container.getId().compareTo(containerID) == 0) { + i.remove(); + ++ctr; + } + } + + if (ctr != 1) { + throw new IllegalStateException("Container " + containerID + + " stopped " + ctr + " times!"); + } + + Resources.addTo(available, container.getResource()); + Resources.subtractFrom(used, container.getResource()); + + if (LOG.isDebugEnabled()) { + LOG.debug("stopContainer:" + " node=" + containerManagerAddress + + " application=" + applicationId + " container=" + containerID + + " available=" + available + " used=" + used); + } + } + return StopContainersResponse.newInstance(null,null); } @Override - synchronized public GetContainerStatusResponse getContainerStatus(GetContainerStatusRequest request) throws YarnException { - ContainerId containerId = request.getContainerId(); - List<Container> appContainers = - containers.get( - containerId.getApplicationAttemptId().getApplicationId()); - Container container = null; - for (Container c : appContainers) { - if (c.getId().equals(containerId)) { - container = c; + synchronized public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException { + List<ContainerStatus> statuses = new ArrayList<ContainerStatus>(); + for (ContainerId containerId : request.getContainerIds()) { + List<Container> appContainers = + containers.get(containerId.getApplicationAttemptId() + .getApplicationId()); + Container container = null; + for (Container c : appContainers) { + if (c.getId().equals(containerId)) { + container = c; + } + } + if (container != null + && containerStatusMap.get(container).getState() != null) { + statuses.add(containerStatusMap.get(container)); } } - GetContainerStatusResponse response = - recordFactory.newRecordInstance(GetContainerStatusResponse.class); - if (container != null && containerStatusMap.get(container).getState() != null) { - response.setStatus(containerStatusMap.get(container)); - } - return response; + return GetContainerStatusesResponse.newInstance(statuses, null); } public static org.apache.hadoop.yarn.server.api.records.NodeStatus Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java?rev=1506393&r1=1506392&r2=1506393&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java Wed Jul 24 03:48:19 2013 @@ -36,14 +36,14 @@ import org.apache.hadoop.security.Creden import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -98,23 +98,23 @@ public class TestAMAuthorization { } @Override - public StartContainerResponse - startContainer(StartContainerRequest request) + public StartContainersResponse + startContainers(StartContainersRequest request) throws YarnException { - containerTokens = request.getContainerLaunchContext().getTokens(); - return null; + containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens(); + return StartContainersResponse.newInstance(null, null, null); } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException { - return null; + return StopContainersResponse.newInstance(null, null); } @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException { - return null; + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException { + return GetContainerStatusesResponse.newInstance(null, null); } public Credentials getContainerCredentials() throws IOException { Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java?rev=1506393&r1=1506392&r2=1506393&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java Wed Jul 24 03:48:19 2013 @@ -19,7 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -27,16 +30,18 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -69,9 +74,10 @@ public class TestApplicationMasterLaunch int maxAppAttempts; @Override - public StartContainerResponse - startContainer(StartContainerRequest request) + public StartContainersResponse + startContainers(StartContainersRequest requests) throws YarnException { + StartContainerRequest request = requests.getStartContainerRequests().get(0); LOG.info("Container started by MyContainerManager: " + request); launched = true; Map<String, String> env = @@ -95,11 +101,13 @@ public class TestApplicationMasterLaunch Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV)); maxAppAttempts = Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV)); - return null; + return StartContainersResponse.newInstance( + new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(), + new HashMap<ContainerId, SerializedException>()); } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException { LOG.info("Container cleaned up by MyContainerManager"); cleanedup = true; @@ -107,11 +115,10 @@ public class TestApplicationMasterLaunch } @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException { + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException { return null; } - } @Test Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java?rev=1506393&r1=1506392&r2=1506393&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java Wed Jul 24 03:48:19 2013 @@ -19,7 +19,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.security; import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.IOException; import java.lang.annotation.Annotation; @@ -50,6 +52,8 @@ import org.apache.hadoop.yarn.api.Contai import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.event.Dispatcher; @@ -158,6 +162,9 @@ public class TestClientToAMTokens { ContainerManagementProtocol containerManager = mock(ContainerManagementProtocol.class); + StartContainersResponse mockResponse = mock(StartContainersResponse.class); + when(containerManager.startContainers((StartContainersRequest) any())) + .thenReturn(mockResponse); final DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) { Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1506393&r1=1506392&r2=1506393&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Wed Jul 24 03:48:19 2013 @@ -23,6 +23,8 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.net.InetSocketAddress; import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.List; import junit.framework.Assert; @@ -32,15 +34,20 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -328,18 +335,21 @@ public class TestContainerManagerSecurit ContainerId containerId, ApplicationAttemptId appAttemptId, NodeId nodeId, boolean isExceptionExpected) throws Exception { - GetContainerStatusRequest request = - Records.newRecord(GetContainerStatusRequest.class); - request.setContainerId(containerId); - + List<ContainerId> containerIds = new ArrayList<ContainerId>(); + containerIds.add(containerId); + GetContainerStatusesRequest request = + GetContainerStatusesRequest.newInstance(containerIds); ContainerManagementProtocol proxy = null; - try { proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.toString()); - proxy.getContainerStatus(request); - + GetContainerStatusesResponse statuses = proxy.getContainerStatuses(request); + if (statuses.getFailedRequests() != null + && statuses.getFailedRequests().containsKey(containerId)) { + parseAndThrowException(statuses.getFailedRequests().get(containerId) + .deSerialize()); + } } finally { if (proxy != null) { rpc.stopProxy(proxy, conf); @@ -352,17 +362,21 @@ public class TestContainerManagerSecurit org.apache.hadoop.yarn.api.records.Token containerToken, NodeId nodeId, String user) throws Exception { - StartContainerRequest request = - Records.newRecord(StartContainerRequest.class); - request.setContainerToken(containerToken); ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); - request.setContainerLaunchContext(context); - + StartContainerRequest scRequest = + StartContainerRequest.newInstance(context,containerToken); + List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); ContainerManagementProtocol proxy = null; try { proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user); - proxy.startContainer(request); + StartContainersResponse response = proxy.startContainers(allRequests); + for(SerializedException ex : response.getFailedRequests().values()){ + parseAndThrowException(ex.deSerialize()); + } } finally { if (proxy != null) { rpc.stopProxy(proxy, conf); @@ -370,6 +384,17 @@ public class TestContainerManagerSecurit } } + private void parseAndThrowException(Throwable t) throws YarnException, + IOException { + if (t instanceof YarnException) { + throw (YarnException) t; + } else if (t instanceof InvalidToken) { + throw (InvalidToken) t; + } else { + throw (IOException) t; + } + } + protected ContainerManagementProtocol getContainerManagementProtocolProxy( final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken, NodeId nodeId, String user) {
