Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java Wed Jan 23 18:51:24 2013 @@ -18,10 +18,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptStateDataPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationStateDataPBImpl; +@Unstable public class NullRMStateStore extends RMStateStore { @Override @@ -36,7 +38,7 @@ public class NullRMStateStore extends RM @Override public RMState loadState() throws Exception { - return null; + throw new UnsupportedOperationException("Cannot load state from null store"); } @Override
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Wed Jan 23 18:51:24 2013 @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; @@ -49,11 +50,11 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent.RMAppNodeUpdateType; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent.RMAppNodeUpdateType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; @@ -82,7 +83,6 @@ public class RMAppImpl implements RMApp, private final String queue; private final String name; private final ApplicationSubmissionContext submissionContext; - private final String clientTokenStr; private final Dispatcher dispatcher; private final YarnScheduler scheduler; private final ApplicationMasterService masterService; @@ -213,9 +213,9 @@ public class RMAppImpl implements RMApp, public RMAppImpl(ApplicationId applicationId, RMContext rmContext, Configuration config, String name, String user, String queue, - ApplicationSubmissionContext submissionContext, String clientTokenStr, - YarnScheduler scheduler, ApplicationMasterService masterService, - long submitTime) { + ApplicationSubmissionContext submissionContext, + YarnScheduler scheduler, + ApplicationMasterService masterService, long submitTime) { this.applicationId = applicationId; this.name = name; @@ -226,7 +226,6 @@ public class RMAppImpl implements RMApp, this.user = user; this.queue = queue; this.submissionContext = submissionContext; - this.clientTokenStr = clientTokenStr; this.scheduler = scheduler; this.masterService = masterService; this.submitTime = submitTime; @@ -402,7 +401,7 @@ public class RMAppImpl implements RMApp, try { ApplicationAttemptId currentApplicationAttemptId = null; - String clientToken = UNAVAILABLE; + ClientToken clientToken = null; String trackingUrl = UNAVAILABLE; String host = UNAVAILABLE; String origTrackingUrl = UNAVAILABLE; @@ -541,9 +540,9 @@ public class RMAppImpl implements RMApp, appAttemptId.setApplicationId(applicationId); appAttemptId.setAttemptId(attempts.size() + 1); - RMAppAttempt attempt = new RMAppAttemptImpl(appAttemptId, - clientTokenStr, rmContext, scheduler, masterService, - submissionContext, conf); + RMAppAttempt attempt = + new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService, + submissionContext, conf); attempts.put(appAttemptId, attempt); currentAttempt = attempt; if(startAttempt) { Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java Wed Jan 23 18:51:24 2013 @@ -23,10 +23,11 @@ import java.util.Set; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; @@ -91,7 +92,7 @@ public interface RMAppAttempt extends Ev * The token required by the clients to talk to the application attempt * @return the token required by the clients to talk to the application attempt */ - String getClientToken(); + ClientToken getClientToken(); /** * Diagnostics information for the application attempt. Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Wed Jan 23 18:51:24 2013 @@ -39,14 +39,17 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -55,6 +58,7 @@ import org.apache.hadoop.yarn.conf.YarnC import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; @@ -119,7 +123,7 @@ public class RMAppAttemptImpl implements private final WriteLock writeLock; private final ApplicationAttemptId applicationAttemptId; - private final String clientToken; + private ClientToken clientToken; private final ApplicationSubmissionContext submissionContext; //nodes on while this attempt's containers ran @@ -347,11 +351,10 @@ public class RMAppAttemptImpl implements .installTopology(); public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, - String clientToken, RMContext rmContext, YarnScheduler scheduler, + RMContext rmContext, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationSubmissionContext submissionContext, Configuration conf) { - this.conf = conf; this.applicationAttemptId = appAttemptId; this.rmContext = rmContext; @@ -359,7 +362,19 @@ public class RMAppAttemptImpl implements this.submissionContext = submissionContext; this.scheduler = scheduler; this.masterService = masterService; - this.clientToken = clientToken; + + if (UserGroupInformation.isSecurityEnabled()) { + + this.rmContext.getClientToAMTokenSecretManager().registerApplication( + appAttemptId); + + Token<ClientTokenIdentifier> token = + new Token<ClientTokenIdentifier>(new ClientTokenIdentifier( + appAttemptId), this.rmContext.getClientToAMTokenSecretManager()); + this.clientToken = + BuilderUtils.newClientToken(token.getIdentifier(), token.getKind() + .toString(), token.getPassword(), token.getService().toString()); + } ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); @@ -477,7 +492,7 @@ public class RMAppAttemptImpl implements } @Override - public String getClientToken() { + public ClientToken getClientToken() { return this.clientToken; } @@ -963,6 +978,13 @@ public class RMAppAttemptImpl implements appAttempt.rmContext.getAMFinishingMonitor().unregister( appAttempt.getAppAttemptId()); + + // Unregister from the ClientTokenSecretManager + if (UserGroupInformation.isSecurityEnabled()) { + appAttempt.rmContext.getClientToAMTokenSecretManager() + .unRegisterApplication(appAttempt.getAppAttemptId()); + } + if(!appAttempt.submissionContext.getUnmanagedAM()) { // Tell the launcher to cleanup. appAttempt.eventHandler.handle(new AMLauncherEvent( Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Wed Jan 23 18:51:24 2013 @@ -275,7 +275,7 @@ public class AppSchedulable extends Sche // The desired container won't fit here, so reserve reserve(application, priority, node, container, reserved); - return Resources.none(); + return FairScheduler.CONTAINER_RESERVED; } } @@ -307,20 +307,27 @@ public class AppSchedulable extends Sche // (not scheduled) in order to promote better locality. synchronized (app) { for (Priority priority : prioritiesToTry) { + if (app.getTotalRequiredResources(priority) <= 0) { + continue; + } + app.addSchedulingOpportunity(priority); - NodeType allowedLocality = app.getAllowedLocalityLevel(priority, - scheduler.getNumClusterNodes(), scheduler.getNodeLocalityThreshold(), - scheduler.getRackLocalityThreshold()); + ResourceRequest rackLocalRequest = app.getResourceRequest(priority, + node.getRackName()); ResourceRequest localRequest = app.getResourceRequest(priority, node.getHostName()); - if (localRequest != null && localRequest.getNumContainers() != 0) { + + NodeType allowedLocality = app.getAllowedLocalityLevel(priority, + scheduler.getNumClusterNodes(), scheduler.getNodeLocalityThreshold(), + scheduler.getRackLocalityThreshold()); + + if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0 + && localRequest != null && localRequest.getNumContainers() != 0) { return assignContainer(node, app, priority, localRequest, NodeType.NODE_LOCAL, reserved); } - ResourceRequest rackLocalRequest = app.getResourceRequest(priority, - node.getRackName()); if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0 && (allowedLocality.equals(NodeType.RACK_LOCAL) || allowedLocality.equals(NodeType.OFF_SWITCH))) { Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java Wed Jan 23 18:51:24 2013 @@ -177,7 +177,10 @@ public class FSLeafQueue extends FSQueue Collections.sort(appScheds, comparator); for (AppSchedulable sched: appScheds) { if (sched.getRunnable()) { - return sched.assignContainer(node, reserved); + Resource assignedResource = sched.assignContainer(node, reserved); + if (!assignedResource.equals(Resources.none())) { + return assignedResource; + } } } Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Wed Jan 23 18:51:24 2013 @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRejectedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; @@ -109,6 +110,10 @@ public class FairScheduler implements Re private Clock clock; private static final Log LOG = LogFactory.getLog(FairScheduler.class); + + // Value that container assignment methods return when a container is + // reserved + public static final Resource CONTAINER_RESERVED = Resources.createResource(-1); // How often fair shares are re-calculated (ms) protected long UPDATE_INTERVAL = 500; @@ -498,8 +503,11 @@ public class FairScheduler implements Re // Enforce ACLs UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user); if (!queue.hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi)) { - LOG.info("User " + userUgi.getUserName() + - " cannot submit applications to queue " + queue.getName()); + String msg = "User " + userUgi.getUserName() + + " cannot submit applications to queue " + queue.getName(); + LOG.info(msg); + rmContext.getDispatcher().getEventHandler().handle( + new RMAppAttemptRejectedEvent(applicationAttemptId, msg)); return; } Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java Wed Jan 23 18:51:24 2013 @@ -56,7 +56,7 @@ public class FairSchedulerConfiguration /** Whether to assign multiple containers in one check-in. */ protected static final String ASSIGN_MULTIPLE = CONF_PREFIX + "assignmultiple"; - protected static final boolean DEFAULT_ASSIGN_MULTIPLE = true; + protected static final boolean DEFAULT_ASSIGN_MULTIPLE = false; /** Whether to give more weight to apps requiring many resources. */ protected static final String SIZE_BASED_WEIGHT = CONF_PREFIX + "sizebasedweight"; Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java Wed Jan 23 18:51:24 2013 @@ -23,26 +23,29 @@ import java.util.Map; import javax.crypto.SecretKey; -import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager; public class ClientToAMTokenSecretManagerInRM extends BaseClientToAMTokenSecretManager { // Per application master-keys for managing client-tokens - private Map<ApplicationId, SecretKey> masterKeys = - new HashMap<ApplicationId, SecretKey>(); + private Map<ApplicationAttemptId, SecretKey> masterKeys = + new HashMap<ApplicationAttemptId, SecretKey>(); - public synchronized void registerApplication(ApplicationId applicationID) { - this.masterKeys.put(applicationID, generateSecret()); + public synchronized void registerApplication( + ApplicationAttemptId applicationAttemptID) { + this.masterKeys.put(applicationAttemptID, generateSecret()); } - public synchronized void unRegisterApplication(ApplicationId applicationID) { - this.masterKeys.remove(applicationID); + public synchronized void unRegisterApplication( + ApplicationAttemptId applicationAttemptID) { + this.masterKeys.remove(applicationAttemptID); } @Override - public synchronized SecretKey getMasterKey(ApplicationId applicationID) { - return this.masterKeys.get(applicationID); + public synchronized SecretKey getMasterKey( + ApplicationAttemptId applicationAttemptID) { + return this.masterKeys.get(applicationAttemptID); } } Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java Wed Jan 23 18:51:24 2013 @@ -338,7 +338,7 @@ public class TestApplicationACLs { Assert.assertEquals("Enemy should not see app rpc port!", -1, appReport.getRpcPort()); Assert.assertEquals("Enemy should not see app client token!", - UNAVAILABLE, appReport.getClientToken()); + null, appReport.getClientToken()); Assert.assertEquals("Enemy should not see app diagnostics!", UNAVAILABLE, appReport.getDiagnostics()); Assert.assertEquals("Enemy should not see app tracking url!", Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Wed Jan 23 18:51:24 2013 @@ -18,10 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.IOException; import java.net.InetSocketAddress; @@ -276,7 +276,7 @@ public class TestClientRMService { private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, ApplicationId applicationId3, YarnConfiguration config, String queueName) { return new RMAppImpl(applicationId3, rmContext, config, null, null, - queueName, null, null, yarnScheduler, null, System + queueName, null, yarnScheduler, null , System .currentTimeMillis()); } } Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java Wed Jan 23 18:51:24 2013 @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationStatus; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -77,7 +78,7 @@ public abstract class MockAsm extends Mo } @Override - public String getClientToken() { + public ClientToken getClientToken() { throw new UnsupportedOperationException("Not supported yet."); } @@ -127,7 +128,7 @@ public abstract class MockAsm extends Mo } @Override - public void setClientToken(String clientToken) { + public void setClientToken(ClientToken clientToken) { throw new UnsupportedOperationException("Not supported yet."); } Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Wed Jan 23 18:51:24 2013 @@ -168,7 +168,6 @@ public class TestRMAppTransitions { Configuration conf = new YarnConfiguration(); // ensure max retries set to known value conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, maxRetries); - String clientTokenStr = "bogusstring"; YarnScheduler scheduler = mock(YarnScheduler.class); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); @@ -177,11 +176,10 @@ public class TestRMAppTransitions { submissionContext = new ApplicationSubmissionContextPBImpl(); } - RMApp application = new RMAppImpl(applicationId, rmContext, - conf, name, user, - queue, submissionContext, clientTokenStr, - scheduler, - masterService, System.currentTimeMillis()); + RMApp application = + new RMAppImpl(applicationId, rmContext, conf, name, user, queue, + submissionContext, scheduler, masterService, + System.currentTimeMillis()); testAppStartState(applicationId, user, name, queue, application); return application; Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Wed Jan 23 18:51:24 2013 @@ -209,9 +209,9 @@ public class TestRMAppAttemptTransitions unmanagedAM = false; application = mock(RMApp.class); - applicationAttempt = - new RMAppAttemptImpl(applicationAttemptId, null, rmContext, scheduler, - masterService, submissionContext, new Configuration()); + applicationAttempt = + new RMAppAttemptImpl(applicationAttemptId, rmContext, scheduler, + masterService, submissionContext, new Configuration()); when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt); when(application.getApplicationId()).thenReturn(applicationId); Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Wed Jan 23 18:51:24 2013 @@ -41,20 +41,29 @@ import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.Clock; +import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -1275,4 +1284,136 @@ public class TestFairScheduler { FSSchedulerApp app2 = scheduler.applications.get(attId2); assertNull("The application was allowed", app2); } + + @Test + public void testMultipleNodesSingleRackRequest() throws Exception { + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); + RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); + RMNode node3 = MockNodes.newNodeInfo(2, Resources.createResource(1024)); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); + scheduler.handle(nodeEvent2); + + ApplicationAttemptId appId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); + scheduler.addApplication(appId, "queue1", "user1"); + + // 1 request with 2 nodes on the same rack. another request with 1 node on + // a different rack + List<ResourceRequest> asks = new ArrayList<ResourceRequest>(); + asks.add(createResourceRequest(1024, node1.getHostName(), 1, 1)); + asks.add(createResourceRequest(1024, node2.getHostName(), 1, 1)); + asks.add(createResourceRequest(1024, node3.getHostName(), 1, 1)); + asks.add(createResourceRequest(1024, node1.getRackName(), 1, 1)); + asks.add(createResourceRequest(1024, node3.getRackName(), 1, 1)); + asks.add(createResourceRequest(1024, RMNode.ANY, 1, 2)); + + scheduler.allocate(appId, asks, new ArrayList<ContainerId>()); + + // node 1 checks in + scheduler.update(); + NodeUpdateSchedulerEvent updateEvent1 = new NodeUpdateSchedulerEvent(node1, + new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>()); + scheduler.handle(updateEvent1); + // should assign node local + assertEquals(1, scheduler.applications.get(appId).getLiveContainers().size()); + + // node 2 checks in + scheduler.update(); + NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2, + new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>()); + scheduler.handle(updateEvent2); + // should assign rack local + assertEquals(2, scheduler.applications.get(appId).getLiveContainers().size()); + } + + @Test + public void testFifoWithinQueue() throws Exception { + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(3072)); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + // Even if submitted at exact same time, apps will be deterministically + // ordered by name. + ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", + "user1", 2); + ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1", + "user1", 2); + FSSchedulerApp app1 = scheduler.applications.get(attId1); + FSSchedulerApp app2 = scheduler.applications.get(attId2); + + FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1"); + queue1.setSchedulingMode(SchedulingMode.FIFO); + + scheduler.update(); + + // First two containers should go to app 1, third should go to app 2. + // Because tests set assignmultiple to false, each heartbeat assigns a single + // container. + + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1, + new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>()); + + scheduler.handle(updateEvent); + assertEquals(1, app1.getLiveContainers().size()); + assertEquals(0, app2.getLiveContainers().size()); + + scheduler.handle(updateEvent); + assertEquals(2, app1.getLiveContainers().size()); + assertEquals(0, app2.getLiveContainers().size()); + + scheduler.handle(updateEvent); + assertEquals(2, app1.getLiveContainers().size()); + assertEquals(1, app2.getLiveContainers().size()); + } + + + @SuppressWarnings("unchecked") + @Test + public void testNotAllowSubmitApplication() throws Exception { + // Set acl's + Configuration conf = createConfiguration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println("<?xml version=\"1.0\"?>"); + out.println("<allocations>"); + out.println("<queue name=\"queue1\">"); + out.println("<aclSubmitApps>userallow</aclSubmitApps>"); + out.println("</queue>"); + out.println("</allocations>"); + out.close(); + QueueManager queueManager = scheduler.getQueueManager(); + queueManager.initialize(); + + int appId = this.APP_ID++; + String user = "usernotallow"; + String queue = "queue1"; + ApplicationId applicationId = MockApps.newAppID(appId); + String name = MockApps.newAppName(); + ApplicationMasterService masterService = + new ApplicationMasterService(resourceManager.getRMContext(), scheduler); + ApplicationSubmissionContext submissionContext = new ApplicationSubmissionContextPBImpl(); + RMApp application = + new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user, + queue, submissionContext, scheduler, masterService, + System.currentTimeMillis()); + resourceManager.getRMContext().getRMApps().putIfAbsent(applicationId, application); + application.handle(new RMAppEvent(applicationId, RMAppEventType.START)); + + ApplicationAttemptId attId = recordFactory.newRecordInstance(ApplicationAttemptId.class); + attId.setAttemptId(this.ATTEMPT_ID++); + attId.setApplicationId(applicationId); + scheduler.addApplication(attId, queue, user); + + final int MAX_TRIES=20; + int numTries = 0; + while (application.getFinishTime() == 0 && numTries < MAX_TRIES) { + try { + Thread.sleep(100); + } catch (InterruptedException ex) {ex.printStackTrace();} + numTries++; + } + assertEquals(FinalApplicationStatus.FAILED, application.getFinalApplicationStatus()); + } } Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java Wed Jan 23 18:51:24 2013 @@ -52,8 +52,10 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; @@ -67,6 +69,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.service.AbstractService; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.util.ProtoUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.Test; @@ -106,14 +109,14 @@ public class TestClientTokens { private static class CustomAM extends AbstractService implements CustomProtocol { - private final ApplicationId appId; + private final ApplicationAttemptId appAttemptId; private final String secretKey; private InetSocketAddress address; private boolean pinged = false; - public CustomAM(ApplicationId appId, String secretKeyStr) { + public CustomAM(ApplicationAttemptId appId, String secretKeyStr) { super("CustomAM"); - this.appId = appId; + this.appAttemptId = appId; this.secretKey = secretKeyStr; } @@ -128,7 +131,7 @@ public class TestClientTokens { ClientToAMTokenSecretManager secretManager = null; byte[] bytes = Base64.decodeBase64(this.secretKey); - secretManager = new ClientToAMTokenSecretManager(this.appId, bytes); + secretManager = new ClientToAMTokenSecretManager(this.appAttemptId, bytes); Server server; try { server = @@ -216,7 +219,7 @@ public class TestClientTokens { GetApplicationReportResponse reportResponse = rm.getClientRMService().getApplicationReport(request); ApplicationReport appReport = reportResponse.getApplicationReport(); - String clientTokenEncoded = appReport.getClientToken(); + ClientToken clientToken = appReport.getClientToken(); // Wait till AM is 'launched' int waitTime = 0; @@ -226,9 +229,11 @@ public class TestClientTokens { Assert.assertNotNull(containerManager.clientTokensSecret); // Start the AM with the correct shared-secret. + ApplicationAttemptId appAttemptId = + app.getAppAttempts().keySet().iterator().next(); + Assert.assertNotNull(appAttemptId); final CustomAM am = - new CustomAM(app.getApplicationId(), - containerManager.clientTokensSecret); + new CustomAM(appAttemptId, containerManager.clientTokensSecret); am.init(conf); am.start(); @@ -249,21 +254,19 @@ public class TestClientTokens { // Verify denial for a malicious user UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me"); - Token<ClientTokenIdentifier> clientToken = - new Token<ClientTokenIdentifier>(); - clientToken.decodeFromUrlString(clientTokenEncoded); - // RPC layer client expects ip:port as service for tokens - SecurityUtil.setTokenService(clientToken, am.address); + Token<ClientTokenIdentifier> token = + ProtoUtils.convertFromProtoFormat(clientToken, am.address); // Malicious user, messes with appId ClientTokenIdentifier maliciousID = - new ClientTokenIdentifier(BuilderUtils.newApplicationId(app - .getApplicationId().getClusterTimestamp(), 42)); + new ClientTokenIdentifier(BuilderUtils.newApplicationAttemptId( + BuilderUtils.newApplicationId(app.getApplicationId() + .getClusterTimestamp(), 42), 43)); Token<ClientTokenIdentifier> maliciousToken = new Token<ClientTokenIdentifier>(maliciousID.getBytes(), - clientToken.getPassword(), clientToken.getKind(), - clientToken.getService()); + token.getPassword(), token.getKind(), + token.getService()); ugi.addToken(maliciousToken); try { @@ -297,7 +300,7 @@ public class TestClientTokens { // Now for an authenticated user ugi = UserGroupInformation.createRemoteUser("me"); - ugi.addToken(clientToken); + ugi.addToken(token); ugi.doAs(new PrivilegedExceptionAction<Void>() { @Override Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1437623&r1=1437622&r2=1437623&view=diff ============================================================================== --- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original) +++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Wed Jan 23 18:51:24 2013 @@ -132,20 +132,45 @@ Hadoop MapReduce Next Generation - Fair * Whether to use the username associated with the allocation as the default queue name, in the event that a queue name is not specified. If this is set to "false" or unset, all jobs have a shared default queue, called "default". + Defaults to true. * <<<yarn.scheduler.fair.preemption>>> * Whether to use preemption. Note that preemption is experimental in the current - version. + version. Defaults to false. * <<<yarn.scheduler.fair.sizebasedweight>>> * Whether to assign shares to individual apps based on their size, rather than - providing an equal share to all apps regardless of size. + providing an equal share to all apps regardless of size. Defaults to false. * <<<yarn.scheduler.fair.assignmultiple>>> - * Whether to allow multiple container assignments in one heartbeat. + * Whether to allow multiple container assignments in one heartbeat. Defaults + to false. + + * <<<yarn.scheduler.fair.max.assign>>> + + * If assignmultiple is true, the maximum amount of containers that can be + assigned in one heartbeat. Defaults to -1, which sets no limit. + + * <<<locality.threshold.node>>> + + * For applications that request containers on particular nodes, the number of + scheduling opportunities since the last container assignment to wait before + accepting a placement on another node. Expressed as a float between 0 and 1, + which, as a fraction of the cluster size, is the number of scheduling + opportunities to pass up. The default value of -1.0 means don't pass up any + scheduling opportunities. + + * <<<locality.threshold.rack>>> + + * For applications that request containers on particular racks, the number of + scheduling opportunities since the last container assignment to wait before + accepting a placement on another rack. Expressed as a float between 0 and 1, + which, as a fraction of the cluster size, is the number of scheduling + opportunities to pass up. The default value of -1.0 means don't pass up any + scheduling opportunities. Allocation file format @@ -166,6 +191,14 @@ Allocation file format * schedulingMode: either "fifo" or "fair" depending on the in-queue scheduling policy desired + * aclSubmitApps: a list of users that can submit apps to the queue. A (default) + value of "*" means that any users can submit apps. A queue inherits the ACL of + its parent, so if a queue2 descends from queue1, and user1 is in queue1's ACL, + and user2 is in queue2's ACL, then both users may submit to queue2. + + * minSharePreemptionTimeout: number of seconds the queue is under its minimum share + before it will try to preempt containers to take resources from other queues. + * <<User elements>>, which represent settings governing the behavior of individual users. They can contain a single property: maxRunningApps, a limit on the number of running apps for a particular user. @@ -173,6 +206,10 @@ Allocation file format * <<A userMaxAppsDefault element>>, which sets the default running app limit for any users whose limit is not otherwise specified. + * <<A fairSharePreemptionTimeout element>>, number of seconds a queue is under + its fair share before it will try to preempt containers to take resources from + other queues. + An example allocation file is given here: ---