Author: tomwhite Date: Thu Jan 3 15:02:51 2013 New Revision: 1428383 URL: http://svn.apache.org/viewvc?rev=1428383&view=rev Log: Merge -r 1428361:1428362 from trunk to branch-2. Fixes: YARN-288. Fair scheduler queue doesn't accept any jobs when ACLs are configured. Contributed by Sandy Ryza.
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1428383&r1=1428382&r2=1428383&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Thu Jan 3 15:02:51 2013 @@ -141,6 +141,9 @@ Release 2.0.3-alpha - Unreleased YARN-192. Node update causes NPE in the fair scheduler. (Sandy Ryza via tomwhite) + YARN-288. Fair scheduler queue doesn't accept any jobs when ACLs are + configured. (Sandy Ryza via tomwhite) + Release 2.0.2-alpha - 2012-09-07 YARN-9. Rename YARN_HOME to HADOOP_YARN_HOME. (vinodkv via acmurthy) Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java?rev=1428383&r1=1428382&r2=1428383&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java Thu Jan 3 15:02:51 2013 @@ -99,20 +99,6 @@ public class FSParentQueue extends FSQue } } - public boolean hasAccess(QueueACL acl, UserGroupInformation user) { - synchronized (this) { - if (getQueueAcls().get(acl).isUserAllowed(user)) { - return true; - } - } - - if (parent != null) { - return parent.hasAccess(acl, user); - } - - return false; - } - private synchronized QueueUserACLInfo getUserAclInfo( UserGroupInformation user) { QueueUserACLInfo userAclInfo = Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1428383&r1=1428382&r2=1428383&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java Thu Jan 3 15:02:51 2013 @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -118,6 +119,16 @@ public abstract class FSQueue extends Sc return metrics; } + public boolean hasAccess(QueueACL acl, UserGroupInformation user) { + // Check if the leaf-queue allows access + if (queueMgr.getQueueAcls(getName()).get(acl).isUserAllowed(user)) { + return true; + } + + // Check if parent-queue allows access + return parent != null && parent.hasAccess(acl, user); + } + /** * Recomputes the fair shares for all queues and applications * under this queue. Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1428383&r1=1428382&r2=1428383&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Thu Jan 3 15:02:51 2013 @@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.Clock; import org.apache.hadoop.yarn.SystemClock; @@ -494,24 +495,15 @@ public class FairScheduler implements Re new FSSchedulerApp(applicationAttemptId, user, queue, new ActiveUsersManager(getRootQueueMetrics()), rmContext); - - // Enforce ACLs - UserGroupInformation userUgi; - try { - userUgi = UserGroupInformation.getCurrentUser(); - } catch (IOException ioe) { - LOG.info("Failed to get current user information"); - return; - } - // Always a singleton list - List<QueueUserACLInfo> info = queue.getQueueUserAclInfo(userUgi); - if (!info.get(0).getUserAcls().contains(QueueACL.SUBMIT_APPLICATIONS)) { + // Enforce ACLs + UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user); + if (!queue.hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi)) { LOG.info("User " + userUgi.getUserName() + - " cannot submit" + " applications to queue " + queue.getName()); + " cannot submit applications to queue " + queue.getName()); return; } - + queue.addApp(schedulerApp); queue.getMetrics().submitApp(user, applicationAttemptId.getAttemptId()); Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1428383&r1=1428382&r2=1428383&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java Thu Jan 3 15:02:51 2013 @@ -387,6 +387,16 @@ public class QueueManager { queueMaxApps, userMaxApps, queueWeights, userMaxAppsDefault, queueMaxAppsDefault, defaultSchedulingMode, minSharePreemptionTimeouts, queueAcls, fairSharePreemptionTimeout, defaultMinSharePreemptionTimeout); + + // Root queue should have empty ACLs. As a queue's ACL is the union of + // its ACL and all its parents' ACLs, setting the roots' to empty will + // neither allow nor prohibit more access to its children. + Map<QueueACL, AccessControlList> rootAcls = + new HashMap<QueueACL, AccessControlList>(); + rootAcls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(" ")); + rootAcls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(" ")); + queueAcls.put(ROOT_QUEUE, rootAcls); + for (String name: queueNamesInAllocFile) { FSLeafQueue queue = getLeafQueue(name); if (queueModes.containsKey(name)) { Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1428383&r1=1428382&r2=1428383&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Thu Jan 3 15:02:51 2013 @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.io.File; @@ -1243,4 +1245,34 @@ public class TestFairScheduler { Assert.assertEquals(2, liveContainer.getContainer().getPriority().getPriority()); } } + + @Test + public void testAclSubmitApplication() throws Exception { + // Set acl's + Configuration conf = createConfiguration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println("<?xml version=\"1.0\"?>"); + out.println("<allocations>"); + out.println("<queue name=\"queue1\">"); + out.println("<aclSubmitApps>norealuserhasthisname</aclSubmitApps>"); + out.println("</queue>"); + out.println("</allocations>"); + out.close(); + + QueueManager queueManager = scheduler.getQueueManager(); + queueManager.initialize(); + + ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", + "norealuserhasthisname", 1); + ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1", + "norealuserhasthisname2", 1); + + FSSchedulerApp app1 = scheduler.applications.get(attId1); + assertNotNull("The application was not allowed", app1); + FSSchedulerApp app2 = scheduler.applications.get(attId2); + assertNull("The application was allowed", app2); + } }