YARN-4997. Update fair scheduler to use pluggable auth provider (Contributed by Tao Jie via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3befc02 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3befc02 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3befc02 Branch: refs/heads/YARN-5085 Commit: b3befc021b0e2d63d1a3710ea450797d1129f1f5 Parents: 625df87 Author: Daniel Templeton <templ...@apache.org> Authored: Wed Nov 30 09:50:33 2016 -0800 Committer: Daniel Templeton <templ...@apache.org> Committed: Wed Nov 30 09:50:33 2016 -0800 ---------------------------------------------------------------------- .../security/YarnAuthorizationProvider.java | 15 +++++ .../scheduler/fair/AllocationConfiguration.java | 38 +++++------ .../fair/AllocationFileLoaderService.java | 68 +++++++++++++++++--- .../resourcemanager/scheduler/fair/FSQueue.java | 22 +++++-- .../scheduler/fair/FairScheduler.java | 45 +++++++++++-- .../scheduler/fair/TestFairScheduler.java | 4 +- 6 files changed, 149 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3befc02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java index 4b43ea1..9ae4bd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java @@ -28,6 +28,7 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import com.google.common.annotations.VisibleForTesting; import java.util.List; /** @@ -61,6 +62,20 @@ public abstract class YarnAuthorizationProvider { } /** + * Destroy the {@link YarnAuthorizationProvider} instance. + * This method is called only in Tests. + */ + @VisibleForTesting + public static void destroy() { + synchronized (YarnAuthorizationProvider.class) { + if (authorizer != null) { + LOG.debug(authorizer.getClass().getName() + " is destroyed."); + authorizer = null; + } + } + } + + /** * Initialize the provider. Invoked on daemon startup. DefaultYarnAuthorizer is * initialized based on configurations. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3befc02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index c771887..7bd2616 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -25,13 +26,14 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.ReservationACL; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; @@ -69,7 +71,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { private final float queueMaxAMShareDefault; // ACL's for each queue. Only specifies non-default ACL's from configuration. - private final Map<String, Map<QueueACL, AccessControlList>> queueAcls; + private final Map<String, Map<AccessType, AccessControlList>> queueAcls; // Reservation ACL's for each queue. Only specifies non-default ACL's from // configuration. @@ -123,7 +125,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { Map<String, Long> minSharePreemptionTimeouts, Map<String, Long> fairSharePreemptionTimeouts, Map<String, Float> fairSharePreemptionThresholds, - Map<String, Map<QueueACL, AccessControlList>> queueAcls, + Map<String, Map<AccessType, AccessControlList>> queueAcls, Map<String, Map<ReservationACL, AccessControlList>> resAcls, QueuePlacementPolicy placementPolicy, Map<FSQueueType, Set<String>> configuredQueues, @@ -191,9 +193,10 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { * nobody ("") */ public AccessControlList getQueueAcl(String queue, QueueACL operation) { - Map<QueueACL, AccessControlList> queueAcls = this.queueAcls.get(queue); - if (queueAcls != null) { - AccessControlList operationAcl = queueAcls.get(operation); + Map<AccessType, AccessControlList> acls = this.queueAcls.get(queue); + if (acls != null) { + AccessControlList operationAcl = + acls.get(SchedulerUtils.toAccessType(operation)); if (operationAcl != null) { return operationAcl; } @@ -201,6 +204,14 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { return (queue.equals("root")) ? EVERYBODY_ACL : NOBODY_ACL; } + /** + * Get the map of ACLs of all queues. + * @return the map of ACLs of all queues + */ + public Map<String, Map<AccessType, AccessControlList>> getQueueAcls() { + return Collections.unmodifiableMap(this.queueAcls); + } + @Override /** * Get the map of reservation ACLs to {@link AccessControlList} for the @@ -315,21 +326,6 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { return maxChildQueueResources.get(queue); } - public boolean hasAccess(String queueName, QueueACL acl, - UserGroupInformation user) { - int lastPeriodIndex = queueName.length(); - while (lastPeriodIndex != -1) { - String queue = queueName.substring(0, lastPeriodIndex); - if (getQueueAcl(queue, acl).isUserAllowed(user)) { - return true; - } - - lastPeriodIndex = queueName.lastIndexOf('.', lastPeriodIndex - 1); - } - - return false; - } - @VisibleForTesting SchedulingPolicy getSchedulingPolicy(String queueName) { SchedulingPolicy policy = schedulingPolicies.get(queueName); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3befc02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index ee71981..3aecbfd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -41,8 +41,13 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.ReservationACL; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.security.Permission; +import org.apache.hadoop.yarn.security.PrivilegedEntity; +import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.hadoop.yarn.util.resource.Resources; @@ -74,6 +79,12 @@ public class AllocationFileLoaderService extends AbstractService { public static final long THREAD_JOIN_TIMEOUT_MS = 1000; + private static final String ROOT = "root"; + private static final AccessControlList EVERYBODY_ACL = + new AccessControlList("*"); + private static final AccessControlList NOBODY_ACL = + new AccessControlList(" "); + private final Clock clock; private long lastSuccessfulReload; // Last time we successfully reloaded queues @@ -93,7 +104,9 @@ public class AllocationFileLoaderService extends AbstractService { public AllocationFileLoaderService() { this(SystemClock.getInstance()); } - + + private List<Permission> defaultPermissions; + public AllocationFileLoaderService(Clock clock) { super(AllocationFileLoaderService.class.getName()); this.clock = clock; @@ -208,6 +221,7 @@ public class AllocationFileLoaderService extends AbstractService { ParserConfigurationException, SAXException, AllocationConfigurationException { if (allocFile == null) { + reloadListener.onReload(null); return; } LOG.info("Loading allocation file " + allocFile); @@ -224,9 +238,10 @@ public class AllocationFileLoaderService extends AbstractService { Map<String, Long> minSharePreemptionTimeouts = new HashMap<>(); Map<String, Long> fairSharePreemptionTimeouts = new HashMap<>(); Map<String, Float> fairSharePreemptionThresholds = new HashMap<>(); - Map<String, Map<QueueACL, AccessControlList>> queueAcls = new HashMap<>(); + Map<String, Map<AccessType, AccessControlList>> queueAcls = + new HashMap<>(); Map<String, Map<ReservationACL, AccessControlList>> reservationAcls = - new HashMap<>(); + new HashMap<>(); Set<String> reservableQueues = new HashSet<>(); Set<String> nonPreemptableQueues = new HashSet<>(); int userMaxAppsDefault = Integer.MAX_VALUE; @@ -444,7 +459,7 @@ public class AllocationFileLoaderService extends AbstractService { Map<String, Long> minSharePreemptionTimeouts, Map<String, Long> fairSharePreemptionTimeouts, Map<String, Float> fairSharePreemptionThresholds, - Map<String, Map<QueueACL, AccessControlList>> queueAcls, + Map<String, Map<AccessType, AccessControlList>> queueAcls, Map<String, Map<ReservationACL, AccessControlList>> resAcls, Map<FSQueueType, Set<String>> configuredQueues, Set<String> reservableQueues, @@ -468,7 +483,7 @@ public class AllocationFileLoaderService extends AbstractService { queueName = parentName + "." + queueName; } - Map<QueueACL, AccessControlList> acls = new HashMap<>(); + Map<AccessType, AccessControlList> acls = new HashMap<>(); Map<ReservationACL, AccessControlList> racls = new HashMap<>(); NodeList fields = element.getChildNodes(); boolean isLeaf = true; @@ -526,10 +541,10 @@ public class AllocationFileLoaderService extends AbstractService { queuePolicies.put(queueName, policy); } else if ("aclSubmitApps".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData(); - acls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(text)); + acls.put(AccessType.SUBMIT_APP, new AccessControlList(text)); } else if ("aclAdministerApps".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData(); - acls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(text)); + acls.put(AccessType.ADMINISTER_QUEUE, new AccessControlList(text)); } else if ("aclAdministerReservations".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData(); racls.put(ReservationACL.ADMINISTER_RESERVATIONS, @@ -578,6 +593,17 @@ public class AllocationFileLoaderService extends AbstractService { } configuredQueues.get(FSQueueType.PARENT).add(queueName); } + // Set default acls if not defined + // The root queue defaults to all access + for (QueueACL acl : QueueACL.values()) { + AccessType accessType = SchedulerUtils.toAccessType(acl); + if (acls.get(accessType) == null){ + AccessControlList defaultAcl = queueName.equals(ROOT) ? + EVERYBODY_ACL : NOBODY_ACL; + acls.put(accessType, defaultAcl); + } + } + queueAcls.put(queueName, acls); resAcls.put(queueName, racls); if (maxQueueResources.containsKey(queueName) && @@ -590,8 +616,30 @@ public class AllocationFileLoaderService extends AbstractService { minQueueResources.get(queueName))); } } - - public interface Listener { - public void onReload(AllocationConfiguration info); + + /** + * Returns the list of default permissions. + * The default permission for the root queue is everybody ("*") + * and the default permission for all other queues is nobody (""). + * The default permission list would be loaded before the permissions + * from allocation file. + * @return default permission list + */ + protected List<Permission> getDefaultPermissions() { + if (defaultPermissions == null) { + defaultPermissions = new ArrayList<>(); + Map<AccessType, AccessControlList> acls = + new HashMap<>(); + for (QueueACL acl : QueueACL.values()) { + acls.put(SchedulerUtils.toAccessType(acl), EVERYBODY_ACL); + } + defaultPermissions.add(new Permission( + new PrivilegedEntity(EntityType.QUEUE, ROOT), acls)); + } + return defaultPermissions; + } + + interface Listener { + void onReload(AllocationConfiguration info) throws IOException; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3befc02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 38c0340..d87668d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Priority; @@ -37,8 +38,13 @@ import org.apache.hadoop.yarn.api.records.QueueStatistics; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.security.AccessRequest; +import org.apache.hadoop.yarn.security.PrivilegedEntity; +import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType; +import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; @@ -53,6 +59,8 @@ public abstract class FSQueue implements Queue, Schedulable { private Resource steadyFairShare = Resources.createResource(0, 0); private final String name; protected final FairScheduler scheduler; + private final YarnAuthorizationProvider authorizer; + private final PrivilegedEntity queueEntity; private final FSQueueMetrics metrics; protected final FSParentQueue parent; @@ -78,6 +86,9 @@ public abstract class FSQueue implements Queue, Schedulable { public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) { this.name = name; this.scheduler = scheduler; + this.authorizer = + YarnAuthorizationProvider.getInstance(scheduler.getConf()); + this.queueEntity = new PrivilegedEntity(EntityType.QUEUE, name); this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf()); this.parent = parent; } @@ -96,16 +107,16 @@ public abstract class FSQueue implements Queue, Schedulable { public String getName() { return name; } - + @Override public String getQueueName() { return name; } - + public SchedulingPolicy getPolicy() { return policy; } - + public FSParentQueue getParent() { return parent; } @@ -266,7 +277,10 @@ public abstract class FSQueue implements Queue, Schedulable { } public boolean hasAccess(QueueACL acl, UserGroupInformation user) { - return scheduler.getAllocationConfiguration().hasAccess(name, acl, user); + return authorizer.checkPermission( + new AccessRequest(queueEntity, user, + SchedulerUtils.toAccessType(acl), null, null, + Server.getRemoteAddress(), null)); } long getFairSharePreemptionTimeout() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3befc02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 4285e29..fbcac76 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -25,6 +25,8 @@ import java.util.Comparator; import java.util.EnumSet; import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -34,6 +36,7 @@ import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; @@ -53,6 +56,11 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; +import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.security.Permission; +import org.apache.hadoop.yarn.security.PrivilegedEntity; +import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType; +import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; @@ -124,6 +132,7 @@ public class FairScheduler extends private FairSchedulerConfiguration conf; private FSContext context; + private YarnAuthorizationProvider authorizer; private Resource incrAllocation; private QueueManager queueMgr; private boolean usePortForNodeName; @@ -1209,6 +1218,7 @@ public class FairScheduler extends writeLock.lock(); this.conf = new FairSchedulerConfiguration(conf); validateConf(this.conf); + authorizer = YarnAuthorizationProvider.getInstance(conf); minimumAllocation = this.conf.getMinimumAllocation(); initMaximumResourceCapability(this.conf.getMaximumAllocation()); incrAllocation = this.conf.getIncrementAllocation(); @@ -1417,23 +1427,46 @@ public class FairScheduler extends AllocationFileLoaderService.Listener { @Override - public void onReload(AllocationConfiguration queueInfo) { + public void onReload(AllocationConfiguration queueInfo) + throws IOException { // Commit the reload; also create any queue defined in the alloc file // if it does not already exist, so it can be displayed on the web UI. writeLock.lock(); try { - allocConf = queueInfo; - allocConf.getDefaultSchedulingPolicy().initialize(getClusterResource()); - queueMgr.updateAllocationConfiguration(allocConf); - applyChildDefaults(); - maxRunningEnforcer.updateRunnabilityOnReload(); + if (queueInfo == null) { + authorizer.setPermission(allocsLoader.getDefaultPermissions(), + UserGroupInformation.getCurrentUser()); + } else { + allocConf = queueInfo; + setQueueAcls(allocConf.getQueueAcls()); + allocConf.getDefaultSchedulingPolicy().initialize( + getClusterResource()); + queueMgr.updateAllocationConfiguration(allocConf); + applyChildDefaults(); + maxRunningEnforcer.updateRunnabilityOnReload(); + } } finally { writeLock.unlock(); } } } + private void setQueueAcls( + Map<String, Map<AccessType, AccessControlList>> queueAcls) + throws IOException { + authorizer.setPermission(allocsLoader.getDefaultPermissions(), + UserGroupInformation.getCurrentUser()); + List<Permission> permissions = new ArrayList<>(); + for (Entry<String, Map<AccessType, AccessControlList>> queueAcl : queueAcls + .entrySet()) { + permissions.add(new Permission(new PrivilegedEntity(EntityType.QUEUE, + queueAcl.getKey()), queueAcl.getValue())); + } + authorizer.setPermission(permissions, + UserGroupInformation.getCurrentUser()); + } + /** * After reloading the allocation config, the max resource settings for any * ad hoc queues will be missing. This method goes through the queue manager's http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3befc02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index c5ff5e7..55f8849 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -94,10 +95,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdate import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; - import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; - import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; @@ -153,6 +152,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { } QueueMetrics.clearQueueMetrics(); DefaultMetricsSystem.shutdown(); + YarnAuthorizationProvider.destroy(); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org