This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 10461e0 YARN-10148. Add Unit test for queue ACL for both FS and CS. Contributed by Kinga Marton 10461e0 is described below commit 10461e01932bcd82a9d4e3ab8109df7ead560b14 Author: Szilard Nemeth <snem...@apache.org> AuthorDate: Thu Feb 27 20:07:32 2020 +0100 YARN-10148. Add Unit test for queue ACL for both FS and CS. Contributed by Kinga Marton --- .../yarn/server/resourcemanager/ACLsTestBase.java | 4 + .../server/resourcemanager/QueueACLsTestBase.java | 146 +++++++++++++++++++++ .../capacity/TestCapacitySchedulerQueueACLs.java | 75 ++++++++++- .../scheduler/fair/TestFairSchedulerQueueACLs.java | 62 ++++++++- 4 files changed, 283 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java index ddebaaa..a011c8a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java @@ -113,5 +113,9 @@ public abstract class ACLsTestBase { return userClient; } + public Configuration getConf() { + return conf; + } + protected abstract Configuration createConfiguration() throws IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java index 82b3e24..ae9487e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java @@ -18,9 +18,12 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.QueueACL; import org.junit.Assert; import org.apache.hadoop.security.authorize.AccessControlList; @@ -43,6 +46,19 @@ import org.junit.Test; public abstract class QueueACLsTestBase extends ACLsTestBase { + protected static final String QUEUED = "D"; + protected static final String QUEUED1 = "D1"; + private static final String ALL_ACL = "*"; + private static final String NONE_ACL = " "; + + + abstract public String getQueueD(); + + abstract public String getQueueD1(); + + abstract public void updateConfigWithDAndD1Queues(String rootAcl, + String queueDAcl, String queueD1Acl) throws IOException; + @After public void tearDown() { if (resourceManager != null) { @@ -75,6 +91,136 @@ public abstract class QueueACLsTestBase extends ACLsTestBase { } + /** + * Test for the case when the following submit application + * and administer queue ACLs are defined: + * root: (none) + * D: * (all) + * D1: * (all) + * Expected result: the user will have access only to D and D1 queues. + * @throws IOException + */ + @Test + public void testQueueAclRestrictedRootACL() throws IOException { + updateConfigWithDAndD1Queues(NONE_ACL, ALL_ACL, ALL_ACL); + checkAccess(false, true, true); + } + + /** + * Test for the case when the following submit application + * and administer queue ACLs are defined: + * root: (none) + * D: (none) + * D1: (none) + * Expected result: the user will have to none of the queues. + * @throws IOException + */ + @Test + public void testQueueAclNoAccess() throws IOException { + updateConfigWithDAndD1Queues(NONE_ACL, NONE_ACL, NONE_ACL); + checkAccess(false, false, false); + } + + /** + * Test for the case when the following submit application + * and administer queue ACLs are defined: + * root: (none) + * D: * (all) + * D1: (none) + * Expected result: access to D1 will be permitted by root.D, + * so the user will be able to access queues D and D1. + * @throws IOException + */ + @Test + public void testQueueAclRestrictedRootAndD1() throws IOException { + updateConfigWithDAndD1Queues(NONE_ACL, ALL_ACL, NONE_ACL); + checkAccess(false, true, true); + } + + /** + * Test for the case when the following submit application + * and administer queue ACLs are defined: + * root: (none) + * D: (none) + * D1: (all) + * Expected result: only queue D1 can be accessed. + * @throws IOException + */ + @Test + public void testQueueAclRestrictedRootAndD() throws IOException { + updateConfigWithDAndD1Queues(NONE_ACL, NONE_ACL, ALL_ACL); + checkAccess(false, false, true); + } + + /** + * Test for the case when the following submit application + * and administer queue ACLs are defined: + * root: * (all) + * D: (none) + * D1: * (all) + * Expected result: access to D will be permitted from the root queue, + * so the user will be able to access queues root, D and D1. + * @throws IOException + */ + @Test + public void testQueueAclRestrictedD() throws IOException { + updateConfigWithDAndD1Queues(ALL_ACL, NONE_ACL, ALL_ACL); + checkAccess(true, true, true); + } + + /** + * Test for the case when the following submit application + * and administer queue ACLs are defined: + * root: * (all) + * D: * (all) + * D1: (none) + * Expected result: access to D1 will be permitted from queue D, + * so the user will be able to access queues root, D and D1. + * @throws IOException + */ + @Test + public void testQueueAclRestrictedD1() throws IOException { + updateConfigWithDAndD1Queues(ALL_ACL, ALL_ACL, NONE_ACL); + checkAccess(true, true, true); + } + + /** + * Test for the case when no ACLs are defined, so the default values are used + * Expected result: The default ACLs for the root queue is "*"(all) and for + * the other queues are " " (none), so the user will have access to all the + * queues because they will have permissions from the root. + * + * @throws IOException + */ + @Test + public void testQueueAclDefaultValues() throws IOException { + updateConfigWithDAndD1Queues(null, null, null); + checkAccess(true, true, true); + } + + private void checkAccess(boolean rootAccess, boolean dAccess, + boolean d1Access)throws IOException { + checkAccess(rootAccess, "root"); + checkAccess(dAccess, getQueueD()); + checkAccess(d1Access, getQueueD1()); + } + + + private void checkAccess(boolean access, String queueName) + throws IOException { + UserGroupInformation user = UserGroupInformation.getCurrentUser(); + + String failureMsg = "Wrong %s access to %s queue"; + Assert.assertEquals( + String.format(failureMsg, QueueACL.ADMINISTER_QUEUE, queueName), + access, resourceManager.getResourceScheduler() + .checkAccess(user, QueueACL.ADMINISTER_QUEUE, queueName)); + Assert.assertEquals( + String.format(failureMsg, QueueACL.SUBMIT_APPLICATIONS, queueName), + access, resourceManager.getResourceScheduler() + .checkAccess(user, QueueACL.SUBMIT_APPLICATIONS, queueName)); + } + private void verifyGetClientAMToken(String submitter, String queueAdmin, String queueName, boolean setupACLs) throws Exception { ApplicationId applicationId = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java index 5feb94b..eeb2955 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -34,8 +35,10 @@ public class TestCapacitySchedulerQueueACLs extends QueueACLsTestBase { csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { QUEUEA, QUEUEB }); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, 50f); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, 50f); + setQueueCapacity(csConf, 50, + CapacitySchedulerConfiguration.ROOT + "." + QUEUEA); + setQueueCapacity(csConf, 50, + CapacitySchedulerConfiguration.ROOT + "." + QUEUEB); Map<QueueACL, AccessControlList> aclsOnQueueA = new HashMap<QueueACL, AccessControlList>(); @@ -71,4 +74,72 @@ public class TestCapacitySchedulerQueueACLs extends QueueACLsTestBase { return csConf; } + + @Override + public String getQueueD() { + return QUEUED; + } + + @Override + public String getQueueD1() { + return QUEUED1; + } + + /** + * Updates the configuration with the following queue hierarchy: + * root + * | + * D + * | + * D1. + * @param rootAcl administer queue and submit application ACL for root queue + * @param queueDAcl administer queue and submit application ACL for D queue + * @param queueD1Acl administer queue and submit application ACL for D1 queue + * @throws IOException + */ + @Override + public void updateConfigWithDAndD1Queues(String rootAcl, String queueDAcl, + String queueD1Acl) throws IOException { + CapacitySchedulerConfiguration csConf = + (CapacitySchedulerConfiguration) getConf(); + csConf.clear(); + csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] {QUEUED, QUEUEA, QUEUEB}); + + String dPath = CapacitySchedulerConfiguration.ROOT + "." + QUEUED; + String d1Path = dPath + "." + QUEUED1; + csConf.setQueues(dPath, new String[] {QUEUED1}); + setQueueCapacity(csConf, 100, d1Path); + setQueueCapacity(csConf, 30, CapacitySchedulerConfiguration.ROOT + + "." + QUEUEA); + setQueueCapacity(csConf, 50, CapacitySchedulerConfiguration.ROOT + + "." + QUEUEB); + setQueueCapacity(csConf, 20, dPath); + + if (rootAcl != null) { + setAdminAndSubmitACL(csConf, rootAcl, + CapacitySchedulerConfiguration.ROOT); + } + + if (queueDAcl != null) { + setAdminAndSubmitACL(csConf, queueDAcl, dPath); + } + + if (queueD1Acl != null) { + setAdminAndSubmitACL(csConf, d1Path, queueD1Acl); + } + resourceManager.getResourceScheduler() + .reinitialize(csConf, resourceManager.getRMContext()); + } + + private void setQueueCapacity(CapacitySchedulerConfiguration csConf, + float capacity, String queuePath) { + csConf.setCapacity(queuePath, capacity); + } + + private void setAdminAndSubmitACL(CapacitySchedulerConfiguration csConf, + String queueAcl, String queuePath) { + csConf.setAcl(queuePath, QueueACL.ADMINISTER_QUEUE, queueAcl); + csConf.setAcl(queuePath, QueueACL.SUBMIT_APPLICATIONS, queueAcl); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java index ad56a20..237f637 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java @@ -18,8 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import java.io.File; +import java.io.IOException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.QueueACLsTestBase; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair @@ -32,8 +34,8 @@ public class TestFairSchedulerQueueACLs extends QueueACLsTestBase { protected Configuration createConfiguration() { FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration(); - final String testDir = new File(System.getProperty("test.build.data", - "/tmp")).getAbsolutePath(); + final String testDir = new File(System.getProperty( + GenericTestUtils.SYSPROP_TEST_DATA_DIR, "/tmp")).getAbsolutePath(); final String allocFile = new File(testDir, "test-queues.xml") .getAbsolutePath(); @@ -57,4 +59,60 @@ public class TestFairSchedulerQueueACLs extends QueueACLsTestBase { return fsConf; } + + @Override + public String getQueueD() { + return "root." + QUEUED; + } + + @Override + public String getQueueD1() { + return "root."+ QUEUED + "." + QUEUED1; + } + + /** + * Creates the following queue hierarchy: + * root + * | + * D + * | + * D1. + * @param rootAcl administer queue and submit application ACL for root queue + * @param queueDAcl administer queue and submit application ACL for D queue + * @param queueD1Acl administer queue and submit application ACL for D1 queue + * @throws IOException + */ + @Override + public void updateConfigWithDAndD1Queues(String rootAcl, String queueDAcl, + String queueD1Acl) throws IOException { + FairSchedulerConfiguration fsConf = (FairSchedulerConfiguration) getConf(); + fsConf.clear(); + final String testDir = new File(System.getProperty( + GenericTestUtils.SYSPROP_TEST_DATA_DIR, "/tmp")).getAbsolutePath(); + final String allocFile = new File(testDir, "test-queues.xml") + .getAbsolutePath(); + + AllocationFileWriter.create() + .addQueue(new AllocationFileQueue.Builder("root") + .aclSubmitApps(rootAcl) + .aclAdministerApps(rootAcl) + .subQueue(new AllocationFileQueue.Builder(QUEUED) + .aclAdministerApps(queueDAcl) + .aclSubmitApps(queueDAcl) + .subQueue(new AllocationFileQueue.Builder(QUEUED1) + .aclSubmitApps(queueD1Acl) + .aclAdministerApps(queueD1Acl) + .build()) + .build()) + .build()) + .writeToFile(allocFile); + + fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); + + fsConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + fsConf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName()); + resourceManager.getResourceScheduler() + .reinitialize(fsConf, resourceManager.getRMContext()); + + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org