Repository: oozie Updated Branches: refs/heads/master 53edaaf61 -> 75879a21b
OOZIE-2897 LauncherAM should support ACLs (pbacsko) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/75879a21 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/75879a21 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/75879a21 Branch: refs/heads/master Commit: 75879a21be330132fd163cc6004075ad4a1c3a6f Parents: 53edaaf Author: Peter Bacsko <pbac...@cloudera.com> Authored: Tue Nov 7 17:37:31 2017 +0100 Committer: Peter Bacsko <pbac...@cloudera.com> Committed: Tue Nov 7 17:37:31 2017 +0100 ---------------------------------------------------------------------- client/src/main/resources/oozie-common-1.0.xsd | 2 + .../oozie/action/hadoop/JavaActionExecutor.java | 8 +- .../oozie/action/hadoop/YarnACLHandler.java | 62 +++++++++ .../workflow/lite/LauncherConfigHandler.java | 5 + core/src/main/resources/oozie-default.xml | 18 +++ .../action/hadoop/ActionExecutorTestCase.java | 1 - .../oozie/action/hadoop/LauncherMainTester.java | 10 ++ .../action/hadoop/TestJavaActionExecutor.java | 46 +++++-- .../TestLauncherConfigurationInjector.java | 6 + .../oozie/action/hadoop/TestYarnACLHandler.java | 127 +++++++++++++++++++ .../apache/oozie/service/TestSchemaService.java | 2 + .../java/org/apache/oozie/test/XTestCase.java | 40 +++++- .../lite/TestLiteWorkflowAppParser.java | 3 + .../src/test/resources/fair-scheduler-alloc.xml | 41 ++++++ .../resources/wf-schema-global-launcherconf.xml | 2 + release-log.txt | 1 + 16 files changed, 356 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/client/src/main/resources/oozie-common-1.0.xsd ---------------------------------------------------------------------- diff --git a/client/src/main/resources/oozie-common-1.0.xsd b/client/src/main/resources/oozie-common-1.0.xsd index ddae912..79e026d 100644 --- a/client/src/main/resources/oozie-common-1.0.xsd +++ b/client/src/main/resources/oozie-common-1.0.xsd @@ -27,6 +27,8 @@ <xs:element name="env" minOccurs="0" type="xs:string"/> <xs:element name="queue" minOccurs="0" type="xs:string"/> <xs:element name="sharelib" minOccurs="0" type="xs:string"/> + <xs:element name="view-acl" minOccurs="0" type="xs:string"/> + <xs:element name="modify-acl" minOccurs="0" type="xs:string"/> </xs:choice> </xs:complexType> http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 6a25e5c..b49ee78 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -55,6 +55,7 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.TaskLog; +import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.AccessControlException; @@ -64,6 +65,7 @@ import org.apache.hadoop.util.DiskChecker; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -131,6 +133,8 @@ public class JavaActionExecutor extends ActionExecutor { public static final String DEFAULT_LAUNCHER_PRIORITY = "oozie.launcher.default.priority"; public static final String DEFAULT_LAUNCHER_QUEUE = "oozie.launcher.default.queue"; public static final String DEFAULT_LAUNCHER_MAX_ATTEMPS = "oozie.launcher.default.max.attempts"; + public static final String LAUNCER_MODIFY_ACL = "oozie.launcher.modify.acl"; + public static final String LAUNCER_VIEW_ACL = "oozie.launcher.view.acl"; public static final String MAX_EXTERNAL_STATS_SIZE = "oozie.external.stats.max.size"; public static final String ACL_VIEW_JOB = "mapreduce.job.acl-view-job"; @@ -1099,6 +1103,8 @@ public class JavaActionExecutor extends ActionExecutor { setMaxAttempts(launcherJobConf, appContext); ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); + YarnACLHandler yarnACL = new YarnACLHandler(launcherJobConf); + yarnACL.setACLs(amContainer); final String user = actionContext.getWorkflow().getUser(); // Set the resources to localize @@ -1876,4 +1882,4 @@ public class JavaActionExecutor extends ActionExecutor { private String getActionTypeLauncherPrefix() { return "oozie.action." + getType() + ".launcher."; } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/main/java/org/apache/oozie/action/hadoop/YarnACLHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/YarnACLHandler.java b/core/src/main/java/org/apache/oozie/action/hadoop/YarnACLHandler.java new file mode 100644 index 0000000..46e621f --- /dev/null +++ b/core/src/main/java/org/apache/oozie/action/hadoop/YarnACLHandler.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.MRConfig; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.oozie.util.XLog; + +class YarnACLHandler { + private XLog LOG = XLog.getLog(getClass()); + private final Configuration launcherConf; + + public YarnACLHandler(Configuration launcherConf) { + this.launcherConf = launcherConf; + } + + public void setACLs(ContainerLaunchContext containerLaunchContext) { + boolean mrAclsEnabled = launcherConf.getBoolean(MRConfig.MR_ACLS_ENABLED, true); + + // This is checked for backward compatibility + if (mrAclsEnabled) { + Map<ApplicationAccessType, String> aclDefinition = new HashMap<>(); + + String viewAcl = launcherConf.get(JavaActionExecutor.LAUNCER_VIEW_ACL); + if (viewAcl != null) { + aclDefinition.put(ApplicationAccessType.VIEW_APP, viewAcl); + } + + String modifyAcl = launcherConf.get(JavaActionExecutor.LAUNCER_MODIFY_ACL); + if (modifyAcl != null) { + aclDefinition.put(ApplicationAccessType.MODIFY_APP, modifyAcl); + } + + if (!aclDefinition.isEmpty()) { + containerLaunchContext.setApplicationACLs(aclDefinition); + } + } else { + LOG.info("Not setting ACLs because {0} is set to false", MRConfig.MR_ACLS_ENABLED); + } + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/main/java/org/apache/oozie/workflow/lite/LauncherConfigHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/workflow/lite/LauncherConfigHandler.java b/core/src/main/java/org/apache/oozie/workflow/lite/LauncherConfigHandler.java index c367742..a6b8585 100644 --- a/core/src/main/java/org/apache/oozie/workflow/lite/LauncherConfigHandler.java +++ b/core/src/main/java/org/apache/oozie/workflow/lite/LauncherConfigHandler.java @@ -18,6 +18,7 @@ package org.apache.oozie.workflow.lite; import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.action.hadoop.JavaActionExecutor; import org.apache.oozie.action.hadoop.LauncherAM; import org.jdom.Element; import org.jdom.Namespace; @@ -30,6 +31,8 @@ class LauncherConfigHandler { private static final String LAUNCHER_ENV = "env"; private static final String LAUNCHER_SHARELIB = "sharelib"; private static final String LAUNCHER_QUEUE = "queue"; + private static final String LAUNCHER_VIEW_ACL = "view-acl"; + private static final String LAUNCHER_MODIFY_ACL = "modify-acl"; private final Configuration entries; private final Element xmlLauncherElement; @@ -63,5 +66,7 @@ class LauncherConfigHandler { setStringCfgSetting(LAUNCHER_ENV, LauncherAM.OOZIE_LAUNCHER_ENV_PROPERTY); setStringCfgSetting(LAUNCHER_QUEUE, LauncherAM.OOZIE_LAUNCHER_QUEUE_PROPERTY); setStringCfgSetting(LAUNCHER_SHARELIB, LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY); + setStringCfgSetting(LAUNCHER_VIEW_ACL, JavaActionExecutor.LAUNCER_VIEW_ACL); + setStringCfgSetting(LAUNCHER_MODIFY_ACL, JavaActionExecutor.LAUNCER_MODIFY_ACL); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 8285df0..91d2811 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -3217,4 +3217,22 @@ will be the requeue interval for the actions which are waiting for a long time w name. The first one that is found will be used. </description> </property> + + <property> + <name>oozie.launcher.override.view.acl</name> + <value>mapreduce.job.acl-view-job</value> + <description> + A comma separated list of MapReduce v1 and MapReduce v2 properties to override MapReduce View ACL settings. + The first one that is found will be used. + </description> + </property> + + <property> + <name>oozie.launcher.override.modify.acl</name> + <value>mapreduce.job.acl-modify-job</value> + <description> + A comma separated list of MapReduce v1 and MapReduce v2 properties to override MapReduce Modify ACL settings. + The first one that is found will be used. + </description> + </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java index 36b3026..f39bba2 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java @@ -255,7 +255,6 @@ public abstract class ActionExecutorTestCase extends XHCatTestCase { wfConf.set(OozieClient.USER_NAME, getTestUser()); wfConf.set(OozieClient.APP_PATH, appUri.toString()); - WorkflowJobBean workflow = createWorkflow(app, wfConf, protoConf); WorkflowActionBean action = new WorkflowActionBean(); http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java b/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java index 43c71b0..fdca570 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java @@ -84,6 +84,16 @@ public class LauncherMainTester { sm.checkPermission(null, sm.getSecurityContext()); } } + + checkAndSleep(args); + } + + private static void checkAndSleep(String args[]) throws InterruptedException { + if (args.length == 2 && args[0].equals("sleep")) { + long sleepTime = Long.parseLong(args[1]); + + Thread.sleep(sleepTime); + } } } http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index adb2168..1cc569d 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -1656,6 +1656,38 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertNotSame(conf.get(JavaActionExecutor.ACL_MODIFY_JOB), actionConf.get(JavaActionExecutor.ACL_MODIFY_JOB)); } + // Only "dummyuser" can kill the app, but we try "test2" instead + public void testCannotKillActionWhenACLSpecified() throws Exception { + // Only "dummyuser" and the owner have the permission to kill the job + String actionXml = "<java>" + + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + + "<name-node>" + getNameNodeUri() + "</name-node>" + + "<configuration>" + + " <property><name>oozie.launcher.modify.acl</name><value>dummyuser</value></property>" + + "</configuration>" + + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + + "<arg>sleep</arg>" + + "<arg>10000</arg>" + + "</java>"; + + // Submitting WF as the default "test" user + Context context = createContext(actionXml, null); + String applicationId = submitAction(context); + waitUntilYarnAppState(applicationId, EnumSet.of(YarnApplicationState.RUNNING)); + + // Modify the user inside the workflow. It's necessary because the Yarn client is always + // retrieved for the owner of the WF and owner always has the permission to kill its + // own application + WorkflowJobBean wfBean = (WorkflowJobBean) context.getWorkflow(); + wfBean.setUser(getTestUser2()); + + JavaActionExecutor jae = new JavaActionExecutor(); + jae.kill(context, context.getAction()); + + // Kill should fail - wait until the application is finished + waitUntilYarnAppDoneAndAssertSuccess(applicationId); + } + public void testParseJobXmlAndConfiguration() throws Exception { String str = "<java>" + "<job-xml>job1.xml</job-xml>" @@ -2554,22 +2586,19 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { "</java>"; final Context context = createContext(actionXml, null); - submitAction(context); + String applicationId = submitAction(context); final ApplicationId appId = ConverterUtils.toApplicationId(context.getAction().getExternalId()); final Configuration conf = getHadoopAccessorService().createConfiguration(getJobTrackerUri()); final String queue = getHadoopAccessorService().createYarnClient(getTestUser(), conf) .getApplicationReport(appId).getQueue(); - assertEquals("queue name", "default1", queue); - final ActionExecutor ae = new JavaActionExecutor(); - ae.check(context, context.getAction()); - assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); - assertNull(context.getAction().getData()); + if (isFairSchedulerUsed(conf)) { + assertEquals("queue name", "root.default1", queue); + } - ae.end(context, context.getAction()); - assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); + waitUntilYarnAppDoneAndAssertSuccess(applicationId); } private HadoopAccessorService getHadoopAccessorService() { @@ -2647,5 +2676,4 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals(FinalApplicationStatus.KILLED, yarnClient.getApplicationReport(ConverterUtils.toApplicationId(runningJob)).getFinalApplicationStatus()); } - } http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherConfigurationInjector.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherConfigurationInjector.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherConfigurationInjector.java index 4c5b0f9..bcc55bf 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherConfigurationInjector.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherConfigurationInjector.java @@ -114,6 +114,8 @@ public class TestLauncherConfigurationInjector extends XTestCase { assertEquals("priority", -1, launcherConf.getInt("priority", -1)); assertTrue("launcher queue", launcherConf.get(LauncherAM.OOZIE_LAUNCHER_QUEUE_PROPERTY).contains("default1")); assertNull("queue", launcherConf.get("queue")); + assertEquals("view ACL", "view", launcherConf.get(JavaActionExecutor.LAUNCER_VIEW_ACL)); + assertEquals("modify ACL", "modify", launcherConf.get(JavaActionExecutor.LAUNCER_MODIFY_ACL)); } public void testMultipleOverrideOrder() { @@ -196,6 +198,8 @@ public class TestLauncherConfigurationInjector extends XTestCase { sourceConf.set("yarn.app.mapreduce.am.env", "PATH=/path1:$PATH"); sourceConf.set("mapreduce.job.priority", "1"); sourceConf.set("mapreduce.job.queuename", "default1"); + sourceConf.set("mapreduce.job.acl-view-job", "view"); + sourceConf.set("mapreduce.job.acl-modify-job", "modify"); return sourceConf; } @@ -222,6 +226,8 @@ public class TestLauncherConfigurationInjector extends XTestCase { sourceConf.set("mapreduce.job.priority", "1"); sourceConf.set("mapred.job.queue.name", "default2"); sourceConf.set("mapreduce.job.queuename", "default1"); + sourceConf.set("mapreduce.job.acl-view-job", "view"); + sourceConf.set("mapreduce.job.acl-modify-job", "modify"); return sourceConf; } http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/test/java/org/apache/oozie/action/hadoop/TestYarnACLHandler.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestYarnACLHandler.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestYarnACLHandler.java new file mode 100644 index 0000000..bcb36d3 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestYarnACLHandler.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.MRConfig; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class TestYarnACLHandler { + private static final String VIEW_ACL_VALUE = "viewacl"; + private static final String MODIFY_ACL_VALUE = "modifyacl"; + + private YarnACLHandler yarnACLSetter; + private Configuration launcherConfig; + + @Mock + private ContainerLaunchContext contextMock; + + @Captor + private ArgumentCaptor<Map<ApplicationAccessType, String>> captor; + + @Test + public void testViewACLisSet() { + setupYarnACLHandler(true); + setViewACL(); + + yarnACLSetter.setACLs(contextMock); + + assertViewACLset(); + } + + @Test + public void testModifyACLisSet() { + setupYarnACLHandler(true); + setModifyACL(); + + yarnACLSetter.setACLs(contextMock); + + assertModifyACLset(); + } + + @Test + public void testACLisEmptyWhenEnabledAndNoACLsDefined() { + setupYarnACLHandler(true); + + yarnACLSetter.setACLs(contextMock); + + assertNoACLset(); + } + + @Test + public void testACLisEmptyWhenMRACLsDisabled() { + setupYarnACLHandler(false); + setModifyACL(); + setViewACL(); + + yarnACLSetter.setACLs(contextMock); + + assertNoACLset(); + } + + private void setupYarnACLHandler(boolean mrACLsEnabled) { + launcherConfig = new Configuration(); + launcherConfig.setBoolean(MRConfig.MR_ACLS_ENABLED, mrACLsEnabled); + + yarnACLSetter = new YarnACLHandler(launcherConfig); + } + + private void setViewACL() { + launcherConfig.set(JavaActionExecutor.LAUNCER_VIEW_ACL, VIEW_ACL_VALUE); + } + + private void setModifyACL() { + launcherConfig.set(JavaActionExecutor.LAUNCER_MODIFY_ACL, MODIFY_ACL_VALUE); + } + + private void assertModifyACLset() { + verifyACLset("Modify ACL", MODIFY_ACL_VALUE, ApplicationAccessType.MODIFY_APP); + } + + private void assertViewACLset() { + verifyACLset("View ACL", VIEW_ACL_VALUE, ApplicationAccessType.VIEW_APP); + } + + @SuppressWarnings("unchecked") + private void assertNoACLset() { + verify(contextMock, never()).setApplicationACLs(any(Map.class)); + } + + private void verifyACLset(String aclMessage, String aclValue, ApplicationAccessType aclType) { + verify(contextMock).setApplicationACLs(captor.capture()); + Map<ApplicationAccessType, String> aclDefinition = captor.getValue(); + assertEquals("ACL size", 1, aclDefinition.size()); + assertEquals(aclMessage, aclValue, aclDefinition.get(aclType)); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/test/java/org/apache/oozie/service/TestSchemaService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestSchemaService.java b/core/src/test/java/org/apache/oozie/service/TestSchemaService.java index 940868a..d0a3fdf 100644 --- a/core/src/test/java/org/apache/oozie/service/TestSchemaService.java +++ b/core/src/test/java/org/apache/oozie/service/TestSchemaService.java @@ -105,6 +105,8 @@ public class TestSchemaService extends XTestCase { " <env>dummyEnv</env>\n" + " <queue>dummyQueue</queue>\n" + " <sharelib>a,b,c</sharelib>\n" + + " <view-acl>oozie</view-acl>\n" + + " <modify-acl>oozie</modify-acl>\n" + " </launcher>\n" + " </global>\n" + " <start to=\"a\"/>\n" + http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/test/java/org/apache/oozie/test/XTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java index 912c3f3..8cecb89 100644 --- a/core/src/test/java/org/apache/oozie/test/XTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java @@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.AppenderSkeleton; import org.apache.log4j.spi.LoggingEvent; @@ -1018,23 +1019,48 @@ public abstract class XTestCase extends TestCase { sb.append(",").append(i.getCanonicalHostName()); } conf.set("hadoop.proxyuser." + getOozieUser() + ".hosts", sb.toString()); - conf.set("hadoop.proxyuser." + getOozieUser() + ".groups", getTestGroup()); conf.set("mapred.tasktracker.map.tasks.maximum", "4"); conf.set("mapred.tasktracker.reduce.tasks.maximum", "4"); - conf.set("hadoop.tmp.dir", "target/test-data"+"/minicluster"); - // Scheduler properties required for YARN CapacityScheduler to work - conf.set("yarn.scheduler.capacity.root.queues", "default"); - conf.set("yarn.scheduler.capacity.root.default.capacity", "100"); - // Required to prevent deadlocks with YARN CapacityScheduler - conf.set("yarn.scheduler.capacity.maximum-am-resource-percent", "0.5"); // Default value is 90 - if you have low disk space, tests will fail. conf.set("yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage", "99"); + configureYarnACL(conf); + return conf; } + /* + * Sets up YARN ACL - necessary for testing application ACLs + * + * If we don't configure queue ACLs, then it's always possible for any + * user to kill a running application. This is not desired, therefore we + * explicitly define what users have the permission to kill applications + * submitted to a given queue. + */ + private void configureYarnACL(JobConf conf) { + conf.set("yarn.acl.enable", "true"); + conf.set("yarn.admin.acl", getOozieUser()); + + String schedClass = conf.get("yarn.resourcemanager.scheduler.class"); + + if (schedClass.contains(FairScheduler.class.getName())) { + conf.set("yarn.scheduler.fair.allocation.file", "fair-scheduler-alloc.xml"); + } + else { + conf.set("yarn.scheduler.capacity.root.acl_administer_queue", getOozieUser()); + conf.set("yarn.scheduler.capacity.root.default.acl_administer_queue", getOozieUser()); + + // Scheduler properties required for YARN CapacityScheduler to work + conf.set("yarn.scheduler.capacity.root.queues", "default,default1"); + conf.set("yarn.scheduler.capacity.root.default.capacity", "50"); + conf.set("yarn.scheduler.capacity.root.default1.capacity", "50"); + // Required to prevent deadlocks with YARN CapacityScheduler + conf.set("yarn.scheduler.capacity.maximum-am-resource-percent", "0.5"); + } + } + protected void setupHCatalogServer() throws Exception { if (hcatServer == null) { hcatServer = new MiniHCatServer(RUNMODE.SERVER, createJobConf()); http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java index a361078..680f4b4 100644 --- a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java +++ b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java @@ -25,6 +25,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.ErrorCode; +import org.apache.oozie.action.hadoop.JavaActionExecutor; import org.apache.oozie.action.hadoop.LauncherAM; import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.LiteWorkflowStoreService; @@ -673,6 +674,8 @@ public class TestLiteWorkflowAppParser extends XTestCase { assertEquals("Queue", "dummyQueue", xconf.get(LauncherAM.OOZIE_LAUNCHER_QUEUE_PROPERTY)); assertEquals("Java opts", "dummyJavaOpts", xconf.get(LauncherAM.OOZIE_LAUNCHER_JAVAOPTS_PROPERTY)); assertEquals("Sharelib", "a,b,c", xconf.get(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY)); + assertEquals("View ACL", "oozieview", xconf.get(JavaActionExecutor.LAUNCER_VIEW_ACL)); + assertEquals("Modify ACL", "ooziemodify", xconf.get(JavaActionExecutor.LAUNCER_MODIFY_ACL)); } public void testParserGlobalLauncherAMOverridden() throws Exception { http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/test/resources/fair-scheduler-alloc.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/fair-scheduler-alloc.xml b/core/src/test/resources/fair-scheduler-alloc.xml new file mode 100644 index 0000000..7170662 --- /dev/null +++ b/core/src/test/resources/fair-scheduler-alloc.xml @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<allocations> + <queue name="root"> + <weight>1.0</weight> + <schedulingPolicy>drf</schedulingPolicy> + <aclAdministerApps>test</aclAdministerApps> + <queue name="default"> + <weight>1.0</weight> + <schedulingPolicy>drf</schedulingPolicy> + </queue> + <queue name="users" type="parent"> + <weight>1.0</weight> + <schedulingPolicy>drf</schedulingPolicy> + </queue> + </queue> + <defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy> + <queuePlacementPolicy> + <rule name="specified" create="true"/> + <rule name="nestedUserQueue"> + <rule name="default" create="true" queue="users"/> + </rule> + <rule name="default"/> + </queuePlacementPolicy> +</allocations> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/core/src/test/resources/wf-schema-global-launcherconf.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/wf-schema-global-launcherconf.xml b/core/src/test/resources/wf-schema-global-launcherconf.xml index 9cd4f6c..b7bc7e9 100644 --- a/core/src/test/resources/wf-schema-global-launcherconf.xml +++ b/core/src/test/resources/wf-schema-global-launcherconf.xml @@ -24,6 +24,8 @@ <env>dummyEnv</env> <queue>dummyQueue</queue> <sharelib>a,b,c</sharelib> + <view-acl>oozieview</view-acl> + <modify-acl>ooziemodify</modify-acl> </launcher> </global> http://git-wip-us.apache.org/repos/asf/oozie/blob/75879a21/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 51a9c9c..08bc721 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-2897 LauncherAM should support ACLs (pbacsko) OOZIE-2962 bump maven-javadoc-plugin to 2.10.4 (dbist13 via andras.piros) OOZIE-3103 Improve log message regarding max concurrency for coordinators (asasvari via rkanter) OOZIE-3102 External Link to "EL Expression Language Quick Reference" not found (asasvari)
