Repository: hadoop Updated Branches: refs/heads/trunk cc3f3eca4 -> d00b6f7c1
YARN-7286. Add support for docker to have no capabilities. Contributed by Eric Badger Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d00b6f7c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d00b6f7c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d00b6f7c Branch: refs/heads/trunk Commit: d00b6f7c1ff2d7569ae9efdc6823ebcfb86ef2d4 Parents: cc3f3ec Author: Jason Lowe <jl...@apache.org> Authored: Thu Nov 2 09:37:17 2017 -0500 Committer: Jason Lowe <jl...@apache.org> Committed: Thu Nov 2 09:37:17 2017 -0500 ---------------------------------------------------------------------- .../src/main/resources/yarn-default.xml | 3 +- .../runtime/DockerLinuxContainerRuntime.java | 30 ++++++++++++-- .../runtime/TestDockerContainerRuntime.java | 43 ++++++++++++++++++++ 3 files changed, 71 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d00b6f7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 8487e72..f4b2e61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1623,7 +1623,8 @@ <description>This configuration setting determines the capabilities assigned to docker containers when they are launched. While these may not be case-sensitive from a docker perspective, it is best to keep these - uppercase.</description> + uppercase. To run without any capabilites, set this value to + "none" or "NONE"</description> <name>yarn.nodemanager.runtime.linux.docker.capabilities</name> <value>CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE</value> </property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/d00b6f7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 6f7b6fd..a425cf8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -58,6 +58,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -187,6 +188,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { private boolean enableUserReMapping; private int userRemappingUidThreshold; private int userRemappingGidThreshold; + private Set<String> capabilities; /** * Return whether the given environment variables indicate that the operation @@ -285,6 +287,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { userRemappingGidThreshold = conf.getInt( YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD, YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD); + + capabilities = getDockerCapabilitiesFromConf(); + } + + private Set<String> getDockerCapabilitiesFromConf() throws + ContainerExecutionException { + Set<String> caps = new HashSet<>(Arrays.asList( + conf.getTrimmedStrings( + YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES))); + if(caps.contains("none") || caps.contains("NONE")) { + if(caps.size() > 1) { + String msg = "Mixing capabilities with the none keyword is" + + " not supported"; + throw new ContainerExecutionException(msg); + } + caps = Collections.emptySet(); + } + + return caps; + } + + public Set<String> getCapabilities() { + return capabilities; } @Override @@ -602,10 +628,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { LOCALIZED_RESOURCES); @SuppressWarnings("unchecked") List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS); - Set<String> capabilities = new HashSet<>(Arrays.asList( - conf.getTrimmedStrings( - YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, - YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES))); @SuppressWarnings("unchecked") DockerRunCommand runCommand = new DockerRunCommand(containerIdStr, http://git-wip-us.apache.org/repos/asf/hadoop/blob/d00b6f7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index b5a6497..76aca04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -65,6 +65,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Random; @@ -1304,4 +1305,46 @@ public class TestDockerContainerRuntime { Assert.assertEquals(" workdir=/test_container_work_dir", dockerCommands.get(counter++)); } + + @Test + public void testDockerCapabilities() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + try { + conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + "none", "CHOWN", "DAC_OVERRIDE"); + runtime.initialize(conf, null); + Assert.fail("Initialize didn't fail with invalid capabilities " + + "'none', 'CHOWN', 'DAC_OVERRIDE'"); + } catch (ContainerExecutionException e) { + } + + try { + conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + "CHOWN", "DAC_OVERRIDE", "NONE"); + runtime.initialize(conf, null); + Assert.fail("Initialize didn't fail with invalid capabilities " + + "'CHOWN', 'DAC_OVERRIDE', 'NONE'"); + } catch (ContainerExecutionException e) { + } + + conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + "NONE"); + runtime.initialize(conf, null); + Assert.assertEquals(0, runtime.getCapabilities().size()); + + conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + "none"); + runtime.initialize(conf, null); + Assert.assertEquals(0, runtime.getCapabilities().size()); + + conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, + "CHOWN", "DAC_OVERRIDE"); + runtime.initialize(conf, null); + Iterator<String> it = runtime.getCapabilities().iterator(); + Assert.assertEquals("CHOWN", it.next()); + Assert.assertEquals("DAC_OVERRIDE", it.next()); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org