YARN-6623. Add support to turn off launching privileged containers in the container-executor. (Varun Vasudev via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2f476f4b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2f476f4b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2f476f4b Branch: refs/heads/branch-2 Commit: 2f476f4b2c78768b4a1dab8e9ce7a5b20b52acf2 Parents: 3ed7a2c Author: Wangda Tan <wan...@apache.org> Authored: Thu Oct 19 15:11:05 2017 -0700 Committer: vrushali <vrush...@apache.org> Committed: Fri Oct 20 11:22:28 2017 -0700 ---------------------------------------------------------------------- .../hadoop-yarn/conf/container-executor.cfg | 12 + .../src/CMakeLists.txt | 5 +- .../runtime/DockerLinuxContainerRuntime.java | 92 +- .../linux/runtime/docker/DockerClient.java | 25 +- .../linux/runtime/docker/DockerCommand.java | 55 +- .../runtime/docker/DockerCommandExecutor.java | 3 +- .../runtime/docker/DockerInspectCommand.java | 13 +- .../linux/runtime/docker/DockerLoadCommand.java | 2 +- .../linux/runtime/docker/DockerPullCommand.java | 2 +- .../linux/runtime/docker/DockerRmCommand.java | 4 +- .../linux/runtime/docker/DockerRunCommand.java | 69 +- .../linux/runtime/docker/DockerStopCommand.java | 6 +- .../container-executor/impl/configuration.c | 17 + .../container-executor/impl/configuration.h | 19 +- .../impl/container-executor.c | 326 +---- .../impl/container-executor.h | 9 - .../container-executor/impl/get_executable.c | 3 - .../container-executor/impl/get_executable.h | 29 + .../main/native/container-executor/impl/main.c | 32 +- .../impl/modules/common/module-configs.c | 3 +- .../impl/modules/common/module-configs.h | 1 + .../main/native/container-executor/impl/util.c | 60 +- .../main/native/container-executor/impl/util.h | 46 +- .../container-executor/impl/utils/docker-util.c | 998 ++++++++++++++++ .../container-executor/impl/utils/docker-util.h | 147 +++ .../impl/utils/string-utils.c | 1 - .../docker-container-executor.cfg | 13 + .../test/test-container-executor.c | 203 +--- .../native/container-executor/test/test_util.cc | 37 +- .../test/utils/test-string-utils.cc | 37 +- .../test/utils/test_docker_util.cc | 1122 ++++++++++++++++++ .../runtime/TestDockerContainerRuntime.java | 399 ++++--- .../docker/TestDockerCommandExecutor.java | 35 +- .../docker/TestDockerInspectCommand.java | 29 +- .../runtime/docker/TestDockerLoadCommand.java | 9 +- .../runtime/docker/TestDockerPullCommand.java | 8 +- .../runtime/docker/TestDockerRmCommand.java | 8 +- .../runtime/docker/TestDockerRunCommand.java | 25 +- .../runtime/docker/TestDockerStopCommand.java | 15 +- .../src/site/markdown/DockerContainers.md | 33 +- 40 files changed, 3192 insertions(+), 760 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg index d68cee8..023654b 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg +++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg @@ -2,3 +2,15 @@ yarn.nodemanager.linux-container-executor.group=#configured value of yarn.nodema banned.users=#comma separated list of users who can not run applications min.user.id=1000#Prevent other super-users allowed.system.users=##comma separated list of system users who CAN run applications +feature.tc.enabled=0 + +# The configs below deal with settings for Docker +#[docker] +# module.enabled=## enable/disable the module. set to "true" to enable, disabled by default +# docker.binary=/usr/bin/docker +# docker.allowed.capabilities=## comma seperated capabilities that can be granted, e.g CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE +# docker.allowed.devices=## comma seperated list of devices that can be mounted into a container +# docker.allowed.networks=## comma seperated networks that can be used. e.g bridge,host,none +# docker.allowed.ro-mounts=## comma seperated volumes that can be mounted as read-only +# docker.allowed.rw-mounts=## comma seperate volumes that can be mounted as read-write, add the yarn local and log dirs to this list to run Hadoop jobs +# docker.privileged-containers.enabled=0 http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt index a083c0c..0b1c3e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt @@ -103,6 +103,7 @@ add_library(container main/native/container-executor/impl/utils/string-utils.c main/native/container-executor/impl/utils/path-utils.c main/native/container-executor/impl/modules/common/module-configs.c + main/native/container-executor/impl/utils/docker-util.c ) add_executable(container-executor @@ -127,11 +128,13 @@ output_directory(test-container-executor target/usr/local/bin) # unit tests for container executor add_executable(cetest + main/native/container-executor/impl/get_executable.c main/native/container-executor/impl/util.c main/native/container-executor/test/test_configuration.cc main/native/container-executor/test/test_main.cc main/native/container-executor/test/utils/test-string-utils.cc main/native/container-executor/test/utils/test-path-utils.cc - main/native/container-executor/test/test_util.cc) + main/native/container-executor/test/test_util.cc + main/native/container-executor/test/utils/test_docker_util.cc) target_link_libraries(cetest gtest container) output_directory(cetest test) http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index a500037d..2013306 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -539,10 +539,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { //List<String> -> stored as List -> fetched/converted to List<String> //we can't do better here thanks to type-erasure @SuppressWarnings("unchecked") - List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS); - @SuppressWarnings("unchecked") - List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS); - @SuppressWarnings("unchecked") List<String> filecacheDirs = ctx.getExecutionAttribute(FILECACHE_DIRS); @SuppressWarnings("unchecked") List<String> containerLocalDirs = ctx.getExecutionAttribute( @@ -570,8 +566,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { runCommand.setCapabilities(capabilities); if(cgroupsRootDirectory != null) { - runCommand.addMountLocation(cgroupsRootDirectory, - cgroupsRootDirectory + ":ro", false); + runCommand.addReadOnlyMountLocation(cgroupsRootDirectory, + cgroupsRootDirectory, false); } List<String> allDirs = new ArrayList<>(containerLocalDirs); @@ -595,7 +591,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } String src = validateMount(dir[0], localizedResources); String dst = dir[1]; - runCommand.addMountLocation(src, dst + ":ro", true); + runCommand.addReadOnlyMountLocation(src, dst, true); } } } @@ -608,9 +604,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { addCGroupParentIfRequired(resourcesOpts, containerIdStr, runCommand); - Path nmPrivateContainerScriptPath = ctx.getExecutionAttribute( - NM_PRIVATE_CONTAINER_SCRIPT_PATH); - String disableOverride = environment.get( ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE); @@ -632,40 +625,15 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { String commandFile = dockerClient.writeCommandToTempFile(runCommand, containerIdStr); - PrivilegedOperation launchOp = new PrivilegedOperation( - PrivilegedOperation.OperationType.LAUNCH_DOCKER_CONTAINER); - - launchOp.appendArgs(runAsUser, ctx.getExecutionAttribute(USER), - Integer.toString(PrivilegedOperation - .RunAsUserCommand.LAUNCH_DOCKER_CONTAINER.getValue()), - ctx.getExecutionAttribute(APPID), - containerIdStr, containerWorkDir.toString(), - nmPrivateContainerScriptPath.toUri().getPath(), - ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(), - ctx.getExecutionAttribute(PID_FILE_PATH).toString(), - StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, - localDirs), - StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, - logDirs), - commandFile, - resourcesOpts); - - String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE); - - if (tcCommandFile != null) { - launchOp.appendArgs(tcCommandFile); - } - if (LOG.isDebugEnabled()) { - LOG.debug("Launching container with cmd: " + runCommand - .getCommandWithArguments()); - } + PrivilegedOperation launchOp = buildLaunchOp(ctx, + commandFile, runCommand); try { privilegedOperationExecutor.executePrivilegedOperation(null, launchOp, null, null, false, false); } catch (PrivilegedOperationException e) { LOG.warn("Launch container failed. Exception: ", e); - LOG.info("Docker command used: " + runCommand.getCommandWithArguments()); + LOG.info("Docker command used: " + runCommand); throw new ContainerExecutionException("Launch container failed", e .getExitCode(), e.getOutput(), e.getErrorOutput()); @@ -754,6 +722,54 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { return null; } + + + private PrivilegedOperation buildLaunchOp(ContainerRuntimeContext ctx, + String commandFile, DockerRunCommand runCommand) { + + String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER); + String containerIdStr = ctx.getContainer().getContainerId().toString(); + Path nmPrivateContainerScriptPath = ctx.getExecutionAttribute( + NM_PRIVATE_CONTAINER_SCRIPT_PATH); + Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR); + //we can't do better here thanks to type-erasure + @SuppressWarnings("unchecked") + List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS); + @SuppressWarnings("unchecked") + List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS); + String resourcesOpts = ctx.getExecutionAttribute(RESOURCES_OPTIONS); + + PrivilegedOperation launchOp = new PrivilegedOperation( + PrivilegedOperation.OperationType.LAUNCH_DOCKER_CONTAINER); + + launchOp.appendArgs(runAsUser, ctx.getExecutionAttribute(USER), + Integer.toString(PrivilegedOperation + .RunAsUserCommand.LAUNCH_DOCKER_CONTAINER.getValue()), + ctx.getExecutionAttribute(APPID), + containerIdStr, + containerWorkDir.toString(), + nmPrivateContainerScriptPath.toUri().getPath(), + ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(), + ctx.getExecutionAttribute(PID_FILE_PATH).toString(), + StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, + localDirs), + StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, + logDirs), + commandFile, + resourcesOpts); + + String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE); + + if (tcCommandFile != null) { + launchOp.appendArgs(tcCommandFile); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Launching container with cmd: " + runCommand); + } + + return launchOp; + } + public static void validateImageName(String imageName) throws ContainerExecutionException { if (imageName == null || imageName.isEmpty()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java index 536a22d..77c53a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java @@ -23,7 +23,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +34,8 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.Writer; +import java.util.List; +import java.util.Map; @InterfaceAudience.Private @InterfaceStability.Unstable @@ -68,10 +70,25 @@ public final class DockerClient { TMP_FILE_SUFFIX, new File(tmpDirPath)); - Writer writer = new OutputStreamWriter(new FileOutputStream(dockerCommandFile), - "UTF-8"); + Writer writer = new OutputStreamWriter( + new FileOutputStream(dockerCommandFile), "UTF-8"); PrintWriter printWriter = new PrintWriter(writer); - printWriter.print(cmd.getCommandWithArguments()); + printWriter.println("[docker-command-execution]"); + for (Map.Entry<String, List<String>> entry : + cmd.getDockerCommandWithArguments().entrySet()) { + if (entry.getKey().contains("=")) { + throw new ContainerExecutionException( + "'=' found in entry for docker command file, key = " + entry + .getKey() + "; value = " + entry.getValue()); + } + if (entry.getValue().contains("\n")) { + throw new ContainerExecutionException( + "'\\n' found in entry for docker command file, key = " + entry + .getKey() + "; value = " + entry.getValue()); + } + printWriter.println(" " + entry.getKey() + "=" + StringUtils + .join(",", entry.getValue())); + } printWriter.close(); return dockerCommandFile.getAbsolutePath(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java index 3b76a5c..440275d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java @@ -25,8 +25,10 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.util.StringUtils; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.TreeMap; @InterfaceAudience.Private @InterfaceStability.Unstable @@ -35,32 +37,55 @@ import java.util.List; * e.g 'run', 'load', 'inspect' etc., */ -public abstract class DockerCommand { +public abstract class DockerCommand { private final String command; - private final List<String> commandWithArguments; + private final Map<String, List<String>> commandArguments; protected DockerCommand(String command) { + String dockerCommandKey = "docker-command"; this.command = command; - this.commandWithArguments = new ArrayList<>(); - commandWithArguments.add(command); + this.commandArguments = new TreeMap<>(); + commandArguments.put(dockerCommandKey, new ArrayList<String>()); + commandArguments.get(dockerCommandKey).add(command); } - /** Returns the docker sub-command string being used - * e.g 'run' + /** + * Returns the docker sub-command string being used + * e.g 'run'. */ public final String getCommandOption() { return this.command; } - /** Add command commandWithArguments - this method is only meant for use by - * sub-classes - * @param arguments to be added + /** + * Add command commandWithArguments - this method is only meant for use by + * sub-classes. + * + * @param key name of the key to be added + * @param value value of the key */ - protected final void addCommandArguments(String... arguments) { - this.commandWithArguments.addAll(Arrays.asList(arguments)); + protected final void addCommandArguments(String key, String value) { + List<String> list = commandArguments.get(key); + if (list != null) { + list.add(value); + return; + } + list = new ArrayList<>(); + list.add(value); + this.commandArguments.put(key, list); } - public String getCommandWithArguments() { - return StringUtils.join(" ", commandWithArguments); + public Map<String, List<String>> getDockerCommandWithArguments() { + return Collections.unmodifiableMap(commandArguments); } -} \ No newline at end of file + + @Override + public String toString() { + StringBuffer ret = new StringBuffer(this.command); + for (Map.Entry<String, List<String>> entry : commandArguments.entrySet()) { + ret.append(" ").append(entry.getKey()); + ret.append("=").append(StringUtils.join(",", entry.getValue())); + } + return ret.toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java index 5739912..76b53af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java @@ -88,8 +88,7 @@ public final class DockerCommandExecutor { dockerOp.disableFailureLogging(); } if (LOG.isDebugEnabled()) { - LOG.debug("Running docker command: " - + dockerCommand.getCommandWithArguments()); + LOG.debug("Running docker command: " + dockerCommand); } try { String result = privilegedOperationExecutor http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java index 812a35f..d27f74d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java @@ -26,16 +26,14 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime */ public class DockerInspectCommand extends DockerCommand { private static final String INSPECT_COMMAND = "inspect"; - private String containerName; public DockerInspectCommand(String containerName) { super(INSPECT_COMMAND); - this.containerName = containerName; + super.addCommandArguments("name", containerName); } public DockerInspectCommand getContainerStatus() { - super.addCommandArguments("--format='{{.State.Status}}'"); - super.addCommandArguments(containerName); + super.addCommandArguments("format", "{{.State.Status}}"); return this; } @@ -43,9 +41,8 @@ public class DockerInspectCommand extends DockerCommand { // Be sure to not use space in the argument, otherwise the // extract_values_delim method in container-executor binary // cannot parse the arguments correctly. - super.addCommandArguments("--format='{{range(.NetworkSettings.Networks)}}" - + "{{.IPAddress}},{{end}}{{.Config.Hostname}}'"); - super.addCommandArguments(containerName); + super.addCommandArguments("format", "{{range(.NetworkSettings.Networks)}}" + + "{{.IPAddress}},{{end}}{{.Config.Hostname}}"); return this; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java index e4d92e0..fa2988c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java @@ -25,6 +25,6 @@ public class DockerLoadCommand extends DockerCommand { public DockerLoadCommand(String localImageFile) { super(LOAD_COMMAND); - super.addCommandArguments("--i=" + localImageFile); + super.addCommandArguments("image", localImageFile); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java index 351e09e..5e6108e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java @@ -25,7 +25,7 @@ public class DockerPullCommand extends DockerCommand { public DockerPullCommand(String imageName) { super(PULL_COMMAND); - super.addCommandArguments(imageName); + super.addCommandArguments("image", imageName); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java index b1aea61..dcfe777 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java @@ -25,6 +25,6 @@ public class DockerRmCommand extends DockerCommand { public DockerRmCommand(String containerName) { super(RM_COMMAND); - super.addCommandArguments(containerName); + super.addCommandArguments("name", containerName); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index 1e1e6e8..7f87b3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -20,42 +20,40 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; -import org.apache.hadoop.util.StringUtils; - import java.io.File; -import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; +import org.apache.hadoop.util.StringUtils; public class DockerRunCommand extends DockerCommand { private static final String RUN_COMMAND = "run"; - private final String image; - private List<String> overrrideCommandWithArgs; /** The following are mandatory: */ public DockerRunCommand(String containerId, String user, String image) { super(RUN_COMMAND); - super.addCommandArguments("--name=" + containerId, "--user=" + user); - this.image = image; + super.addCommandArguments("name", containerId); + super.addCommandArguments("user", user); + super.addCommandArguments("image", image); } public DockerRunCommand removeContainerOnExit() { - super.addCommandArguments("--rm"); + super.addCommandArguments("rm", "true"); return this; } public DockerRunCommand detachOnRun() { - super.addCommandArguments("-d"); + super.addCommandArguments("detach", "true"); return this; } public DockerRunCommand setContainerWorkDir(String workdir) { - super.addCommandArguments("--workdir=" + workdir); + super.addCommandArguments("workdir", workdir); return this; } public DockerRunCommand setNetworkType(String type) { - super.addCommandArguments("--net=" + type); + super.addCommandArguments("net", type); return this; } @@ -65,79 +63,80 @@ public class DockerRunCommand extends DockerCommand { if (!sourceExists && !createSource) { return this; } - super.addCommandArguments("-v", sourcePath + ":" + destinationPath); + super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath); + return this; + } + + public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String + destinationPath, boolean createSource) { + boolean sourceExists = new File(sourcePath).exists(); + if (!sourceExists && !createSource) { + return this; + } + super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath); return this; } public DockerRunCommand setCGroupParent(String parentPath) { - super.addCommandArguments("--cgroup-parent=" + parentPath); + super.addCommandArguments("cgroup-parent", parentPath); return this; } /* Run a privileged container. Use with extreme care */ public DockerRunCommand setPrivileged() { - super.addCommandArguments("--privileged"); + super.addCommandArguments("privileged", "true"); return this; } public DockerRunCommand setCapabilities(Set<String> capabilties) { //first, drop all capabilities - super.addCommandArguments("--cap-drop=ALL"); + super.addCommandArguments("cap-drop", "ALL"); //now, add the capabilities supplied for (String capability : capabilties) { - super.addCommandArguments("--cap-add=" + capability); + super.addCommandArguments("cap-add", capability); } return this; } public DockerRunCommand setHostname(String hostname) { - super.addCommandArguments("--hostname=" + hostname); + super.addCommandArguments("hostname", hostname); return this; } public DockerRunCommand addDevice(String sourceDevice, String destinationDevice) { - super.addCommandArguments("--device=" + sourceDevice + ":" + + super.addCommandArguments("devices", sourceDevice + ":" + destinationDevice); return this; } public DockerRunCommand enableDetach() { - super.addCommandArguments("--detach=true"); + super.addCommandArguments("detach", "true"); return this; } public DockerRunCommand disableDetach() { - super.addCommandArguments("--detach=false"); + super.addCommandArguments("detach", "false"); return this; } public DockerRunCommand groupAdd(String[] groups) { - for(int i = 0; i < groups.length; i++) { - super.addCommandArguments("--group-add " + groups[i]); - } + super.addCommandArguments("group-add", StringUtils.join(",", groups)); return this; } public DockerRunCommand setOverrideCommandWithArgs( List<String> overrideCommandWithArgs) { - this.overrrideCommandWithArgs = overrideCommandWithArgs; + for(String override: overrideCommandWithArgs) { + super.addCommandArguments("launch-command", override); + } return this; } @Override - public String getCommandWithArguments() { - List<String> argList = new ArrayList<>(); - - argList.add(super.getCommandWithArguments()); - argList.add(image); - - if (overrrideCommandWithArgs != null) { - argList.addAll(overrrideCommandWithArgs); - } - - return StringUtils.join(" ", argList); + public Map<String, List<String>> getDockerCommandWithArguments() { + return super.getDockerCommandWithArguments(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java index e9d6c43..ccfff82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java @@ -29,11 +29,11 @@ public class DockerStopCommand extends DockerCommand { public DockerStopCommand(String containerName) { super(STOP_COMMAND); - super.addCommandArguments(containerName); + super.addCommandArguments("name", containerName); } public DockerStopCommand setGracePeriod(int value) { - super.addCommandArguments("--time=" + Integer.toString(value)); + super.addCommandArguments("time", Integer.toString(value)); return this; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c index 12dbc4c..f23cff0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c @@ -21,6 +21,7 @@ #include "configuration.h" #include "util.h" +#include "get_executable.h" #define __STDC_FORMAT_MACROS #include <inttypes.h> @@ -696,3 +697,19 @@ int get_kv_value(const char *input, char *out, size_t out_len) { return 0; } + +char *get_config_path(const char *argv0) { + char *executable_file = get_executable((char *) argv0); + if (!executable_file) { + fprintf(ERRORFILE, "realpath of executable: %s\n", + errno != 0 ? strerror(errno) : "unknown"); + return NULL; + } + + const char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME; + char *conf_file = resolve_config_path(orig_conf_file, executable_file); + if (conf_file == NULL) { + fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file); + } + return conf_file; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h index 1ea5561..7fa684e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h @@ -23,10 +23,21 @@ #define _WITH_GETLINE #endif -#include <stddef.h> +#include "config.h" + +#define CONF_FILENAME "container-executor.cfg" + +// When building as part of a Maven build this value gets defined by using +// container-executor.conf.dir property. See: +// hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +// for details. +// NOTE: if this ends up being a relative path it gets resolved relative to +// the location of the container-executor binary itself, not getwd(3) +#ifndef HADOOP_CONF_DIR +#error HADOOP_CONF_DIR must be defined +#endif -/** Define a platform-independent constant instead of using PATH_MAX */ -#define EXECUTOR_PATH_MAX 4096 +#include <stddef.h> // Configuration data structures. struct kv_pair { @@ -207,4 +218,6 @@ int get_kv_key(const char *input, char *out, size_t out_len); */ int get_kv_value(const char *input, char *out, size_t out_len); +char *get_config_path(const char* argv0); + #endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index c2e6f13..a623658 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -18,7 +18,7 @@ #include "configuration.h" #include "container-executor.h" -#include "utils/string-utils.h" +#include "utils/docker-util.h" #include "util.h" #include "config.h" @@ -43,7 +43,6 @@ #include <sys/mount.h> #include <sys/wait.h> #include <getopt.h> -#include <regex.h> #ifndef HAVE_FCHMODAT #include "compat/fchmodat.h" @@ -81,11 +80,6 @@ static const char* TC_READ_STATS_OPTS [] = { "-s", "-b", NULL}; //struct to store the user details struct passwd *user_detail = NULL; -//Docker container related constants. -static const char* DOCKER_CONTAINER_NAME_PREFIX = "container_"; -static const char* DOCKER_CLIENT_CONFIG_ARG = "--config="; -static const char* DOCKER_PULL_COMMAND = "pull"; - FILE* LOGFILE = NULL; FILE* ERRORFILE = NULL; @@ -465,8 +459,9 @@ int is_feature_enabled(const char* feature_key, int default_value, } int is_docker_support_enabled() { - return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY, - DEFAULT_DOCKER_SUPPORT_ENABLED, &executor_cfg); + return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY, + DEFAULT_DOCKER_SUPPORT_ENABLED, &executor_cfg) + || docker_module_enabled(&CFG); } int is_tc_support_enabled() { @@ -474,13 +469,6 @@ int is_tc_support_enabled() { DEFAULT_TC_SUPPORT_ENABLED, &executor_cfg); } -char* check_docker_binary(char *docker_binary) { - if (docker_binary == NULL) { - return "docker"; - } - return docker_binary; -} - /** * Utility function to concatenate argB to argA using the concat_pattern. */ @@ -1157,274 +1145,28 @@ int initialize_app(const char *user, const char *app_id, return -1; } -static char* escape_single_quote(const char *str) { - int p = 0; - int i = 0; - char replacement[] = "'\"'\"'"; - size_t replacement_length = strlen(replacement); - size_t ret_size = strlen(str) * replacement_length + 1; - char *ret = (char *) calloc(ret_size, sizeof(char)); - if(ret == NULL) { - exit(OUT_OF_MEMORY); - } - while(str[p] != '\0') { - if(str[p] == '\'') { - strncat(ret, replacement, ret_size - strlen(ret)); - i += replacement_length; - } - else { - ret[i] = str[p]; - ret[i + 1] = '\0'; - i++; - } - p++; - } - return ret; -} - -static void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) { - char *tmp = escape_single_quote(arg); - strcat(*str, param); - strcat(*str, "'"); - if(strlen(*str) + strlen(tmp) > *size) { - *str = (char *) realloc(*str, strlen(*str) + strlen(tmp) + 1024); - if(*str == NULL) { - exit(OUT_OF_MEMORY); - } - *size = strlen(*str) + strlen(tmp) + 1024; - } - strcat(*str, tmp); - strcat(*str, "' "); - free(tmp); -} - -char** tokenize_docker_command(const char *input, int *split_counter) { - char *line = (char *)calloc(strlen(input) + 1, sizeof(char)); - char **linesplit = (char **) malloc(sizeof(char *)); - char *p = NULL; - *split_counter = 0; - strncpy(line, input, strlen(input)); - - p = strtok(line, " "); - while(p != NULL) { - linesplit[*split_counter] = p; - (*split_counter)++; - linesplit = realloc(linesplit, (sizeof(char *) * (*split_counter + 1))); - if(linesplit == NULL) { - fprintf(ERRORFILE, "Cannot allocate memory to parse docker command %s", - strerror(errno)); - fflush(ERRORFILE); - exit(OUT_OF_MEMORY); - } - p = strtok(NULL, " "); - } - linesplit[*split_counter] = NULL; - return linesplit; -} - -int execute_regex_match(const char *regex_str, const char *input) { - regex_t regex; - int regex_match; - if (0 != regcomp(®ex, regex_str, REG_EXTENDED|REG_NOSUB)) { - fprintf(LOGFILE, "Unable to compile regex."); - fflush(LOGFILE); - exit(ERROR_COMPILING_REGEX); - } - regex_match = regexec(®ex, input, (size_t) 0, NULL, 0); - regfree(®ex); - if(0 == regex_match) { - return 0; - } - return 1; -} - -int validate_docker_image_name(const char *image_name) { - char *regex_str = "^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$"; - return execute_regex_match(regex_str, image_name); -} - -char* sanitize_docker_command(const char *line) { - static struct option long_options[] = { - {"name", required_argument, 0, 'n' }, - {"user", required_argument, 0, 'u' }, - {"rm", no_argument, 0, 'r' }, - {"workdir", required_argument, 0, 'w' }, - {"net", required_argument, 0, 'e' }, - {"hostname", required_argument, 0, 'h' }, - {"cgroup-parent", required_argument, 0, 'g' }, - {"privileged", no_argument, 0, 'p' }, - {"cap-add", required_argument, 0, 'a' }, - {"cap-drop", required_argument, 0, 'o' }, - {"device", required_argument, 0, 'i' }, - {"detach", required_argument, 0, 't' }, - {"format", required_argument, 0, 'f' }, - {"group-add", required_argument, 0, 'x' }, - {0, 0, 0, 0} - }; - - int c = 0; - int option_index = 0; - char *output = NULL; - size_t output_size = 0; - char **linesplit; - int split_counter = 0; - int len = strlen(line); - - linesplit = tokenize_docker_command(line, &split_counter); - - output_size = len * 2; - output = (char *) calloc(output_size, sizeof(char)); - if(output == NULL) { - exit(OUT_OF_MEMORY); - } - - // Handle docker client config option. - if(0 == strncmp(linesplit[0], DOCKER_CLIENT_CONFIG_ARG, strlen(DOCKER_CLIENT_CONFIG_ARG))) { - strcat(output, linesplit[0]); - strcat(output, " "); - long index = 0; - while(index < split_counter) { - linesplit[index] = linesplit[index + 1]; - if (linesplit[index] == NULL) { - split_counter--; - break; - } - index++; - } - } - - // Handle docker pull and image name validation. - if (0 == strncmp(linesplit[0], DOCKER_PULL_COMMAND, strlen(DOCKER_PULL_COMMAND))) { - if (0 != validate_docker_image_name(linesplit[1])) { - fprintf(ERRORFILE, "Invalid Docker image name, exiting."); - fflush(ERRORFILE); - exit(DOCKER_IMAGE_INVALID); - } - strcat(output, linesplit[0]); - strcat(output, " "); - strcat(output, linesplit[1]); - return output; - } - - strcat(output, linesplit[0]); - strcat(output, " "); - optind = 1; - while((c=getopt_long(split_counter, linesplit, "dv:", long_options, &option_index)) != -1) { - switch(c) { - case 'n': - quote_and_append_arg(&output, &output_size, "--name=", optarg); - break; - case 'w': - quote_and_append_arg(&output, &output_size, "--workdir=", optarg); - break; - case 'u': - quote_and_append_arg(&output, &output_size, "--user=", optarg); - break; - case 'e': - quote_and_append_arg(&output, &output_size, "--net=", optarg); - break; - case 'h': - quote_and_append_arg(&output, &output_size, "--hostname=", optarg); - break; - case 'v': - quote_and_append_arg(&output, &output_size, "-v ", optarg); - break; - case 'a': - quote_and_append_arg(&output, &output_size, "--cap-add=", optarg); - break; - case 'o': - quote_and_append_arg(&output, &output_size, "--cap-drop=", optarg); - break; - case 'd': - strcat(output, "-d "); - break; - case 'r': - strcat(output, "--rm "); - break; - case 'g': - quote_and_append_arg(&output, &output_size, "--cgroup-parent=", optarg); - break; - case 'p': - strcat(output, "--privileged "); - break; - case 'i': - quote_and_append_arg(&output, &output_size, "--device=", optarg); - break; - case 't': - quote_and_append_arg(&output, &output_size, "--detach=", optarg); - break; - case 'f': - strcat(output, "--format="); - strcat(output, optarg); - strcat(output, " "); - break; - case 'x': - quote_and_append_arg(&output, &output_size, "--group-add ", optarg); - break; - default: - fprintf(LOGFILE, "Unknown option in docker command, character %d %c, optionindex = %d\n", c, c, optind); - fflush(LOGFILE); - return NULL; - break; - } - } - - if(optind < split_counter) { - while(optind < split_counter) { - if (0 == strncmp(linesplit[optind], DOCKER_CONTAINER_NAME_PREFIX, strlen(DOCKER_CONTAINER_NAME_PREFIX))) { - if (1 != validate_container_id(linesplit[optind])) { - fprintf(ERRORFILE, "Specified container_id=%s is invalid\n", linesplit[optind]); - fflush(ERRORFILE); - exit(DOCKER_CONTAINER_NAME_INVALID); - } - strcat(output, linesplit[optind++]); - } else { - quote_and_append_arg(&output, &output_size, "", linesplit[optind++]); - } - } - } - - return output; -} - -char* parse_docker_command_file(const char* command_file) { - - size_t len = 0; - char *line = NULL; - ssize_t read; - FILE *stream; - stream = fopen(command_file, "r"); - if (stream == NULL) { - fprintf(ERRORFILE, "Cannot open file %s - %s", - command_file, strerror(errno)); - fflush(ERRORFILE); - exit(ERROR_OPENING_DOCKER_FILE); - } - if ((read = getline(&line, &len, stream)) == -1) { - fprintf(ERRORFILE, "Error reading command_file %s\n", command_file); - fflush(ERRORFILE); - exit(ERROR_READING_DOCKER_FILE); - } - fclose(stream); - - char* ret = sanitize_docker_command(line); - if(ret == NULL) { - exit(ERROR_SANITIZING_DOCKER_COMMAND); +char *construct_docker_command(const char *command_file) { + int ret = 0; + size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024); + char *buffer = alloc_and_clear_memory(command_size, sizeof(char)); + ret = get_docker_command(command_file, &CFG, buffer, command_size); + if (ret != 0) { + fprintf(ERRORFILE, "Error constructing docker command, docker error code=%d, error message='%s'\n", ret, + get_docker_error_message(ret)); + fflush(ERRORFILE); + exit(DOCKER_RUN_FAILED); } - fprintf(ERRORFILE, "Using command %s\n", ret); - fflush(ERRORFILE); - - return ret; + return buffer; } int run_docker(const char *command_file) { - char* docker_command = parse_docker_command_file(command_file); - char* docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg); - docker_binary = check_docker_binary(docker_binary); + char* docker_command = construct_docker_command(command_file); + char* docker_binary = get_docker_binary(&CFG); size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024); - char* docker_command_with_binary = calloc(sizeof(char), command_size); + char* docker_command_with_binary = alloc_and_clear_memory(command_size, sizeof(char)); snprintf(docker_command_with_binary, command_size, "%s %s", docker_binary, docker_command); + fprintf(LOGFILE, "Invoking '%s'\n", docker_command_with_binary); char **args = split_delimiter(docker_command_with_binary, " "); int exit_code = -1; @@ -1438,8 +1180,9 @@ int run_docker(const char *command_file) { free(docker_command_with_binary); free(docker_command); exit_code = DOCKER_RUN_FAILED; + } else { + exit_code = 0; } - exit_code = 0; return exit_code; } @@ -1584,18 +1327,17 @@ int launch_docker_container_as_user(const char * user, const char *app_id, size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024); - docker_command_with_binary = calloc(sizeof(char), command_size); - docker_wait_command = calloc(sizeof(char), command_size); - docker_logs_command = calloc(sizeof(char), command_size); - docker_inspect_command = calloc(sizeof(char), command_size); - docker_rm_command = calloc(sizeof(char), command_size); + docker_command_with_binary = (char *) alloc_and_clear_memory(command_size, sizeof(char)); + docker_wait_command = (char *) alloc_and_clear_memory(command_size, sizeof(char)); + docker_logs_command = (char *) alloc_and_clear_memory(command_size, sizeof(char)); + docker_inspect_command = (char *) alloc_and_clear_memory(command_size, sizeof(char)); + docker_rm_command = (char *) alloc_and_clear_memory(command_size, sizeof(char)); gid_t user_gid = getegid(); uid_t prev_uid = geteuid(); - char *docker_command = parse_docker_command_file(command_file); - char *docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg); - docker_binary = check_docker_binary(docker_binary); + char *docker_command = NULL; + char *docker_binary = NULL; fprintf(LOGFILE, "Creating script paths...\n"); exit_code = create_script_paths( @@ -1618,6 +1360,9 @@ int launch_docker_container_as_user(const char * user, const char *app_id, goto cleanup; } + docker_command = construct_docker_command(command_file); + docker_binary = get_docker_binary(&CFG); + fprintf(LOGFILE, "Getting exit code file...\n"); exit_code_file = get_exit_code_file(pid_file); if (NULL == exit_code_file) { @@ -2460,3 +2205,12 @@ int traffic_control_read_state(char *command_file) { int traffic_control_read_stats(char *command_file) { return run_traffic_control(TC_READ_STATS_OPTS, command_file); } + +/** + * FIXME: (wangda) it's better to move executor_cfg out of container-executor.c + * Now initialize of executor_cfg and data structures are stored inside + * container-executor which is not a good design. + */ +struct configuration* get_cfg() { + return &CFG; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 76a832f..2a442c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -59,7 +59,6 @@ enum operations { #define MIN_USERID_KEY "min.user.id" #define BANNED_USERS_KEY "banned.users" #define ALLOWED_SYSTEM_USERS_KEY "allowed.system.users" -#define DOCKER_BINARY_KEY "docker.binary" #define DOCKER_SUPPORT_ENABLED_KEY "feature.docker.enabled" #define TC_SUPPORT_ENABLED_KEY "feature.tc.enabled" #define TMP_DIR "tmp" @@ -74,9 +73,6 @@ enum operations { extern struct passwd *user_detail; -// get the executable's filename -char* get_executable(char *argv0); - //function used to load the configurations present in the secure config void read_executor_config(const char* file_name); @@ -266,11 +262,6 @@ int is_docker_support_enabled(); */ int run_docker(const char *command_file); -/** - * Sanitize docker commands. Returns NULL if there was any failure. -*/ -char* sanitize_docker_command(const char *line); - /* * Compile the regex_str and determine if the input string matches. * Return 0 on match, 1 of non-match. http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c index 55973a2..e1ec293 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c @@ -29,12 +29,9 @@ */ #include "config.h" -#include "configuration.h" -#include "container-executor.h" #include "util.h" #include <errno.h> -#include <stdio.h> #include <string.h> #include <stdlib.h> #include <unistd.h> http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h new file mode 100644 index 0000000..ee6b375 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_GET_EXECUTABLE_H__ +#define __YARN_POSIX_CONTAINER_EXECUTOR_GET_EXECUTABLE_H__ + +/** + * Get the path to executable that is currently running + * @param argv0 the name of the executable + * @return the path to the currently running executable + */ +char* get_executable(char *argv0); + +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 0f294f6..6e32825 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -20,28 +20,13 @@ #include "configuration.h" #include "container-executor.h" #include "util.h" +#include "get_executable.h" #include <errno.h> #include <grp.h> -#include <limits.h> #include <unistd.h> -#include <signal.h> -#include <stdio.h> #include <stdlib.h> #include <string.h> -#include <sys/stat.h> - -#define CONF_FILENAME "container-executor.cfg" - -// When building as part of a Maven build this value gets defined by using -// container-executor.conf.dir property. See: -// hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml -// for details. -// NOTE: if this ends up being a relative path it gets resolved relative to -// the location of the container-executor binary itself, not getwd(3) -#ifndef HADOOP_CONF_DIR - #error HADOOP_CONF_DIR must be defined -#endif static void display_usage(FILE *stream) { fprintf(stream, @@ -143,24 +128,21 @@ of whether an explicit checksetup operation is requested. */ static void assert_valid_setup(char *argv0) { int ret; char *executable_file = get_executable(argv0); - if (!executable_file) { - fprintf(ERRORFILE,"realpath of executable: %s\n",strerror(errno)); + if (!executable_file || executable_file[0] == 0) { + fprintf(ERRORFILE, "realpath of executable: %s\n", + errno != 0 ? strerror(errno) : "unknown"); flush_and_close_log_files(); - exit(-1); + exit(INVALID_CONFIG_FILE); } - char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME; - char *conf_file = resolve_config_path(orig_conf_file, executable_file); + char *conf_file = get_config_path(argv0); if (conf_file == NULL) { - free(executable_file); - fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file); flush_and_close_log_files(); exit(INVALID_CONFIG_FILE); } if (check_configuration_permissions(conf_file) != 0) { - free(executable_file); flush_and_close_log_files(); exit(INVALID_CONFIG_FILE); } @@ -179,7 +161,7 @@ static void assert_valid_setup(char *argv0) { if (group_info == NULL) { free(executable_file); fprintf(ERRORFILE, "Can't get group information for %s - %s.\n", nm_group, - strerror(errno)); + errno != 0 ? strerror(errno) : "unknown"); flush_and_close_log_files(); exit(INVALID_CONFIG_FILE); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c index f0c6d16..4c76e51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c @@ -16,9 +16,8 @@ * limitations under the License. */ +#include "module-configs.h" #include "util.h" -#include "configuration.h" -#include "container-executor.h" #include "modules/common/constants.h" #include <string.h> http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h index d58c618..b8c57df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h @@ -23,6 +23,7 @@ #ifndef _MODULES_COMMON_MODULE_CONFIGS_H_ #define _MODULES_COMMON_MODULE_CONFIGS_H_ +#include "configuration.h" /** * check if module enabled given name of module. http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c index 8e39ca8..a9539cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c @@ -17,10 +17,10 @@ */ #include "util.h" -#include <stddef.h> #include <stdlib.h> #include <string.h> #include <ctype.h> +#include <regex.h> char** split_delimiter(char *value, const char *delim) { char **return_values = NULL; @@ -132,3 +132,61 @@ char* trim(const char* input) { ret[val_end - val_begin] = '\0'; return ret; } + +int execute_regex_match(const char *regex_str, const char *input) { + regex_t regex; + int regex_match; + if (0 != regcomp(®ex, regex_str, REG_EXTENDED|REG_NOSUB)) { + fprintf(LOGFILE, "Unable to compile regex."); + fflush(LOGFILE); + exit(ERROR_COMPILING_REGEX); + } + regex_match = regexec(®ex, input, (size_t) 0, NULL, 0); + regfree(®ex); + if(0 == regex_match) { + return 0; + } + return 1; +} + +char* escape_single_quote(const char *str) { + int p = 0; + int i = 0; + char replacement[] = "'\"'\"'"; + size_t replacement_length = strlen(replacement); + size_t ret_size = strlen(str) * replacement_length + 1; + char *ret = (char *) alloc_and_clear_memory(ret_size, sizeof(char)); + if(ret == NULL) { + exit(OUT_OF_MEMORY); + } + while(str[p] != '\0') { + if(str[p] == '\'') { + strncat(ret, replacement, ret_size - strlen(ret)); + i += replacement_length; + } + else { + ret[i] = str[p]; + i++; + } + p++; + } + ret[i] = '\0'; + return ret; +} + +void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) { + char *tmp = escape_single_quote(arg); + int alloc_block = 1024; + strcat(*str, param); + strcat(*str, "'"); + if (strlen(*str) + strlen(tmp) > *size) { + *size = (strlen(*str) + strlen(tmp) + alloc_block) * sizeof(char); + *str = (char *) realloc(*str, *size); + if (*str == NULL) { + exit(OUT_OF_MEMORY); + } + } + strcat(*str, tmp); + strcat(*str, "' "); + free(tmp); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f476f4b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h index a8a12a9..c54bfbb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h @@ -19,7 +19,11 @@ #ifndef __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__ #define __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__ +/** Define a platform-independent constant instead of using PATH_MAX, set to 4K */ +#define EXECUTOR_PATH_MAX 4096 + #include <stdio.h> +#include <stdlib.h> enum errorcodes { INVALID_ARGUMENT_NUMBER = 1, @@ -62,7 +66,7 @@ enum errorcodes { ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38, ERROR_SANITIZING_DOCKER_COMMAND = 39, DOCKER_IMAGE_INVALID = 40, - DOCKER_CONTAINER_NAME_INVALID = 41, + // DOCKER_CONTAINER_NAME_INVALID = 41, (NOT USED) ERROR_COMPILING_REGEX = 42 }; @@ -112,4 +116,44 @@ void free_values(char **values); */ char* trim(const char *input); +/** + * Run a regex to check if the provided input matches against it + * @param regex_str Regex to run + * @param input String to run the regex against + * @return 0 on match, non-zero on no match + */ +int execute_regex_match(const char *regex_str, const char *input); + +/** + * Helper function to escape single-quotes in a string. The assumption is that the string passed will be enclosed in + * single quotes and passed to bash for a command invocation. + * @param str The string in which to esacpe single quotes + * @return String with single quotes escaped, must be freed by the user. + */ +char* escape_single_quote(const char *str); + +/** + * Helper function to quote the argument to a parameter and then append it to the provided string. + * @param str Buffer to which the param='arg' string must be appended + * @param size Size of the buffer + * @param param Param name + * @param arg Argument to be quoted + */ +void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg); + +/** + * Helper function to allocate and clear a block of memory. It'll call exit if the allocation fails. + * @param num Num of elements to be allocated + * @param size Size of each element + * @return Pointer to the allocated memory, must be freed by the user + */ +inline void* alloc_and_clear_memory(size_t num, size_t size) { + void *ret = calloc(num, size); + if (ret == NULL) { + printf("Could not allocate memory, exiting\n"); + exit(OUT_OF_MEMORY); + } + return ret; +} + #endif --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
