YARN-7224. Support GPU isolation for docker container. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9114d7a5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9114d7a5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9114d7a5 Branch: refs/heads/YARN-1011 Commit: 9114d7a5a0159bbe70e9c289dbe1fc5db9101db5 Parents: e62bbbc Author: Sunil G <sun...@apache.org> Authored: Sun Oct 29 11:08:44 2017 +0530 Committer: Sunil G <sun...@apache.org> Committed: Sun Oct 29 11:08:44 2017 +0530 ---------------------------------------------------------------------- .../hadoop-yarn/conf/container-executor.cfg | 1 + .../hadoop/yarn/conf/YarnConfiguration.java | 29 ++ .../src/main/resources/yarn-default.xml | 42 ++- .../nodemanager/LinuxContainerExecutor.java | 3 +- .../resources/gpu/GpuResourceAllocator.java | 102 +++--- .../resources/gpu/GpuResourceHandlerImpl.java | 90 ++++-- .../runtime/DefaultLinuxContainerRuntime.java | 3 +- .../DelegatingLinuxContainerRuntime.java | 9 +- .../runtime/DockerLinuxContainerRuntime.java | 91 +++++- .../JavaSandboxLinuxContainerRuntime.java | 5 +- .../linux/runtime/LinuxContainerRuntime.java | 4 +- .../linux/runtime/docker/DockerRunCommand.java | 5 + .../runtime/docker/DockerVolumeCommand.java | 49 +++ .../resourceplugin/DockerCommandPlugin.java | 59 ++++ .../resourceplugin/ResourcePlugin.java | 11 + .../resourceplugin/gpu/GpuDevice.java | 78 +++++ .../resourceplugin/gpu/GpuDiscoverer.java | 30 +- .../gpu/GpuDockerCommandPluginFactory.java | 41 +++ .../gpu/GpuNodeResourceUpdateHandler.java | 10 +- .../resourceplugin/gpu/GpuResourcePlugin.java | 9 + .../gpu/NvidiaDockerV1CommandPlugin.java | 319 +++++++++++++++++++ .../recovery/NMLeveldbStateStoreService.java | 62 ++-- .../recovery/NMNullStateStoreService.java | 3 +- .../recovery/NMStateStoreService.java | 15 +- .../container-executor/impl/utils/docker-util.c | 130 ++++++++ .../container-executor/impl/utils/docker-util.h | 18 +- .../test/utils/test_docker_util.cc | 42 +++ .../TestLinuxContainerExecutorWithMocks.java | 6 +- .../TestContainerManagerRecovery.java | 9 +- .../resources/gpu/TestGpuResourceHandler.java | 156 ++++++--- .../TestDelegatingLinuxContainerRuntime.java | 14 +- .../runtime/TestDockerContainerRuntime.java | 204 ++++++++++-- .../TestJavaSandboxLinuxContainerRuntime.java | 3 +- .../docker/TestDockerCommandExecutor.java | 3 +- .../runtime/docker/TestDockerVolumeCommand.java | 45 +++ .../resourceplugin/gpu/TestGpuDiscoverer.java | 34 +- .../gpu/TestNvidiaDockerV1CommandPlugin.java | 217 +++++++++++++ .../recovery/NMMemoryStateStoreService.java | 8 +- .../TestNMLeveldbStateStoreService.java | 22 +- 39 files changed, 1721 insertions(+), 260 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg index 023654b..7a84d76 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg +++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg @@ -14,3 +14,4 @@ feature.tc.enabled=0 # docker.allowed.ro-mounts=## comma seperated volumes that can be mounted as read-only # docker.allowed.rw-mounts=## comma seperate volumes that can be mounted as read-write, add the yarn local and log dirs to this list to run Hadoop jobs # docker.privileged-containers.enabled=0 +# docker.allowed.volume-drivers=## comma seperated list of allowed volume-drivers http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ca2fb66..640e86e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1483,6 +1483,35 @@ public class YarnConfiguration extends Configuration { @Private public static final String DEFAULT_NM_GPU_PATH_TO_EXEC = ""; + /** + * Settings to control which implementation of docker plugin for GPU will be + * used. + * + * By default uses NVIDIA docker v1. + */ + @Private + public static final String NM_GPU_DOCKER_PLUGIN_IMPL = + NM_GPU_RESOURCE_PREFIX + "docker-plugin"; + + @Private + public static final String NVIDIA_DOCKER_V1 = "nvidia-docker-v1"; + + @Private + public static final String DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL = + NVIDIA_DOCKER_V1; + + /** + * This setting controls end point of nvidia-docker-v1 plugin + */ + @Private + public static final String NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT = + NM_GPU_RESOURCE_PREFIX + "docker-plugin." + NVIDIA_DOCKER_V1 + + ".endpoint"; + + @Private + public static final String DEFAULT_NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT = + "http://localhost:3476/v1.0/docker/cli";; + /** NM Webapp address.**/ public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 99e74f0..8487e72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3451,6 +3451,15 @@ <property> <description> + Provides an option for client to load supported resource types from RM + instead of depending on local resource-types.xml file. + </description> + <name>yarn.client.load.resource-types.from-server</name> + <value>false</value> + </property> + + <property> + <description> When yarn.nodemanager.resource.gpu.allowed-gpu-devices=auto specified, YARN NodeManager needs to run GPU discovery binary (now only support nvidia-smi) to get GPU-related information. @@ -3477,12 +3486,18 @@ Number of GPU devices will be reported to RM to make scheduling decisions. Set to auto (default) let YARN automatically discover GPU resource from system. + Manually specify GPU devices if auto detect GPU device failed or admin only want subset of GPU devices managed by YARN. GPU device is identified - by their minor device number. A common approach to get minor device number - of GPUs is using "nvidia-smi -q" and search "Minor Number" output. An - example of manual specification is "0,1,2,4" to allow YARN NodeManager - to manage GPU devices with minor number 0/1/2/4. + by their minor device number and index. A common approach to get minor + device number of GPUs is using "nvidia-smi -q" and search "Minor Number" + output. + + When manual specify minor numbers, admin needs to include indice of GPUs + as well, format is index:minor_number[,index:minor_number...]. An example + of manual specification is "0:0,1:1,2:2,3:4" to allow YARN NodeManager to + manage GPU devices with indice 0/1/2/3 and minor number 0/1/2/4. + numbers . </description> <name>yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices</name> <value>auto</value> @@ -3490,11 +3505,22 @@ <property> <description> - Provides an option for client to load supported resource types from RM - instead of depending on local resource-types.xml file. + Specify docker command plugin for GPU. By default uses Nvidia docker V1. </description> - <name>yarn.client.load.resource-types.from-server</name> - <value>false</value> + <name>yarn.nodemanager.resource-plugins.gpu.docker-plugin</name> + <value>nvidia-docker-v1</value> + </property> + + <property> + <description> + Specify end point of nvidia-docker-plugin. + Please find documentation: https://github.com/NVIDIA/nvidia-docker/wiki + For more details. + </description> + <name>yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint</name> + <value>http://localhost:3476/v1.0/docker/cli</value> </property> +>>>>>>> theirs + </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index 865d18d..e8c46a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.nodemanager; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -325,7 +324,7 @@ public class LinuxContainerExecutor extends ContainerExecutor { if (linuxContainerRuntime == null) { LinuxContainerRuntime runtime = new DelegatingLinuxContainerRuntime(); - runtime.initialize(conf); + runtime.initialize(conf, nmContext); this.linuxContainerRuntime = runtime; } } catch (ContainerExecutionException e) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java index d6bae09..f2bb342 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java @@ -26,12 +26,11 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice; import java.io.IOException; import java.io.Serializable; @@ -54,8 +53,8 @@ import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI; public class GpuResourceAllocator { final static Log LOG = LogFactory.getLog(GpuResourceAllocator.class); - private Set<Integer> allowedGpuDevices = new TreeSet<>(); - private Map<Integer, ContainerId> usedDevices = new TreeMap<>(); + private Set<GpuDevice> allowedGpuDevices = new TreeSet<>(); + private Map<GpuDevice, ContainerId> usedDevices = new TreeMap<>(); private Context nmContext; public GpuResourceAllocator(Context ctx) { @@ -63,14 +62,14 @@ public class GpuResourceAllocator { } /** - * Contains allowed and denied devices with minor number. + * Contains allowed and denied devices * Denied devices will be useful for cgroups devices module to do blacklisting */ static class GpuAllocation { - private Set<Integer> allowed = Collections.emptySet(); - private Set<Integer> denied = Collections.emptySet(); + private Set<GpuDevice> allowed = Collections.emptySet(); + private Set<GpuDevice> denied = Collections.emptySet(); - GpuAllocation(Set<Integer> allowed, Set<Integer> denied) { + GpuAllocation(Set<GpuDevice> allowed, Set<GpuDevice> denied) { if (allowed != null) { this.allowed = ImmutableSet.copyOf(allowed); } @@ -79,21 +78,21 @@ public class GpuResourceAllocator { } } - public Set<Integer> getAllowedGPUs() { + public Set<GpuDevice> getAllowedGPUs() { return allowed; } - public Set<Integer> getDeniedGPUs() { + public Set<GpuDevice> getDeniedGPUs() { return denied; } } /** * Add GPU to allowed list - * @param minorNumber minor number of the GPU device. + * @param gpuDevice gpu device */ - public synchronized void addGpu(int minorNumber) { - allowedGpuDevices.add(minorNumber); + public synchronized void addGpu(GpuDevice gpuDevice) { + allowedGpuDevices.add(gpuDevice); } private String getResourceHandlerExceptionMessage(int numRequestedGpuDevices, @@ -117,42 +116,42 @@ public class GpuResourceAllocator { + containerId); } - for (Serializable deviceId : c.getResourceMappings().getAssignedResources( - GPU_URI)){ - if (!(deviceId instanceof String)) { + for (Serializable gpuDeviceSerializable : c.getResourceMappings() + .getAssignedResources(GPU_URI)) { + if (!(gpuDeviceSerializable instanceof GpuDevice)) { throw new ResourceHandlerException( "Trying to recover device id, however it" - + " is not String, this shouldn't happen"); + + " is not GpuDevice, this shouldn't happen"); } - - int devId; - try { - devId = Integer.parseInt((String)deviceId); - } catch (NumberFormatException e) { - throw new ResourceHandlerException("Failed to recover device id because" - + "it is not a valid integer, devId:" + deviceId); - } + GpuDevice gpuDevice = (GpuDevice) gpuDeviceSerializable; // Make sure it is in allowed GPU device. - if (!allowedGpuDevices.contains(devId)) { - throw new ResourceHandlerException("Try to recover device id = " + devId - + " however it is not in allowed device list:" + StringUtils - .join(",", allowedGpuDevices)); + if (!allowedGpuDevices.contains(gpuDevice)) { + throw new ResourceHandlerException( + "Try to recover device = " + gpuDevice + + " however it is not in allowed device list:" + StringUtils + .join(",", allowedGpuDevices)); } // Make sure it is not occupied by anybody else - if (usedDevices.containsKey(devId)) { - throw new ResourceHandlerException("Try to recover device id = " + devId - + " however it is already assigned to container=" + usedDevices - .get(devId) + ", please double check what happened."); + if (usedDevices.containsKey(gpuDevice)) { + throw new ResourceHandlerException( + "Try to recover device id = " + gpuDevice + + " however it is already assigned to container=" + usedDevices + .get(gpuDevice) + ", please double check what happened."); } - usedDevices.put(devId, containerId); + usedDevices.put(gpuDevice, containerId); } } - private int getRequestedGpus(Resource requestedResource) { + /** + * Get number of requested GPUs from resource. + * @param requestedResource requested resource + * @return #gpus. + */ + public static int getRequestedGpus(Resource requestedResource) { try { return Long.valueOf(requestedResource.getResourceValue( GPU_URI)).intValue(); @@ -164,8 +163,8 @@ public class GpuResourceAllocator { /** * Assign GPU to requestor * @param container container to allocate - * @return List of denied Gpus with minor numbers - * @throws ResourceHandlerException When failed to + * @return allocation results. + * @throws ResourceHandlerException When failed to assign GPUs. */ public synchronized GpuAllocation assignGpus(Container container) throws ResourceHandlerException { @@ -180,12 +179,12 @@ public class GpuResourceAllocator { containerId)); } - Set<Integer> assignedGpus = new HashSet<>(); + Set<GpuDevice> assignedGpus = new TreeSet<>(); - for (int deviceNum : allowedGpuDevices) { - if (!usedDevices.containsKey(deviceNum)) { - usedDevices.put(deviceNum, containerId); - assignedGpus.add(deviceNum); + for (GpuDevice gpu : allowedGpuDevices) { + if (!usedDevices.containsKey(gpu)) { + usedDevices.put(gpu, containerId); + assignedGpus.add(gpu); if (assignedGpus.size() == numRequestedGpuDevices) { break; } @@ -194,21 +193,10 @@ public class GpuResourceAllocator { // Record in state store if we allocated anything if (!assignedGpus.isEmpty()) { - List<Serializable> allocatedDevices = new ArrayList<>(); - for (int gpu : assignedGpus) { - allocatedDevices.add(String.valueOf(gpu)); - } try { - // Update Container#getResourceMapping. - ResourceMappings.AssignedResources assignedResources = - new ResourceMappings.AssignedResources(); - assignedResources.updateAssignedResources(allocatedDevices); - container.getResourceMappings().addAssignedResources(GPU_URI, - assignedResources); - // Update state store. - nmContext.getNMStateStore().storeAssignedResources(containerId, - GPU_URI, allocatedDevices); + nmContext.getNMStateStore().storeAssignedResources(container, GPU_URI, + new ArrayList<>(assignedGpus)); } catch (IOException e) { cleanupAssignGpus(containerId); throw new ResourceHandlerException(e); @@ -226,7 +214,7 @@ public class GpuResourceAllocator { * @param containerId containerId */ public synchronized void cleanupAssignGpus(ContainerId containerId) { - Iterator<Map.Entry<Integer, ContainerId>> iter = + Iterator<Map.Entry<GpuDevice, ContainerId>> iter = usedDevices.entrySet().iterator(); while (iter.hasNext()) { if (iter.next().getValue().equals(containerId)) { @@ -236,7 +224,7 @@ public class GpuResourceAllocator { } @VisibleForTesting - public synchronized Map<Integer, ContainerId> getDeviceAllocationMapping() { + public synchronized Map<GpuDevice, ContainerId> getDeviceAllocationMapping() { return new HashMap<>(usedDevices); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java index 7144bb2..2dbd3e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java @@ -24,8 +24,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ResourceInformation; -import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -35,6 +33,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDiscoverer; import java.util.ArrayList; @@ -64,17 +64,23 @@ public class GpuResourceHandlerImpl implements ResourceHandler { @Override public List<PrivilegedOperation> bootstrap(Configuration configuration) throws ResourceHandlerException { - List<Integer> minorNumbersOfUsableGpus; + List<GpuDevice> usableGpus; try { - minorNumbersOfUsableGpus = GpuDiscoverer.getInstance() - .getMinorNumbersOfGpusUsableByYarn(); + usableGpus = GpuDiscoverer.getInstance() + .getGpusUsableByYarn(); + if (usableGpus == null || usableGpus.isEmpty()) { + String message = "GPU is enabled on the NodeManager, but couldn't find " + + "any usable GPU devices, please double check configuration."; + LOG.error(message); + throw new ResourceHandlerException(message); + } } catch (YarnException e) { LOG.error("Exception when trying to get usable GPU device", e); throw new ResourceHandlerException(e); } - for (int minorNumber : minorNumbersOfUsableGpus) { - gpuAllocator.addGpu(minorNumber); + for (GpuDevice gpu : usableGpus) { + gpuAllocator.addGpu(gpu); } // And initialize cgroups @@ -96,33 +102,55 @@ public class GpuResourceHandlerImpl implements ResourceHandler { // Create device cgroups for the container cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES, containerIdStr); - try { - // Execute c-e to setup GPU isolation before launch the container - PrivilegedOperation privilegedOperation = new PrivilegedOperation( - PrivilegedOperation.OperationType.GPU, Arrays - .asList(CONTAINER_ID_CLI_OPTION, containerIdStr)); - if (!allocation.getDeniedGPUs().isEmpty()) { - privilegedOperation.appendArgs(Arrays.asList(EXCLUDED_GPUS_CLI_OPTION, - StringUtils.join(",", allocation.getDeniedGPUs()))); + if (!DockerLinuxContainerRuntime.isDockerContainerRequested( + container.getLaunchContext().getEnvironment())) { + // Write to devices cgroup only for non-docker container. The reason is + // docker engine runtime runc do the devices cgroups initialize in the + // pre-hook, see: + // https://github.com/opencontainers/runc/blob/master/libcontainer/configs/device_defaults.go + // + // YARN by default runs docker container inside cgroup, if we setup cgroups + // devices.deny for the parent cgroup for launched container, we can see + // errors like: failed to write c *:* m to devices.allow: + // write path-to-parent-cgroup/<container-id>/devices.allow: + // operation not permitted. + // + // To avoid this happen, if docker is requested when container being + // launched, we will not setup devices.deny for the container. Instead YARN + // will pass --device parameter to docker engine. See NvidiaDockerV1CommandPlugin + try { + // Execute c-e to setup GPU isolation before launch the container + PrivilegedOperation privilegedOperation = new PrivilegedOperation( + PrivilegedOperation.OperationType.GPU, + Arrays.asList(CONTAINER_ID_CLI_OPTION, containerIdStr)); + if (!allocation.getDeniedGPUs().isEmpty()) { + List<Integer> minorNumbers = new ArrayList<>(); + for (GpuDevice deniedGpu : allocation.getDeniedGPUs()) { + minorNumbers.add(deniedGpu.getMinorNumber()); + } + privilegedOperation.appendArgs(Arrays.asList(EXCLUDED_GPUS_CLI_OPTION, + StringUtils.join(",", minorNumbers))); + } + + privilegedOperationExecutor.executePrivilegedOperation( + privilegedOperation, true); + } catch (PrivilegedOperationException e) { + cGroupsHandler.deleteCGroup(CGroupsHandler.CGroupController.DEVICES, + containerIdStr); + LOG.warn("Could not update cgroup for container", e); + throw new ResourceHandlerException(e); } - privilegedOperationExecutor.executePrivilegedOperation( - privilegedOperation, true); - } catch (PrivilegedOperationException e) { - cGroupsHandler.deleteCGroup(CGroupsHandler.CGroupController.DEVICES, - containerIdStr); - LOG.warn("Could not update cgroup for container", e); - throw new ResourceHandlerException(e); - } + List<PrivilegedOperation> ret = new ArrayList<>(); + ret.add(new PrivilegedOperation( + PrivilegedOperation.OperationType.ADD_PID_TO_CGROUP, + PrivilegedOperation.CGROUP_ARG_PREFIX + cGroupsHandler + .getPathForCGroupTasks(CGroupsHandler.CGroupController.DEVICES, + containerIdStr))); - List<PrivilegedOperation> ret = new ArrayList<>(); - ret.add(new PrivilegedOperation( - PrivilegedOperation.OperationType.ADD_PID_TO_CGROUP, - PrivilegedOperation.CGROUP_ARG_PREFIX - + cGroupsHandler.getPathForCGroupTasks( - CGroupsHandler.CGroupController.DEVICES, containerIdStr))); - - return ret; + return ret; + } + return null; } @VisibleForTesting http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java index e9c58b8..b50d56c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; +import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; @@ -67,7 +68,7 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime { } @Override - public void initialize(Configuration conf) + public void initialize(Configuration conf, Context nmContext) throws ContainerExecutionException { this.conf = conf; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java index 517a4e2..dd10617 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; @@ -57,7 +58,7 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime { EnumSet.noneOf(LinuxContainerRuntimeConstants.RuntimeType.class); @Override - public void initialize(Configuration conf) + public void initialize(Configuration conf, Context nmContext) throws ContainerExecutionException { String[] configuredRuntimes = conf.getTrimmedStrings( YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES, @@ -77,19 +78,19 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime { LinuxContainerRuntimeConstants.RuntimeType.JAVASANDBOX)) { javaSandboxLinuxContainerRuntime = new JavaSandboxLinuxContainerRuntime( PrivilegedOperationExecutor.getInstance(conf)); - javaSandboxLinuxContainerRuntime.initialize(conf); + javaSandboxLinuxContainerRuntime.initialize(conf, nmContext); } if (isRuntimeAllowed( LinuxContainerRuntimeConstants.RuntimeType.DOCKER)) { dockerLinuxContainerRuntime = new DockerLinuxContainerRuntime( PrivilegedOperationExecutor.getInstance(conf)); - dockerLinuxContainerRuntime.initialize(conf); + dockerLinuxContainerRuntime.initialize(conf, nmContext); } if (isRuntimeAllowed( LinuxContainerRuntimeConstants.RuntimeType.DEFAULT)) { defaultLinuxContainerRuntime = new DefaultLinuxContainerRuntime( PrivilegedOperationExecutor.getInstance(conf)); - defaultLinuxContainerRuntime.initialize(conf); + defaultLinuxContainerRuntime.initialize(conf, nmContext); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 2013306..6f7b6fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -21,6 +21,10 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerVolumeCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.DockerCommandPlugin; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -172,6 +176,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { "YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS"; private Configuration conf; + private Context nmContext; private DockerClient dockerClient; private PrivilegedOperationExecutor privilegedOperationExecutor; private Set<String> allowedNetworks = new HashSet<>(); @@ -220,14 +225,14 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { * Create an instance using the given {@link PrivilegedOperationExecutor} * instance for performing operations and the given {@link CGroupsHandler} * instance. This constructor is intended for use in testing. - * - * @param privilegedOperationExecutor the {@link PrivilegedOperationExecutor} + * @param privilegedOperationExecutor the {@link PrivilegedOperationExecutor} * instance * @param cGroupsHandler the {@link CGroupsHandler} instance */ @VisibleForTesting - public DockerLinuxContainerRuntime(PrivilegedOperationExecutor - privilegedOperationExecutor, CGroupsHandler cGroupsHandler) { + public DockerLinuxContainerRuntime( + PrivilegedOperationExecutor privilegedOperationExecutor, + CGroupsHandler cGroupsHandler) { this.privilegedOperationExecutor = privilegedOperationExecutor; if (cGroupsHandler == null) { @@ -239,8 +244,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } @Override - public void initialize(Configuration conf) + public void initialize(Configuration conf, Context nmContext) throws ContainerExecutionException { + this.nmContext = nmContext; this.conf = conf; dockerClient = new DockerClient(conf); allowedNetworks.clear(); @@ -288,9 +294,54 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { return false; } + private void runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand, + Container container) throws ContainerExecutionException { + try { + String commandFile = dockerClient.writeCommandToTempFile( + dockerVolumeCommand, container.getContainerId().toString()); + PrivilegedOperation privOp = new PrivilegedOperation( + PrivilegedOperation.OperationType.RUN_DOCKER_CMD); + privOp.appendArgs(commandFile); + String output = privilegedOperationExecutor + .executePrivilegedOperation(null, privOp, null, + null, true, false); + LOG.info("ContainerId=" + container.getContainerId() + + ", docker volume output for " + dockerVolumeCommand + ": " + + output); + } catch (ContainerExecutionException e) { + LOG.error("Error when writing command to temp file, command=" + + dockerVolumeCommand, + e); + throw e; + } catch (PrivilegedOperationException e) { + LOG.error("Error when executing command, command=" + + dockerVolumeCommand, e); + throw new ContainerExecutionException(e); + } + + } + @Override public void prepareContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { + Container container = ctx.getContainer(); + + // Create volumes when needed. + if (nmContext != null + && nmContext.getResourcePluginManager().getNameToPlugins() != null) { + for (ResourcePlugin plugin : nmContext.getResourcePluginManager() + .getNameToPlugins().values()) { + DockerCommandPlugin dockerCommandPlugin = + plugin.getDockerCommandPluginInstance(); + if (dockerCommandPlugin != null) { + DockerVolumeCommand dockerVolumeCommand = + dockerCommandPlugin.getCreateDockerVolumeCommand(ctx.getContainer()); + if (dockerVolumeCommand != null) { + runDockerVolumeCommand(dockerVolumeCommand, container); + } + } + } + } } private void validateContainerNetworkType(String network) @@ -623,6 +674,19 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { runCommand.groupAdd(groups); } + // use plugins to update docker run command. + if (nmContext != null + && nmContext.getResourcePluginManager().getNameToPlugins() != null) { + for (ResourcePlugin plugin : nmContext.getResourcePluginManager() + .getNameToPlugins().values()) { + DockerCommandPlugin dockerCommandPlugin = + plugin.getDockerCommandPluginInstance(); + if (dockerCommandPlugin != null) { + dockerCommandPlugin.updateDockerRunCommand(runCommand, container); + } + } + } + String commandFile = dockerClient.writeCommandToTempFile(runCommand, containerIdStr); PrivilegedOperation launchOp = buildLaunchOp(ctx, @@ -683,6 +747,23 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { @Override public void reapContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { + // Cleanup volumes when needed. + if (nmContext != null + && nmContext.getResourcePluginManager().getNameToPlugins() != null) { + for (ResourcePlugin plugin : nmContext.getResourcePluginManager() + .getNameToPlugins().values()) { + DockerCommandPlugin dockerCommandPlugin = + plugin.getDockerCommandPluginInstance(); + if (dockerCommandPlugin != null) { + DockerVolumeCommand dockerVolumeCommand = + dockerCommandPlugin.getCleanupDockerVolumesCommand( + ctx.getContainer()); + if (dockerVolumeCommand != null) { + runDockerVolumeCommand(dockerVolumeCommand, ctx.getContainer()); + } + } + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java index cfafcde..245b38f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.Groups; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; @@ -143,7 +144,7 @@ public class JavaSandboxLinuxContainerRuntime } @Override - public void initialize(Configuration conf) + public void initialize(Configuration conf, Context nmContext) throws ContainerExecutionException { this.configuration = conf; this.sandboxMode = @@ -151,7 +152,7 @@ public class JavaSandboxLinuxContainerRuntime this.configuration.get(YARN_CONTAINER_SANDBOX, YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX)); - super.initialize(conf); + super.initialize(conf, nmContext); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java index cd7a2f3..1203ce5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java @@ -23,6 +23,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime; @@ -38,9 +39,10 @@ public interface LinuxContainerRuntime extends ContainerRuntime { * Initialize the runtime. * * @param conf the {@link Configuration} to use + * @param nmContext NMContext * @throws ContainerExecutionException if an error occurs while initializing * the runtime */ - void initialize(Configuration conf) throws ContainerExecutionException; + void initialize(Configuration conf, Context nmContext) throws ContainerExecutionException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index c7bf827..8734ba6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -76,6 +76,11 @@ public class DockerRunCommand extends DockerCommand { return this; } + public DockerRunCommand setVolumeDriver(String volumeDriver) { + super.addCommandArguments("volume-driver", volumeDriver); + return this; + } + public DockerRunCommand setCGroupParent(String parentPath) { super.addCommandArguments("cgroup-parent", parentPath); return this; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerVolumeCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerVolumeCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerVolumeCommand.java new file mode 100644 index 0000000..a477c93 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerVolumeCommand.java @@ -0,0 +1,49 @@ +/* + * * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +import java.util.regex.Pattern; + +/** + * Docker Volume Command, run "docker volume --help" for more details. + */ +public class DockerVolumeCommand extends DockerCommand { + public static final String VOLUME_COMMAND = "volume"; + public static final String VOLUME_CREATE_COMMAND = "create"; + // Regex pattern for volume name + public static final Pattern VOLUME_NAME_PATTERN = Pattern.compile( + "[a-zA-Z0-9][a-zA-Z0-9_.-]*"); + + public DockerVolumeCommand(String subCommand) { + super(VOLUME_COMMAND); + super.addCommandArguments("sub-command", subCommand); + } + + public DockerVolumeCommand setVolumeName(String volumeName) { + super.addCommandArguments("volume", volumeName); + return this; + } + + public DockerVolumeCommand setDriverName(String driverName) { + super.addCommandArguments("driver", driverName); + return this; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/DockerCommandPlugin.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/DockerCommandPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/DockerCommandPlugin.java new file mode 100644 index 0000000..9d046bc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/DockerCommandPlugin.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin; + +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerVolumeCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; + +/** + * Interface to make different resource plugins (e.g. GPU) can update docker run + * command without adding logic to Docker runtime. + */ +public interface DockerCommandPlugin { + /** + * Update docker run command + * @param dockerRunCommand docker run command + * @param container NM container + * @throws ContainerExecutionException if any issue occurs + */ + void updateDockerRunCommand(DockerRunCommand dockerRunCommand, + Container container) throws ContainerExecutionException; + + /** + * Create volume when needed. + * @param container container + * @return {@link DockerVolumeCommand} to create volume + * @throws ContainerExecutionException when any issue happens + */ + DockerVolumeCommand getCreateDockerVolumeCommand(Container container) + throws ContainerExecutionException; + + /** + * Cleanup volumes created for one docker container + * @param container container + * @return {@link DockerVolumeCommand} to remove volume + * @throws ContainerExecutionException when any issue happens + */ + DockerVolumeCommand getCleanupDockerVolumesCommand(Container container) + throws ContainerExecutionException; + + // Add support to other docker command when required. +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java index 6e134b3..99a18ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; /** * {@link ResourcePlugin} is an interface for node manager to easier support @@ -80,4 +81,14 @@ public interface ResourcePlugin { * @throws YarnException if any issue occurs */ void cleanup() throws YarnException; + + /** + * Plugin need to get {@link DockerCommandPlugin}. This will be invoked by + * {@link DockerLinuxContainerRuntime} when execute docker commands such as + * run/stop/pull, etc. + * + * @return DockerCommandPlugin instance. return null if plugin doesn't + * have requirement to update docker command. + */ + DockerCommandPlugin getDockerCommandPluginInstance(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java new file mode 100644 index 0000000..8119924 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu; + +import java.io.Serializable; + +/** + * This class is used to represent GPU device while allocation. + */ +public class GpuDevice implements Serializable, Comparable { + private int index; + private int minorNumber; + private static final long serialVersionUID = -6812314470754667710L; + + public GpuDevice(int index, int minorNumber) { + this.index = index; + this.minorNumber = minorNumber; + } + + public int getIndex() { + return index; + } + + public int getMinorNumber() { + return minorNumber; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof GpuDevice)) { + return false; + } + GpuDevice other = (GpuDevice) obj; + return index == other.index && minorNumber == other.minorNumber; + } + + @Override + public int compareTo(Object obj) { + if (obj == null || (!(obj instanceof GpuDevice))) { + return -1; + } + + GpuDevice other = (GpuDevice) obj; + + int result = Integer.compare(index, other.index); + if (0 != result) { + return result; + } + return Integer.compare(minorNumber, other.minorNumber); + } + + @Override + public int hashCode() { + final int prime = 47; + return prime * index + minorNumber; + } + + @Override + public String toString() { + return "(index=" + index + ",minor_number=" + minorNumber + ")"; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java index 61b8ce5..6e3cf13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java @@ -136,12 +136,12 @@ public class GpuDiscoverer { } /** - * Get list of minor device numbers of Gpu devices usable by YARN. + * Get list of GPU devices usable by YARN. * - * @return List of minor device numbers of Gpu devices. + * @return List of GPU devices * @throws YarnException when any issue happens */ - public synchronized List<Integer> getMinorNumbersOfGpusUsableByYarn() + public synchronized List<GpuDevice> getGpusUsableByYarn() throws YarnException { validateConfOrThrowException(); @@ -149,7 +149,7 @@ public class GpuDiscoverer { YarnConfiguration.NM_GPU_ALLOWED_DEVICES, YarnConfiguration.AUTOMATICALLY_DISCOVER_GPU_DEVICES); - List<Integer> minorNumbers = new ArrayList<>(); + List<GpuDevice> gpuDevices = new ArrayList<>(); if (allowedDevicesStr.equals( YarnConfiguration.AUTOMATICALLY_DISCOVER_GPU_DEVICES)) { @@ -167,21 +167,31 @@ public class GpuDiscoverer { } if (lastDiscoveredGpuInformation.getGpus() != null) { - for (PerGpuDeviceInformation gpu : lastDiscoveredGpuInformation - .getGpus()) { - minorNumbers.add(gpu.getMinorNumber()); + for (int i = 0; i < lastDiscoveredGpuInformation.getGpus().size(); + i++) { + List<PerGpuDeviceInformation> gpuInfos = + lastDiscoveredGpuInformation.getGpus(); + gpuDevices.add(new GpuDevice(i, gpuInfos.get(i).getMinorNumber())); } } } else{ for (String s : allowedDevicesStr.split(",")) { if (s.trim().length() > 0) { - minorNumbers.add(Integer.valueOf(s.trim())); + String[] kv = s.trim().split(":"); + if (kv.length != 2) { + throw new YarnException( + "Illegal format, it should be index:minor_number format, now it=" + + s); + } + + gpuDevices.add( + new GpuDevice(Integer.parseInt(kv[0]), Integer.parseInt(kv[1]))); } } - LOG.info("Allowed GPU devices with minor numbers:" + allowedDevicesStr); + LOG.info("Allowed GPU devices:" + gpuDevices); } - return minorNumbers; + return gpuDevices; } public synchronized void initialize(Configuration conf) throws YarnException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java new file mode 100644 index 0000000..db4589a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.DockerCommandPlugin; + +/** + * Factory to create GpuDocker Command Plugin instance + */ +public class GpuDockerCommandPluginFactory { + public static DockerCommandPlugin createGpuDockerCommandPlugin( + Configuration conf) throws YarnException { + String impl = conf.get(YarnConfiguration.NM_GPU_DOCKER_PLUGIN_IMPL, + YarnConfiguration.DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL); + if (impl.equals(YarnConfiguration.NVIDIA_DOCKER_V1)) { + return new NvidiaDockerV1CommandPlugin(conf); + } + + throw new YarnException( + "Unkown implementation name for Gpu docker plugin, impl=" + impl); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java index f6bf506..796eb25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java @@ -40,12 +40,14 @@ public class GpuNodeResourceUpdateHandler extends NodeResourceUpdaterPlugin { public void updateConfiguredResource(Resource res) throws YarnException { LOG.info("Initializing configured GPU resources for the NodeManager."); - List<Integer> usableGpus = - GpuDiscoverer.getInstance().getMinorNumbersOfGpusUsableByYarn(); + List<GpuDevice> usableGpus = + GpuDiscoverer.getInstance().getGpusUsableByYarn(); if (null == usableGpus || usableGpus.isEmpty()) { - LOG.info("Didn't find any usable GPUs on the NodeManager."); + String message = "GPU is enabled, but couldn't find any usable GPUs on the " + + "NodeManager."; + LOG.error(message); // No gpu can be used by YARN. - return; + throw new YarnException(message); } long nUsableGpus = usableGpus.size(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java index 9576ce7..4ff186f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java @@ -24,17 +24,22 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceHandlerImpl; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.DockerCommandPlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; public class GpuResourcePlugin implements ResourcePlugin { private ResourceHandler gpuResourceHandler = null; private GpuNodeResourceUpdateHandler resourceDiscoverHandler = null; + private DockerCommandPlugin dockerCommandPlugin = null; @Override public synchronized void initialize(Context context) throws YarnException { resourceDiscoverHandler = new GpuNodeResourceUpdateHandler(); GpuDiscoverer.getInstance().initialize(context.getConf()); + dockerCommandPlugin = + GpuDockerCommandPluginFactory.createGpuDockerCommandPlugin( + context.getConf()); } @Override @@ -58,4 +63,8 @@ public class GpuResourcePlugin implements ResourcePlugin { public void cleanup() throws YarnException { // Do nothing. } + + public DockerCommandPlugin getDockerCommandPluginInstance() { + return dockerCommandPlugin; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/NvidiaDockerV1CommandPlugin.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/NvidiaDockerV1CommandPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/NvidiaDockerV1CommandPlugin.java new file mode 100644 index 0000000..73d7048 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/NvidiaDockerV1CommandPlugin.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceAllocator; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerVolumeCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.DockerCommandPlugin; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; + +import java.io.IOException; +import java.io.Serializable; +import java.io.StringWriter; +import java.net.URL; +import java.net.URLConnection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerVolumeCommand.VOLUME_NAME_PATTERN; + +/** + * Implementation to use nvidia-docker v1 as GPU docker command plugin. + */ +public class NvidiaDockerV1CommandPlugin implements DockerCommandPlugin { + final static Log LOG = LogFactory.getLog(NvidiaDockerV1CommandPlugin.class); + + private Configuration conf; + private Map<String, Set<String>> additionalCommands = null; + private String volumeDriver = "local"; + + // Known option + private String DEVICE_OPTION = "--device"; + private String VOLUME_DRIVER_OPTION = "--volume-driver"; + private String MOUNT_RO_OPTION = "--volume"; + + public NvidiaDockerV1CommandPlugin(Configuration conf) { + this.conf = conf; + } + + // Get value from key=value + // Throw exception if '=' not found + private String getValue(String input) throws IllegalArgumentException { + int index = input.indexOf('='); + if (index < 0) { + throw new IllegalArgumentException( + "Failed to locate '=' from input=" + input); + } + return input.substring(index + 1); + } + + private void addToCommand(String key, String value) { + if (additionalCommands == null) { + additionalCommands = new HashMap<>(); + } + if (!additionalCommands.containsKey(key)) { + additionalCommands.put(key, new HashSet<>()); + } + additionalCommands.get(key).add(value); + } + + private void init() throws ContainerExecutionException { + String endpoint = conf.get( + YarnConfiguration.NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT, + YarnConfiguration.DEFAULT_NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT); + if (null == endpoint || endpoint.isEmpty()) { + LOG.info(YarnConfiguration.NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT + + " set to empty, skip init .."); + return; + } + String cliOptions; + try { + // Talk to plugin server and get options + URL url = new URL(endpoint); + URLConnection uc = url.openConnection(); + uc.setRequestProperty("X-Requested-With", "Curl"); + + StringWriter writer = new StringWriter(); + IOUtils.copy(uc.getInputStream(), writer, "utf-8"); + cliOptions = writer.toString(); + + LOG.info("Additional docker CLI options from plugin to run GPU " + + "containers:" + cliOptions); + + // Parse cli options + // Examples like: + // --device=/dev/nvidiactl --device=/dev/nvidia-uvm --device=/dev/nvidia0 + // --volume-driver=nvidia-docker + // --volume=nvidia_driver_352.68:/usr/local/nvidia:ro + + for (String str : cliOptions.split(" ")) { + str = str.trim(); + if (str.startsWith(DEVICE_OPTION)) { + addToCommand(DEVICE_OPTION, getValue(str)); + } else if (str.startsWith(VOLUME_DRIVER_OPTION)) { + volumeDriver = getValue(str); + if (LOG.isDebugEnabled()) { + LOG.debug("Found volume-driver:" + volumeDriver); + } + } else if (str.startsWith(MOUNT_RO_OPTION)) { + String mount = getValue(str); + if (!mount.endsWith(":ro")) { + throw new IllegalArgumentException( + "Should not have mount other than ro, command=" + str); + } + addToCommand(MOUNT_RO_OPTION, + mount.substring(0, mount.lastIndexOf(':'))); + } else{ + throw new IllegalArgumentException("Unsupported option:" + str); + } + } + } catch (RuntimeException e) { + LOG.warn( + "RuntimeException of " + this.getClass().getSimpleName() + " init:", + e); + throw new ContainerExecutionException(e); + } catch (IOException e) { + LOG.warn("IOException of " + this.getClass().getSimpleName() + " init:", + e); + throw new ContainerExecutionException(e); + } + } + + private int getGpuIndexFromDeviceName(String device) { + final String NVIDIA = "nvidia"; + int idx = device.lastIndexOf(NVIDIA); + if (idx < 0) { + return -1; + } + // Get last part + String str = device.substring(idx + NVIDIA.length()); + for (int i = 0; i < str.length(); i++) { + if (!Character.isDigit(str.charAt(i))) { + return -1; + } + } + return Integer.parseInt(str); + } + + private Set<GpuDevice> getAssignedGpus(Container container) { + ResourceMappings resourceMappings = container.getResourceMappings(); + + // Copy of assigned Resources + Set<GpuDevice> assignedResources = null; + if (resourceMappings != null) { + assignedResources = new HashSet<>(); + for (Serializable s : resourceMappings.getAssignedResources( + ResourceInformation.GPU_URI)) { + assignedResources.add((GpuDevice) s); + } + } + + if (assignedResources == null || assignedResources.isEmpty()) { + // When no GPU resource assigned, don't need to update docker command. + return Collections.emptySet(); + } + + return assignedResources; + } + + @VisibleForTesting + protected boolean requestsGpu(Container container) { + return GpuResourceAllocator.getRequestedGpus(container.getResource()) > 0; + } + + /** + * Do initialize when GPU requested + * @param container nmContainer + * @return if #GPU-requested > 0 + * @throws ContainerExecutionException when any issue happens + */ + private boolean initializeWhenGpuRequested(Container container) + throws ContainerExecutionException { + if (!requestsGpu(container)) { + return false; + } + + // Do lazy initialization of gpu-docker plugin + if (additionalCommands == null) { + init(); + } + + return true; + } + + @Override + public synchronized void updateDockerRunCommand( + DockerRunCommand dockerRunCommand, Container container) + throws ContainerExecutionException { + if (!initializeWhenGpuRequested(container)) { + return; + } + + Set<GpuDevice> assignedResources = getAssignedGpus(container); + if (assignedResources == null || assignedResources.isEmpty()) { + return; + } + + // Write to dockerRunCommand + for (Map.Entry<String, Set<String>> option : additionalCommands + .entrySet()) { + String key = option.getKey(); + Set<String> values = option.getValue(); + if (key.equals(DEVICE_OPTION)) { + int foundGpuDevices = 0; + for (String deviceName : values) { + // When specified is a GPU card (device name like /dev/nvidia[n] + // Get index of the GPU (which is [n]). + Integer gpuIdx = getGpuIndexFromDeviceName(deviceName); + if (gpuIdx >= 0) { + // Use assignedResources to filter --device given by + // nvidia-docker-plugin. + for (GpuDevice gpuDevice : assignedResources) { + if (gpuDevice.getIndex() == gpuIdx) { + foundGpuDevices++; + dockerRunCommand.addDevice(deviceName, deviceName); + } + } + } else{ + // When gpuIdx < 0, it is a controller device (such as + // /dev/nvidiactl). In this case, add device directly. + dockerRunCommand.addDevice(deviceName, deviceName); + } + } + + // Cannot get all assigned Gpu devices from docker plugin output + if (foundGpuDevices < assignedResources.size()) { + throw new ContainerExecutionException( + "Cannot get all assigned Gpu devices from docker plugin output"); + } + } else if (key.equals(MOUNT_RO_OPTION)) { + for (String value : values) { + int idx = value.indexOf(':'); + String source = value.substring(0, idx); + String target = value.substring(idx + 1); + dockerRunCommand.addReadOnlyMountLocation(source, target, true); + } + } else{ + throw new ContainerExecutionException("Unsupported option:" + key); + } + } + } + + @Override + public DockerVolumeCommand getCreateDockerVolumeCommand(Container container) + throws ContainerExecutionException { + if (!initializeWhenGpuRequested(container)) { + return null; + } + + String newVolumeName = null; + + // Get volume name + Set<String> mounts = additionalCommands.get(MOUNT_RO_OPTION); + for (String mount : mounts) { + int idx = mount.indexOf(':'); + if (idx >= 0) { + String mountSource = mount.substring(0, idx); + if (VOLUME_NAME_PATTERN.matcher(mountSource).matches()) { + // This is a valid named volume + newVolumeName = mountSource; + if (LOG.isDebugEnabled()) { + LOG.debug("Found volume name for GPU:" + newVolumeName); + } + break; + } else{ + if (LOG.isDebugEnabled()) { + LOG.debug("Failed to match " + mountSource + + " to named-volume regex pattern"); + } + } + } + } + + if (newVolumeName != null) { + DockerVolumeCommand command = new DockerVolumeCommand( + DockerVolumeCommand.VOLUME_CREATE_COMMAND); + command.setDriverName(volumeDriver); + command.setVolumeName(newVolumeName); + return command; + } + + return null; + } + + @Override + public DockerVolumeCommand getCleanupDockerVolumesCommand(Container container) + throws ContainerExecutionException { + // No cleanup needed. + return null; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9114d7a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java index c361d00..3455874 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java @@ -18,27 +18,9 @@ package org.apache.hadoop.yarn.server.nodemanager.recovery; -import static org.fusesource.leveldbjni.JniDBFactory.asString; -import static org.fusesource.leveldbjni.JniDBFactory.bytes; - -import org.apache.hadoop.yarn.api.records.Token; -import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Timer; -import java.util.TimerTask; -import java.util.Set; - +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -50,9 +32,11 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestP import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; +import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ContainerTokenIdentifierProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto; @@ -60,9 +44,10 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.Deletion import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto; -import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ContainerTokenIdentifierProto; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl; @@ -74,10 +59,24 @@ import org.iq80.leveldb.DB; import org.iq80.leveldb.DBException; import org.iq80.leveldb.Options; import org.iq80.leveldb.WriteBatch; +import org.slf4j.LoggerFactory; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ListMultimap; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; + +import static org.fusesource.leveldbjni.JniDBFactory.asString; +import static org.fusesource.leveldbjni.JniDBFactory.bytes; public class NMLeveldbStateStoreService extends NMStateStoreService { @@ -1173,15 +1172,17 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { } @Override - public void storeAssignedResources(ContainerId containerId, + public void storeAssignedResources(Container container, String resourceType, List<Serializable> assignedResources) throws IOException { if (LOG.isDebugEnabled()) { - LOG.debug("storeAssignedResources: containerId=" + containerId - + ", assignedResources=" + StringUtils.join(",", assignedResources)); + LOG.debug( + "storeAssignedResources: containerId=" + container.getContainerId() + + ", assignedResources=" + StringUtils + .join(",", assignedResources)); } - String keyResChng = CONTAINERS_KEY_PREFIX + containerId.toString() + String keyResChng = CONTAINERS_KEY_PREFIX + container.getContainerId().toString() + CONTAINER_ASSIGNED_RESOURCES_KEY_SUFFIX + resourceType; try { WriteBatch batch = db.createWriteBatch(); @@ -1199,6 +1200,9 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { } catch (DBException e) { throw new IOException(e); } + + // update container resource mapping. + updateContainerResourceMapping(container, resourceType, assignedResources); } @SuppressWarnings("deprecation") --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
