Repository: hadoop Updated Branches: refs/heads/branch-2 421e51cb9 -> 49a2f286f
YARN-6500. Do not mount inaccessible cgroups directories in CgroupsLCEResourcesHandler. (Miklos Szegedi via Haibo Chen) (cherry picked from commit 8ac50e1322cb3f84bd998635924d85846aa47c94) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49a2f286 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49a2f286 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49a2f286 Branch: refs/heads/branch-2 Commit: 49a2f286f72bab644fd6aa55b5bc51ae47120e1f Parents: 421e51c Author: Haibo Chen <haiboc...@cloudera.com> Authored: Mon Apr 24 11:37:52 2017 -0700 Committer: Haibo Chen <haiboc...@cloudera.com> Committed: Mon Apr 24 11:39:55 2017 -0700 ---------------------------------------------------------------------- .../linux/resources/CGroupsHandlerImpl.java | 3 +- .../util/CgroupsLCEResourcesHandler.java | 13 ++++++-- .../linux/resources/TestCGroupsHandlerImpl.java | 32 ++++++++++++++++++++ .../util/TestCgroupsLCEResourcesHandler.java | 29 ++++++++++++++++++ 4 files changed, 73 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a2f286/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java index d5295c5..0f4c17e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java @@ -232,7 +232,8 @@ class CGroupsHandlerImpl implements CGroupsHandler { * @param entries map of paths to mount options * @return the first mount path that has the requested subsystem */ - private static String findControllerInMtab(String controller, + @VisibleForTesting + static String findControllerInMtab(String controller, Map<String, List<String>> entries) { for (Map.Entry<String, List<String>> e : entries.entrySet()) { if (e.getValue().contains(controller)) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a2f286/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java index f04fcd2..cb4dcf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java @@ -428,11 +428,18 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler { return ret; } - private String findControllerInMtab(String controller, + @VisibleForTesting + String findControllerInMtab(String controller, Map<String, List<String>> entries) { for (Entry<String, List<String>> e : entries.entrySet()) { - if (e.getValue().contains(controller)) - return e.getKey(); + if (e.getValue().contains(controller)) { + if (new File(e.getKey()).canRead()) { + return e.getKey(); + } else { + LOG.warn(String.format( + "Skipping inaccessible cgroup mount point %s", e.getKey())); + } + } } return null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a2f286/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java index 4c0829e..804f8e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java @@ -40,6 +40,9 @@ import java.io.FileWriter; import java.io.IOException; import java.nio.file.Files; import java.security.Permission; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.UUID; @@ -432,6 +435,35 @@ public class TestCGroupsHandlerImpl { } } + @Test + public void testSelectCgroup() throws Exception { + File cpu = new File(tmpPath, "cpu"); + File cpuNoExist = new File(tmpPath, "cpuNoExist"); + File memory = new File(tmpPath, "memory"); + try { + CGroupsHandlerImpl handler = new CGroupsHandlerImpl( + conf, + privilegedOperationExecutorMock); + Map<String, List<String>> cgroups = new LinkedHashMap<>(); + + Assert.assertTrue("temp dir should be created", cpu.mkdirs()); + Assert.assertTrue("temp dir should be created", memory.mkdirs()); + Assert.assertFalse("temp dir should not be created", cpuNoExist.exists()); + + cgroups.put( + memory.getAbsolutePath(), Collections.singletonList("memory")); + cgroups.put( + cpuNoExist.getAbsolutePath(), Collections.singletonList("cpu")); + cgroups.put(cpu.getAbsolutePath(), Collections.singletonList("cpu")); + String selectedCPU = handler.findControllerInMtab("cpu", cgroups); + Assert.assertEquals("Wrong CPU mount point selected", + cpu.getAbsolutePath(), selectedCPU); + } finally { + FileUtils.deleteQuietly(cpu); + FileUtils.deleteQuietly(memory); + } + } + @After public void teardown() { FileUtil.fullyDelete(new File(tmpPath)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a2f286/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java index 34c9ad1..83c8d5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java @@ -33,7 +33,10 @@ import org.junit.Before; import org.mockito.Mockito; import java.io.*; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Scanner; import java.util.concurrent.CountDownLatch; @@ -342,4 +345,30 @@ public class TestCgroupsLCEResourcesHandler { FileUtils.deleteQuietly(cgroupDir); } + @Test + public void testSelectCgroup() { + File cpu = new File(cgroupDir, "cpu"); + File cpuNoExist = new File(cgroupDir, "cpuNoExist"); + File memory = new File(cgroupDir, "memory"); + try { + CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler(); + Map<String, List<String>> cgroups = new LinkedHashMap<>(); + + Assert.assertTrue("temp dir should be created", cpu.mkdirs()); + Assert.assertTrue("temp dir should be created", memory.mkdirs()); + Assert.assertFalse("temp dir should not be created", cpuNoExist.exists()); + + cgroups.put( + memory.getAbsolutePath(), Collections.singletonList("memory")); + cgroups.put( + cpuNoExist.getAbsolutePath(), Collections.singletonList("cpu")); + cgroups.put(cpu.getAbsolutePath(), Collections.singletonList("cpu")); + String selectedCPU = handler.findControllerInMtab("cpu", cgroups); + Assert.assertEquals("Wrong CPU mount point selected", + cpu.getAbsolutePath(), selectedCPU); + } finally { + FileUtils.deleteQuietly(cpu); + FileUtils.deleteQuietly(memory); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org