Repository: hadoop Updated Branches: refs/heads/branch-2 017dff987 -> c829be513
YARN-4095. Avoid sharing AllocatorPerContext object in LocalDirAllocator between ShuffleHandler and LocalDirsHandlerService. Contributed by Zhihai Xu (cherry picked from commit c890c51a916894a985439497b8a44e8eee82d762) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c829be51 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c829be51 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c829be51 Branch: refs/heads/branch-2 Commit: c829be5133a428b38b42de483bf23ce026d7ae1d Parents: 017dff9 Author: Jason Lowe <jl...@apache.org> Authored: Wed Sep 23 15:42:01 2015 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Wed Sep 23 15:44:13 2015 +0000 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../nodemanager/LocalDirsHandlerService.java | 33 +++++++++++++++++--- .../TestLocalDirsHandlerService.java | 18 +++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c829be51/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index df4bec9..f7ad148 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -408,6 +408,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12428. Fix inconsistency between log-level guards and statements. (Jagadesh Kiran N and Jackie Chang via ozawa) + YARN-4095. Avoid sharing AllocatorPerContext object in LocalDirAllocator + between ShuffleHandler and LocalDirsHandlerService. (Zhihai Xu via jlowe) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/c829be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java index 6709c90..769044a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java @@ -30,6 +30,7 @@ import java.util.TimerTask; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; @@ -52,6 +53,22 @@ public class LocalDirsHandlerService extends AbstractService { private static Log LOG = LogFactory.getLog(LocalDirsHandlerService.class); + /** + * Good local directories, use internally, + * initial value is the same as NM_LOCAL_DIRS. + */ + @Private + static final String NM_GOOD_LOCAL_DIRS = + YarnConfiguration.NM_PREFIX + "good-local-dirs"; + + /** + * Good log directories, use internally, + * initial value is the same as NM_LOG_DIRS. + */ + @Private + static final String NM_GOOD_LOG_DIRS = + YarnConfiguration.NM_PREFIX + "good-log-dirs"; + /** Timer used to schedule disk health monitoring code execution */ private Timer dirsHandlerScheduler; private long diskHealthCheckInterval; @@ -113,9 +130,17 @@ public class LocalDirsHandlerService extends AbstractService { new DirectoryCollection( validatePaths(conf.getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS)), maxUsableSpacePercentagePerDisk, minFreeSpacePerDiskMB); + + String local = conf.get(YarnConfiguration.NM_LOCAL_DIRS); + conf.set(NM_GOOD_LOCAL_DIRS, + (local != null) ? local : ""); localDirsAllocator = new LocalDirAllocator( - YarnConfiguration.NM_LOCAL_DIRS); - logDirsAllocator = new LocalDirAllocator(YarnConfiguration.NM_LOG_DIRS); + NM_GOOD_LOCAL_DIRS); + String log = conf.get(YarnConfiguration.NM_LOG_DIRS); + conf.set(NM_GOOD_LOG_DIRS, + (log != null) ? log : ""); + logDirsAllocator = new LocalDirAllocator( + NM_GOOD_LOG_DIRS); } @Override @@ -373,10 +398,10 @@ public class LocalDirsHandlerService extends AbstractService { Configuration conf = getConfig(); List<String> localDirs = getLocalDirs(); - conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, + conf.setStrings(NM_GOOD_LOCAL_DIRS, localDirs.toArray(new String[localDirs.size()])); List<String> logDirs = getLogDirs(); - conf.setStrings(YarnConfiguration.NM_LOG_DIRS, + conf.setStrings(NM_GOOD_LOG_DIRS, logDirs.toArray(new String[logDirs.size()])); if (!areDisksHealthy()) { // Just log. http://git-wip-us.apache.org/repos/asf/hadoop/blob/c829be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java index c61d1f0..e704c8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java @@ -120,6 +120,15 @@ public class TestLocalDirsHandlerService { Assert.assertEquals(0, nm.getGoodLocalDirsDiskUtilizationPerc()); Assert.assertEquals(0, nm.getGoodLogDirsDiskUtilizationPerc()); + Assert.assertEquals("", + dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOCAL_DIRS)); + Assert.assertEquals("", + dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOG_DIRS)); + Assert.assertEquals(localDir1 + "," + localDir2, + dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS)); + Assert.assertEquals(logDir1 + "," + logDir2, + dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS)); + conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100.0f); nm = NodeManagerMetrics.create(); @@ -141,6 +150,15 @@ public class TestLocalDirsHandlerService { Assert .assertEquals(utilizationPerc, nm.getGoodLogDirsDiskUtilizationPerc()); + Assert.assertEquals(localDir2, + dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOCAL_DIRS)); + Assert.assertEquals(logDir2, + dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOG_DIRS)); + Assert.assertEquals(localDir1 + "," + localDir2, + dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS)); + Assert.assertEquals(logDir1 + "," + logDir2, + dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS)); + FileUtils.deleteDirectory(new File(localDir1)); FileUtils.deleteDirectory(new File(localDir2)); FileUtils.deleteDirectory(new File(logDir1));