YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5acf94e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5acf94e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5acf94e Branch: refs/heads/yarn-2877 Commit: f5acf94ecafb301a0cc8e8f91f19c8bcbc8da701 Parents: 4ac6799 Author: Jason Lowe <jl...@apache.org> Authored: Tue Nov 24 23:47:10 2015 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Tue Nov 24 23:47:10 2015 +0000 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 6 ++++ .../nodelabels/FileSystemNodeLabelsStore.java | 6 ++-- .../TestFileSystemNodeLabelsStore.java | 29 ++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5acf94e/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4483589..155d5c9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1100,6 +1100,9 @@ Release 2.7.3 - UNRELEASED YARN-4344. NMs reconnecting with changed capabilities can lead to wrong cluster resource calculations (Varun Vasudev via jlowe) + YARN-4365. FileSystemNodeLabelStore should check for root dir existence on + startup (Kuhu Shukla via jlowe) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES @@ -1960,6 +1963,9 @@ Release 2.6.3 - UNRELEASED YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container log files from full disks. (zhihai xu via jlowe) + YARN-4365. FileSystemNodeLabelStore should check for root dir existence on + startup (Kuhu Shukla via jlowe) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5acf94e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java index 20dc67c..c9727a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java @@ -88,7 +88,9 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { setFileSystem(conf); // mkdir of root dir path - fs.mkdirs(fsWorkingPath); + if (!fs.exists(fsWorkingPath)) { + fs.mkdirs(fsWorkingPath); + } } @Override @@ -96,7 +98,7 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { IOUtils.cleanup(LOG, fs, editlogOs); } - private void setFileSystem(Configuration conf) throws IOException { + void setFileSystem(Configuration conf) throws IOException { Configuration confCopy = new Configuration(conf); confCopy.setBoolean("dfs.client.retry.policy.enabled", true); String retryPolicy = http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5acf94e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java index 4b052c9..4929f95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java @@ -24,6 +24,8 @@ import java.util.Arrays; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.InlineDispatcher; @@ -33,6 +35,7 @@ import org.junit.Before; import org.junit.Test; import com.google.common.collect.ImmutableMap; +import org.mockito.Mockito; public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { MockNodeLabelManager mgr = null; @@ -317,4 +320,30 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { Arrays.asList("p2", "p4", "p6", "p7", "p8", "p9"))); mgr.stop(); } + + @Test + public void testRootMkdirOnInitStore() throws Exception { + final FileSystem mockFs = Mockito.mock(FileSystem.class); + FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore(mgr) { + void setFileSystem(Configuration conf) throws IOException { + fs = mockFs; + } + }; + mockStore.fs = mockFs; + verifyMkdirsCount(mockStore, true, 0); + verifyMkdirsCount(mockStore, false, 1); + verifyMkdirsCount(mockStore, true, 1); + verifyMkdirsCount(mockStore, false, 2); + } + + private void verifyMkdirsCount(FileSystemNodeLabelsStore store, + boolean existsRetVal, int expectedNumOfCalls) + throws Exception { + Mockito.when(store.fs.exists(Mockito.any( + Path.class))).thenReturn(existsRetVal); + store.init(conf); + Mockito.verify(store.fs,Mockito.times( + expectedNumOfCalls)).mkdirs(Mockito.any(Path + .class)); + } }
