Repository: hadoop Updated Branches: refs/heads/trunk 2b0fa20f6 -> 902c6ea7e
YARN-3088. LinuxContainerExecutor.deleteAsUser can throw NPE if native executor returns an error. Contributed by Eric Payne Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/902c6ea7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/902c6ea7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/902c6ea7 Branch: refs/heads/trunk Commit: 902c6ea7e4d3b49e49d9ce51ae9d12694ecfcf89 Parents: 2b0fa20 Author: Jason Lowe <jl...@apache.org> Authored: Mon Jan 26 15:40:21 2015 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Mon Jan 26 15:40:21 2015 +0000 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../nodemanager/LinuxContainerExecutor.java | 9 +++- .../TestLinuxContainerExecutorWithMocks.java | 49 +++++++++++++++++++- 3 files changed, 58 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/902c6ea7/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0808678..872f16e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -409,6 +409,9 @@ Release 2.7.0 - UNRELEASED YARN-3082. Non thread safe access to systemCredentials in NodeHeartbeatResponse processing. (Anubhav Dhoot via ozawa) + YARN-3088. LinuxContainerExecutor.deleteAsUser can throw NPE if native + executor returns an error (Eric Payne via jlowe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/902c6ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index 4606f0c..d6e6894 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -392,18 +392,23 @@ public class LinuxContainerExecutor extends ContainerExecutor { verifyUsernamePattern(user); String runAsUser = getRunAsUser(user); + String dirString = dir == null ? "" : dir.toUri().getPath(); + List<String> command = new ArrayList<String>( Arrays.asList(containerExecutorExe, runAsUser, user, Integer.toString(Commands.DELETE_AS_USER.getValue()), - dir == null ? "" : dir.toUri().getPath())); + dirString)); + List<String> pathsToDelete = new ArrayList<String>(); if (baseDirs == null || baseDirs.length == 0) { LOG.info("Deleting absolute path : " + dir); + pathsToDelete.add(dirString); } else { for (Path baseDir : baseDirs) { Path del = dir == null ? baseDir : new Path(baseDir, dir); LOG.info("Deleting path : " + del); + pathsToDelete.add(del.toString()); command.add(baseDir.toUri().getPath()); } } @@ -419,7 +424,7 @@ public class LinuxContainerExecutor extends ContainerExecutor { } } catch (IOException e) { int exitCode = shExec.getExitCode(); - LOG.error("DeleteAsUser for " + dir.toUri().getPath() + LOG.error("DeleteAsUser for " + StringUtils.join(" ", pathsToDelete) + " returned with exit code: " + exitCode, e); LOG.error("Output from LinuxContainerExecutor's deleteAsUser follows:"); logOutput(shExec.getOutput()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/902c6ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index d54367a..98ab8e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -319,10 +319,57 @@ public class TestLinuxContainerExecutorWithMocks { String cmd = String.valueOf( LinuxContainerExecutor.Commands.DELETE_AS_USER.getValue()); Path dir = new Path("/tmp/testdir"); - + Path testFile = new Path("testfile"); + Path baseDir0 = new Path("/grid/0/BaseDir"); + Path baseDir1 = new Path("/grid/1/BaseDir"); + + mockExec.deleteAsUser(appSubmitter, dir); + assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, + appSubmitter, cmd, "/tmp/testdir"), + readMockParams()); + + mockExec.deleteAsUser(appSubmitter, null); + assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, + appSubmitter, cmd, ""), + readMockParams()); + + mockExec.deleteAsUser(appSubmitter, testFile, baseDir0, baseDir1); + assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, + appSubmitter, cmd, testFile.toString(), baseDir0.toString(), baseDir1.toString()), + readMockParams()); + + mockExec.deleteAsUser(appSubmitter, null, baseDir0, baseDir1); + assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, + appSubmitter, cmd, "", baseDir0.toString(), baseDir1.toString()), + readMockParams()); + + File f = new File("./src/test/resources/mock-container-executer-with-error"); + if (!FileUtil.canExecute(f)) { + FileUtil.setExecutable(f, true); + } + String executorPath = f.getAbsolutePath(); + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath); + mockExec.setConf(conf); + mockExec.deleteAsUser(appSubmitter, dir); assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, appSubmitter, cmd, "/tmp/testdir"), readMockParams()); + + mockExec.deleteAsUser(appSubmitter, null); + assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, + appSubmitter, cmd, ""), + readMockParams()); + + mockExec.deleteAsUser(appSubmitter, testFile, baseDir0, baseDir1); + assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, + appSubmitter, cmd, testFile.toString(), baseDir0.toString(), baseDir1.toString()), + readMockParams()); + + mockExec.deleteAsUser(appSubmitter, null, baseDir0, baseDir1); + assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, + appSubmitter, cmd, "", baseDir0.toString(), baseDir1.toString()), + readMockParams()); } }
