Repository: hadoop Updated Branches: refs/heads/branch-2.6 e4360fd42 -> 2ecd17342
YARN-4005. Completed container whose app is finished is possibly not removed from NMStateStore. Contributed by Jun Gong (cherry picked from commit 38aed1a94ed7b6da62e2445b5610bc02b1cddeeb) Conflicts: hadoop-yarn-project/CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ecd1734 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ecd1734 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ecd1734 Branch: refs/heads/branch-2.6 Commit: 2ecd17342697520c1a02bf30446fa90f1384f413 Parents: e4360fd Author: Jason Lowe <jl...@apache.org> Authored: Thu Oct 8 15:16:11 2015 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Thu Oct 8 15:16:11 2015 +0000 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../nodemanager/NodeStatusUpdaterImpl.java | 8 ++--- .../nodemanager/TestNodeStatusUpdater.java | 34 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecd1734/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 2037c23..fe58854 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -30,6 +30,9 @@ Release 2.6.2 - UNRELEASED YARN-3727. For better error recovery, check if the directory exists before using it for localization. (Zhihai Xu via jlowe) + YARN-4005. Completed container whose app is finished is possibly not + removed from NMStateStore. (Jun Gong via jianhe) + Release 2.6.1 - 2015-09-23 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecd1734/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 6ddd7e4..795dcfa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -378,12 +378,12 @@ public class NodeStatusUpdaterImpl extends AbstractService implements } else { if (!isContainerRecentlyStopped(containerId)) { pendingCompletedContainers.put(containerId, containerStatus); - // Adding to finished containers cache. Cache will keep it around at - // least for #durationToTrackStoppedContainers duration. In the - // subsequent call to stop container it will get removed from cache. - addCompletedContainer(containerId); } } + // Adding to finished containers cache. Cache will keep it around at + // least for #durationToTrackStoppedContainers duration. In the + // subsequent call to stop container it will get removed from cache. + addCompletedContainer(containerId); } else { containerStatuses.add(containerStatus); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecd1734/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index 46d7b10..7d25bf3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -944,6 +944,40 @@ public class TestNodeStatusUpdater { Assert.assertTrue(containerIdSet.contains(runningContainerId)); } + @Test(timeout = 10000) + public void testCompletedContainersIsRecentlyStopped() throws Exception { + NodeManager nm = new NodeManager(); + nm.init(conf); + NodeStatusUpdaterImpl nodeStatusUpdater = + (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater(); + ApplicationId appId = ApplicationId.newInstance(0, 0); + Application completedApp = mock(Application.class); + when(completedApp.getApplicationState()).thenReturn( + ApplicationState.FINISHED); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 0); + ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); + Token containerToken = + BuilderUtils.newContainerToken(containerId, "host", 1234, "user", + BuilderUtils.newResource(1024, 1), 0, 123, + "password".getBytes(), 0); + Container completedContainer = new ContainerImpl(conf, null, + null, null, null, null, + BuilderUtils.newContainerTokenIdentifier(containerToken)) { + @Override + public ContainerState getCurrentState() { + return ContainerState.COMPLETE; + } + }; + + nm.getNMContext().getApplications().putIfAbsent(appId, completedApp); + nm.getNMContext().getContainers().put(containerId, completedContainer); + + Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size()); + Assert.assertTrue(nodeStatusUpdater.isContainerRecentlyStopped( + containerId)); + } + @Test public void testCleanedupApplicationContainerCleanup() throws IOException { NodeManager nm = new NodeManager();