Repository: hadoop Updated Branches: refs/heads/branch-2.7 8ea88ab0b -> e1716c7cf
Revert "YARN-4354. Public resource localization fails with NPE. Contributed by Jason Lowe." This reverts commit 8ea88ab0b87482b779ec9f629b46e9ac7ad53cb5. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38f37ca2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38f37ca2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38f37ca2 Branch: refs/heads/branch-2.7 Commit: 38f37ca2ebe6bb7504b841e0a10b6ec0d73c9595 Parents: 8ea88ab Author: Jason Lowe <jl...@apache.org> Authored: Mon Nov 16 14:30:22 2015 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Mon Nov 16 14:30:22 2015 +0000 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 -- .../localizer/LocalResourcesTrackerImpl.java | 10 +--- .../TestLocalResourcesTrackerImpl.java | 56 ++------------------ .../TestResourceLocalizationService.java | 4 +- 4 files changed, 6 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/38f37ca2/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index cf647c1..f961b0c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -138,9 +138,6 @@ Release 2.7.2 - UNRELEASED YARN-4127. RM fail with noAuth error if switched from failover mode to non-failover mode. (Varun Saxena via jianhe) - YARN-4354. Public resource localization fails with NPE. (Jason Lowe via - junping_du) - Release 2.7.1 - 2015-07-06 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/38f37ca2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java index 60a9529..f9346d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java @@ -171,22 +171,14 @@ class LocalResourcesTrackerImpl implements LocalResourcesTracker { break; } - if (rsrc == null) { - LOG.warn("Received " + event.getType() + " event for request " + req - + " but localized resource is missing"); - return; - } rsrc.handle(event); // Remove the resource if its downloading and its reference count has // become 0 after RELEASE. This maybe because a container was killed while // localizing and no other container is referring to the resource. - // NOTE: This should NOT be done for public resources since the - // download is not associated with a container-specific localizer. if (event.getType() == ResourceEventType.RELEASE) { if (rsrc.getState() == ResourceState.DOWNLOADING && - rsrc.getRefCount() <= 0 && - rsrc.getRequest().getVisibility() != LocalResourceVisibility.PUBLIC) { + rsrc.getRefCount() <= 0) { removeResource(req); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/38f37ca2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java index e9a789b..bd33213 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java @@ -138,12 +138,12 @@ public class TestLocalResourcesTrackerImpl { tracker.handle(rel21Event); dispatcher.await(); - verifyTrackedResourceCount(tracker, 2); + verifyTrackedResourceCount(tracker, 1); // Verify resource with non zero ref count is not removed. Assert.assertEquals(2, lr1.getRefCount()); Assert.assertFalse(tracker.remove(lr1, mockDelService)); - verifyTrackedResourceCount(tracker, 2); + verifyTrackedResourceCount(tracker, 1); // Localize resource1 ResourceLocalizedEvent rle = @@ -158,7 +158,7 @@ public class TestLocalResourcesTrackerImpl { // Verify resources in state LOCALIZED with ref-count=0 is removed. Assert.assertTrue(tracker.remove(lr1, mockDelService)); - verifyTrackedResourceCount(tracker, 1); + verifyTrackedResourceCount(tracker, 0); } finally { if (dispatcher != null) { dispatcher.stop(); @@ -829,56 +829,6 @@ public class TestLocalResourcesTrackerImpl { } } - @Test - @SuppressWarnings("unchecked") - public void testReleaseWhileDownloading() throws Exception { - String user = "testuser"; - DrainDispatcher dispatcher = null; - try { - Configuration conf = new Configuration(); - dispatcher = createDispatcher(conf); - EventHandler<LocalizerEvent> localizerEventHandler = - mock(EventHandler.class); - EventHandler<LocalizerEvent> containerEventHandler = - mock(EventHandler.class); - dispatcher.register(LocalizerEventType.class, localizerEventHandler); - dispatcher.register(ContainerEventType.class, containerEventHandler); - - ContainerId cId = BuilderUtils.newContainerId(1, 1, 1, 1); - LocalizerContext lc = new LocalizerContext(user, cId, null); - - LocalResourceRequest req = - createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); - LocalizedResource lr = createLocalizedResource(req, dispatcher); - ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = - new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(); - localrsrc.put(req, lr); - LocalResourcesTracker tracker = - new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, - false, conf, new NMNullStateStoreService(), null); - - // request the resource - ResourceEvent reqEvent = - new ResourceRequestEvent(req, LocalResourceVisibility.PUBLIC, lc); - tracker.handle(reqEvent); - - // release the resource - ResourceEvent relEvent = new ResourceReleaseEvent(req, cId); - tracker.handle(relEvent); - - // download completing after release - ResourceLocalizedEvent rle = - new ResourceLocalizedEvent(req, new Path("file:///tmp/r1"), 1); - tracker.handle(rle); - - dispatcher.await(); - } finally { - if (dispatcher != null) { - dispatcher.stop(); - } - } - } - private boolean createdummylocalizefile(Path path) { boolean ret = false; File file = new File(path.toUri().getRawPath().toString()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/38f37ca2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index cfed5cd..7cbbce7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -482,8 +482,8 @@ public class TestResourceLocalizationService { Assert.assertEquals("Incorrect reference count", 0, lr.getRefCount()); pubRsrcs.remove(lr.getRequest()); } - Assert.assertEquals(0, pubRsrcs.size()); - Assert.assertEquals(2, pubRsrcCount); + Assert.assertEquals(2, pubRsrcs.size()); + Assert.assertEquals(0, pubRsrcCount); appRsrcCount = 0; for (LocalizedResource lr : appTracker) {
