YARN-3011. Possible IllegalArgumentException in ResourceLocalizationService might lead NM to crash. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec5b3071 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec5b3071 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec5b3071 Branch: refs/heads/HDFS-EC Commit: ec5b307145b0762003f9d8a1ccb4b34fd1a658f0 Parents: 32548f4 Author: Jian He <jia...@apache.org> Authored: Tue Jan 27 13:31:22 2015 -0800 Committer: Zhe Zhang <z...@apache.org> Committed: Thu Jan 29 10:05:24 2015 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../localizer/ResourceLocalizationService.java | 10 +++++++++ .../TestResourceLocalizationService.java | 23 ++++++++++++++++++-- 3 files changed, 34 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec5b3071/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 52b3cab..054b394 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -418,6 +418,9 @@ Release 2.7.0 - UNRELEASED YARN-3088. LinuxContainerExecutor.deleteAsUser can throw NPE if native executor returns an error (Eric Payne via jlowe) + YARN-3011. Possible IllegalArgumentException in ResourceLocalizationService + might lead NM to crash. (Varun Saxena via jianhe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec5b3071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index 2f4fa5e..8c84132 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -794,6 +794,13 @@ public class ResourceLocalizationService extends CompositeService .getResource().getRequest(), e.getMessage())); LOG.error("Local path for public localization is not found. " + " May be disks failed.", e); + } catch (IllegalArgumentException ie) { + rsrc.unlock(); + publicRsrc.handle(new ResourceFailedLocalizationEvent(request + .getResource().getRequest(), ie.getMessage())); + LOG.error("Local path for public localization is not found. " + + " Incorrect path. " + request.getResource().getRequest() + .getPath(), ie); } catch (RejectedExecutionException re) { rsrc.unlock(); publicRsrc.handle(new ResourceFailedLocalizationEvent(request @@ -1015,6 +1022,9 @@ public class ResourceLocalizationService extends CompositeService } catch (IOException e) { LOG.error("local path for PRIVATE localization could not be " + "found. Disks might have failed.", e); + } catch (IllegalArgumentException e) { + LOG.error("Inorrect path for PRIVATE localization." + + next.getResource().getFile(), e); } catch (URISyntaxException e) { //TODO fail? Already translated several times... } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec5b3071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index 9ed18dd..30af5a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -1253,14 +1253,33 @@ public class TestResourceLocalizationService { user, appId); Assert.assertNull(tracker.getLocalizedResource(pubReq)); - // test RejectedExecutionException + // test IllegalArgumentException + String name = Long.toHexString(r.nextLong()); + URL url = getPath("/local/PRIVATE/" + name + "/"); + final LocalResource rsrc = + BuilderUtils.newLocalResource(url, LocalResourceType.FILE, + LocalResourceVisibility.PUBLIC, r.nextInt(1024) + 1024L, + r.nextInt(1024) + 2048L, false); + final LocalResourceRequest pubReq1 = new LocalResourceRequest(rsrc); + Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 = + new HashMap<LocalResourceVisibility, + Collection<LocalResourceRequest>>(); + req1.put(LocalResourceVisibility.PUBLIC, + Collections.singletonList(pubReq1)); Mockito .doCallRealMethod() .when(dirsHandlerSpy) .getLocalPathForWrite(isA(String.class), Mockito.anyLong(), Mockito.anyBoolean()); + // send request + spyService.handle(new ContainerLocalizationRequestEvent(c, req1)); + dispatcher.await(); + tracker = + spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, + user, appId); + Assert.assertNull(tracker.getLocalizedResource(pubReq)); - // shutdown the thread pool + // test RejectedExecutionException by shutting down the thread pool PublicLocalizer publicLocalizer = spyService.getPublicLocalizer(); publicLocalizer.threadPool.shutdown();
