Repository: hadoop Updated Branches: refs/heads/branch-2.8 8504e20b2 -> 4a70bd86d
YARN-5001. Aggregated Logs root directory is created with wrong group if nonexistent. Contributed by Haibo Chen (cherry picked from commit 76893a41003d57d94eb1a5f486010815266046af) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a70bd86 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a70bd86 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a70bd86 Branch: refs/heads/branch-2.8 Commit: 4a70bd86da359ce62fa0a4ff3ef86669f41ec5c8 Parents: 8504e20 Author: Jason Lowe <jl...@apache.org> Authored: Tue Nov 1 20:23:24 2016 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Tue Nov 1 20:24:44 2016 +0000 ---------------------------------------------------------------------- .../logaggregation/LogAggregationService.java | 15 ++++++++++++ .../TestLogAggregationService.java | 24 +++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a70bd86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index 48bf96b..de89a96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -218,6 +218,21 @@ public class LogAggregationService extends AbstractService implements remoteFS.getWorkingDirectory()); remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS)); remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); + + UserGroupInformation loginUser = UserGroupInformation.getLoginUser(); + String primaryGroupName = null; + try { + primaryGroupName = loginUser.getPrimaryGroupName(); + } catch (IOException e) { + LOG.warn("No primary group found. The remote root log directory" + + " will be created with the HDFS superuser being its group " + + "owner. JobHistoryServer may be unable to read the directory."); + } + // set owner on the remote directory only if the primary group exists + if (primaryGroupName != null) { + remoteFS.setOwner(qualified, + loginUser.getShortUserName(), primaryGroupName); + } } catch (IOException e) { throw new YarnRuntimeException("Failed to create remoteLogDir [" + this.remoteRootLogDir + "]", e); http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a70bd86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index c0a3f3a..0667d19 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -605,7 +605,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest { throws Exception { this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); File aNewFile = new File(String.valueOf("tmp"+System.currentTimeMillis())); - this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, aNewFile.getAbsolutePath()); LogAggregationService logAggregationService = spy( @@ -623,6 +623,28 @@ public class TestLogAggregationService extends BaseContainerManagerTest { } @Test + public void testRemoteRootLogDirIsCreatedWithCorrectGroupOwner() + throws IOException { + this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); + Path aNewFile = new Path(String.valueOf("tmp"+System.currentTimeMillis())); + this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, aNewFile.getName()); + + LogAggregationService logAggregationService = new LogAggregationService( + dispatcher, this.context, this.delSrvc, super.dirsHandler); + logAggregationService.init(this.conf); + logAggregationService.verifyAndCreateRemoteLogDir(this.conf); + + String targetGroup = + UserGroupInformation.getLoginUser().getPrimaryGroupName(); + FileSystem fs = FileSystem.get(this.conf); + FileStatus fileStatus = fs.getFileStatus(aNewFile); + Assert.assertEquals("The new aggregate file is not successfully created", + fileStatus.getGroup(), targetGroup); + + fs.delete(aNewFile, true); + } + + @Test public void testAppLogDirCreation() throws Exception { final String logSuffix = "logs"; this.conf.set(YarnConfiguration.NM_LOG_DIRS, --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
