YARN-4842. Fixed "yarn logs" command to guess (and thus not require) the appOwner argument when viewing another user's logs. Contributed by Ram Venkatesh and Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87f5e351 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87f5e351 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87f5e351 Branch: refs/heads/HDFS-1312 Commit: 87f5e351337a905af5215af76c72b9312616cd4f Parents: 996a210 Author: Vinod Kumar Vavilapalli <vino...@apache.org> Authored: Mon May 9 22:33:14 2016 -0700 Committer: Vinod Kumar Vavilapalli <vino...@apache.org> Committed: Mon May 9 22:41:18 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 55 +++-- .../hadoop/yarn/client/cli/TestLogsCLI.java | 203 ++++++++++++++++--- .../yarn/logaggregation/LogCLIHelpers.java | 82 +++++++- 3 files changed, 290 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f5e351/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 487b694..926ba79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -151,14 +151,12 @@ public class LogsCLI extends Configured implements Tool { LogCLIHelpers logCliHelper = new LogCLIHelpers(); logCliHelper.setConf(getConf()); - if (appOwner == null || appOwner.isEmpty()) { - appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); - } - boolean appStateObtainedSuccessfully = true; YarnApplicationState appState = YarnApplicationState.NEW; + ApplicationReport appReport = null; try { - appState = getApplicationState(appId); + appReport = getApplicationReport(appId); + appState = appReport.getYarnApplicationState(); if (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING || appState == YarnApplicationState.SUBMITTED) { @@ -171,6 +169,16 @@ public class LogsCLI extends Configured implements Tool { + " Attempting to fetch logs directly from the filesystem."); } + if (appOwner == null || appOwner.isEmpty()) { + appOwner = guessAppOwner(appReport, appId); + if (appOwner == null) { + System.err.println("Can not find the appOwner. " + + "Please specify the correct appOwner"); + System.err.println("Could not locate application logs for " + appId); + return -1; + } + } + if (showMetaInfo) { return showMetaInfo(appState, appStateObtainedSuccessfully, logCliHelper, appId, containerIdStr, nodeAddress, appOwner); @@ -201,6 +209,10 @@ public class LogsCLI extends Configured implements Tool { if (nodeAddress == null) { resultCode = logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out); + if (resultCode == -1) { + System.err.println("Can not find the logs for the application: " + + appId + " with the appOwner: " + appOwner); + } } else { System.err.println("Should at least provide ContainerId!"); printHelpMessage(printOpts); @@ -210,13 +222,12 @@ public class LogsCLI extends Configured implements Tool { return resultCode; } - private YarnApplicationState getApplicationState(ApplicationId appId) + private ApplicationReport getApplicationReport(ApplicationId appId) throws IOException, YarnException { YarnClient yarnClient = createYarnClient(); try { - ApplicationReport appReport = yarnClient.getApplicationReport(appId); - return appReport.getYarnApplicationState(); + return yarnClient.getApplicationReport(appId); } finally { yarnClient.close(); } @@ -693,11 +704,12 @@ public class LogsCLI extends Configured implements Tool { amContainersList, logFiles, logCliHelper, appOwner, true); } else { System.err.println("Can not get AMContainers logs for " - + "the application:" + appId); - System.err.println("This application:" + appId + " is finished." - + " Please enable the application history service. Or Using " - + "yarn logs -applicationId <appId> -containerId <containerId> " - + "--nodeAddress <nodeHttpAddress> to get the container logs"); + + "the application:" + appId + " with the appOwner:" + appOwner); + System.err.println("This application:" + appId + " has finished." + + " Please enable the application-history service or explicitly" + + " use 'yarn logs -applicationId <appId> " + + "-containerId <containerId> --nodeAddress <nodeHttpAddress>' " + + "to get the container logs."); return -1; } } @@ -750,7 +762,8 @@ public class LogsCLI extends Configured implements Tool { appOwner); } else if (!isApplicationFinished(appState)) { System.err.println("Unable to get logs for this container:" - + containerIdStr + "for the application:" + appIdStr); + + containerIdStr + "for the application:" + appIdStr + + " with the appOwner: " + appOwner); System.err.println("The application: " + appIdStr + " is still running, and we can not get Container report " + "for the container: " + containerIdStr +". Please try later " @@ -821,4 +834,18 @@ public class LogsCLI extends Configured implements Tool { return isAppFinished; } } + + private String guessAppOwner(ApplicationReport appReport, + ApplicationId appId) throws IOException { + String appOwner = null; + if (appReport != null) { + //always use the app owner from the app report if possible + appOwner = appReport.getUser(); + } else { + appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); + appOwner = LogCLIHelpers.getOwnerForAppIdOrNull( + appId, appOwner, getConf()); + } + return appOwner; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f5e351/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index b950764..4f1c629 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -42,11 +42,11 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import org.junit.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -55,22 +55,21 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat; import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; public class TestLogsCLI { + ByteArrayOutputStream sysOutStream; private PrintStream sysOut; - + ByteArrayOutputStream sysErrStream; private PrintStream sysErr; @@ -79,7 +78,7 @@ public class TestLogsCLI { sysOutStream = new ByteArrayOutputStream(); sysOut = new PrintStream(sysOutStream); System.setOut(sysOut); - + sysErrStream = new ByteArrayOutputStream(); sysErr = new PrintStream(sysErrStream); System.setErr(sysErr); @@ -91,16 +90,18 @@ public class TestLogsCLI { conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class); LogCLIHelpers cliHelper = new LogCLIHelpers(); cliHelper.setConf(conf); - YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); + YarnClient mockYarnClient = createMockYarnClient( + YarnApplicationState.FINISHED, + UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI dumper = new LogsCLIForTest(mockYarnClient); dumper.setConf(conf); - + // verify dumping a non-existent application's logs returns a failure code int exitCode = dumper.run( new String[] { "-applicationId", "application_0_0" } ); assertTrue("Should return an error code", exitCode != 0); - - // verify dumping a non-existent container log is a failure code + + // verify dumping a non-existent container log is a failure code exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0", "nonexistentnode:1234", "nobody"); assertTrue("Should return an error code", exitCode != 0); @@ -109,10 +110,12 @@ public class TestLogsCLI { @Test(timeout = 5000l) public void testInvalidApplicationId() throws Exception { Configuration conf = new YarnConfiguration(); - YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); + YarnClient mockYarnClient = createMockYarnClient( + YarnApplicationState.FINISHED, + UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); - + int exitCode = cli.run( new String[] { "-applicationId", "not_an_app_id"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified")); @@ -137,7 +140,9 @@ public class TestLogsCLI { @Test(timeout = 5000l) public void testHelpMessage() throws Exception { Configuration conf = new YarnConfiguration(); - YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); + YarnClient mockYarnClient = createMockYarnClient( + YarnApplicationState.FINISHED, + UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI dumper = new LogsCLIForTest(mockYarnClient); dumper.setConf(conf); @@ -187,7 +192,7 @@ public class TestLogsCLI { String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); } - + @Test (timeout = 15000) public void testFetchApplictionLogs() throws Exception { String remoteLogRootDir = "target/logs/"; @@ -200,13 +205,13 @@ public class TestLogsCLI { FileSystem fs = FileSystem.get(configuration); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1); + ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = - ApplicationAttemptIdPBImpl.newInstance(appId, 1); - ContainerId containerId0 = ContainerIdPBImpl.newContainerId(appAttemptId, 0); - ContainerId containerId1 = ContainerIdPBImpl.newContainerId(appAttemptId, 1); - ContainerId containerId2 = ContainerIdPBImpl.newContainerId(appAttemptId, 2); - ContainerId containerId3 = ContainerIdPBImpl.newContainerId(appAttemptId, 3); + ApplicationAttemptId.newInstance(appId, 1); + ContainerId containerId0 = ContainerId.newContainerId(appAttemptId, 0); + ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1); + ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2); + ContainerId containerId3 = ContainerId.newContainerId(appAttemptId, 3); NodeId nodeId = NodeId.newInstance("localhost", 1234); // create local logs @@ -222,6 +227,7 @@ public class TestLogsCLI { fs.delete(appLogsDir, true); } assertTrue(fs.mkdirs(appLogsDir)); + List<String> rootLogDirs = Arrays.asList(rootLogDir); List<String> logTypes = new ArrayList<String>(); @@ -258,7 +264,8 @@ public class TestLogsCLI { containerId3, path, fs); YarnClient mockYarnClient = - createMockYarnClient(YarnApplicationState.FINISHED); + createMockYarnClient( + YarnApplicationState.FINISHED, ugi.getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(configuration); @@ -349,6 +356,143 @@ public class TestLogsCLI { } @Test (timeout = 15000) + public void testFetchApplictionLogsAsAnotherUser() throws Exception { + String remoteLogRootDir = "target/logs/"; + String rootLogDir = "target/LocalLogs"; + + String testUser = "test"; + UserGroupInformation testUgi = UserGroupInformation + .createRemoteUser(testUser); + + Configuration configuration = new Configuration(); + configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); + configuration + .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); + configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); + FileSystem fs = FileSystem.get(configuration); + + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + ContainerId containerId = ContainerId + .newContainerId(appAttemptId, 1); + NodeId nodeId = NodeId.newInstance("localhost", 1234); + + try { + Path rootLogDirPath = new Path(rootLogDir); + if (fs.exists(rootLogDirPath)) { + fs.delete(rootLogDirPath, true); + } + assertTrue(fs.mkdirs(rootLogDirPath)); + + // create local app dir for app + final Path appLogsDir = new Path(rootLogDirPath, appId.toString()); + if (fs.exists(appLogsDir)) { + fs.delete(appLogsDir, true); + } + assertTrue(fs.mkdirs(appLogsDir)); + + List<String> rootLogDirs = Arrays.asList(rootLogDir); + List<String> logTypes = new ArrayList<String>(); + logTypes.add("syslog"); + + // create container logs in localLogDir for app + createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes); + + // create the remote app dir for app + // but for a different user testUser" + Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId); + if (fs.exists(path)) { + fs.delete(path, true); + } + assertTrue(fs.mkdirs(path)); + + // upload container logs for app into remote dir + uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs, + nodeId, containerId, path, fs); + + YarnClient mockYarnClient = createMockYarnClient( + YarnApplicationState.FINISHED, testUgi.getShortUserName()); + LogsCLI cli = new LogsCLIForTest(mockYarnClient); + cli.setConf(configuration); + + // Verify that we can get the application logs by specifying + // a correct appOwner + int exitCode = cli.run(new String[] { + "-applicationId", appId.toString(), + "-appOwner", testUser}); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains( + "Hello " + containerId + " in syslog!")); + sysOutStream.reset(); + + // Verify that we can not get the application logs + // if an invalid user is specified + exitCode = cli.run(new String[] { + "-applicationId", appId.toString(), + "-appOwner", "invalid"}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains("Can not find the logs " + + "for the application: " + appId.toString())); + sysErrStream.reset(); + + // Verify that we do not specify appOwner, and can not + // get appReport from RM, we still can figure out the appOwner + // and can get app logs successfully. + YarnClient mockYarnClient2 = createMockYarnClientUnknownApp(); + cli = new LogsCLIForTest(mockYarnClient2); + cli.setConf(configuration); + exitCode = cli.run(new String[] { + "-applicationId", appId.toString()}); + assertTrue(exitCode == 0); + assertTrue(sysOutStream.toString().contains("Hello " + + containerId + " in syslog!")); + sysOutStream.reset(); + + // Verify that we could get the err message "Can not find the appOwner" + // if we do not specify the appOwner, can not get appReport, and + // the app does not exist in remote dir. + ApplicationId appId2 = ApplicationId.newInstance( + System.currentTimeMillis(), 2); + exitCode = cli.run(new String[] { + "-applicationId", appId2.toString()}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains( + "Can not find the appOwner")); + sysErrStream.reset(); + + // Verify that we could not get appOwner + // because we don't have file-system permissions + ApplicationId appTest = ApplicationId.newInstance( + System.currentTimeMillis(), 1000); + String priorityUser = "priority"; + Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser + + "/logs/" + appTest); + if (fs.exists(pathWithoutPerm)) { + fs.delete(pathWithoutPerm, true); + } + // The user will not have read permission for this directory. + // To mimic the scenario that the user can not get file status + FsPermission permission = FsPermission + .createImmutable((short) 01300); + assertTrue(fs.mkdirs(pathWithoutPerm, permission)); + + exitCode = cli.run(new String[] { + "-applicationId", appTest.toString()}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains( + "Guessed logs' owner is " + priorityUser + " and current user " + + UserGroupInformation.getCurrentUser().getUserName() + + " does not have permission to access")); + sysErrStream.reset(); + } finally { + fs.delete(new Path(remoteLogRootDir), true); + fs.delete(new Path(rootLogDir), true); + } + } + + @Test (timeout = 15000) public void testPrintContainerLogMetadata() throws Exception { String remoteLogRootDir = "target/logs/"; Configuration configuration = new Configuration(); @@ -380,7 +524,8 @@ public class TestLogsCLI { appId, containerIds, nodeIds); YarnClient mockYarnClient = - createMockYarnClient(YarnApplicationState.FINISHED); + createMockYarnClient(YarnApplicationState.FINISHED, + UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(configuration); @@ -466,7 +611,8 @@ public class TestLogsCLI { appId, containerIds, nodeIds); YarnClient mockYarnClient = - createMockYarnClient(YarnApplicationState.FINISHED); + createMockYarnClient(YarnApplicationState.FINISHED, + UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(configuration); @@ -508,7 +654,8 @@ public class TestLogsCLI { assertTrue(fs.exists(harPath)); YarnClient mockYarnClient = - createMockYarnClient(YarnApplicationState.FINISHED); + createMockYarnClient(YarnApplicationState.FINISHED, + ugi.getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(configuration); int exitCode = cli.run(new String[]{"-applicationId", @@ -630,10 +777,12 @@ public class TestLogsCLI { writer.close(); } - private YarnClient createMockYarnClient(YarnApplicationState appState) + private YarnClient createMockYarnClient(YarnApplicationState appState, + String user) throws YarnException, IOException { YarnClient mockClient = mock(YarnClient.class); ApplicationReport mockAppReport = mock(ApplicationReport.class); + doReturn(user).when(mockAppReport).getUser(); doReturn(appState).when(mockAppReport).getYarnApplicationState(); doReturn(mockAppReport).when(mockClient).getApplicationReport( any(ApplicationId.class)); @@ -659,9 +808,9 @@ public class TestLogsCLI { } private static class LogsCLIForTest extends LogsCLI { - + private YarnClient yarnClient; - + public LogsCLIForTest(YarnClient yarnClient) { super(); this.yarnClient = yarnClient; http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f5e351/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index e3ea1f6..ba0dd89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -23,6 +23,7 @@ import java.io.EOFException; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; +import java.nio.file.AccessDeniedException; import java.util.List; import org.apache.commons.lang.StringUtils; @@ -34,6 +35,8 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.HarFs; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; @@ -56,6 +59,53 @@ public class LogCLIHelpers implements Configurable { @Private @VisibleForTesting + /** + * Return the owner for a given AppId + * @param remoteRootLogDir + * @param appId + * @param bestGuess + * @param conf + * @return the owner or null + * @throws IOException + */ + public static String getOwnerForAppIdOrNull( + ApplicationId appId, String bestGuess, + Configuration conf) throws IOException { + Path remoteRootLogDir = new Path(conf.get( + YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); + String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); + Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, + appId, bestGuess, suffix); + FileContext fc = + FileContext.getFileContext(remoteRootLogDir.toUri(), conf); + String pathAccess = fullPath.toString(); + try { + if (fc.util().exists(fullPath)) { + return bestGuess; + } + Path toMatch = LogAggregationUtils. + getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix); + pathAccess = toMatch.toString(); + FileStatus[] matching = fc.util().globStatus(toMatch); + if (matching == null || matching.length != 1) { + return null; + } + //fetch the user from the full path /app-logs/user[/suffix]/app_id + Path parent = matching[0].getPath().getParent(); + //skip the suffix too + if (suffix != null && !StringUtils.isEmpty(suffix)) { + parent = parent.getParent(); + } + return parent.getName(); + } catch (AccessControlException | AccessDeniedException ex) { + logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage()); + return null; + } + } + + @Private + @VisibleForTesting public int dumpAContainersLogsForALogType(String appId, String containerId, String nodeId, String jobOwner, List<String> logType) throws IOException { @@ -93,12 +143,12 @@ public class LogCLIHelpers implements Configurable { thisNodeFile.getPath()); if (logType == null) { if (dumpAContainerLogs(containerId, reader, System.out, - thisNodeFile.getModificationTime()) > -1) { + thisNodeFile.getModificationTime()) > -1) { foundContainerLogs = true; } } else { if (dumpAContainerLogsForALogType(containerId, reader, System.out, - thisNodeFile.getModificationTime(), logType) > -1) { + thisNodeFile.getModificationTime(), logType) > -1) { foundContainerLogs = true; } } @@ -182,7 +232,7 @@ public class LogCLIHelpers implements Configurable { while (true) { try { LogReader.readAContainerLogsForALogType(valueStream, out, - logUploadedTime); + logUploadedTime); foundContainerLogs = true; } catch (EOFException eof) { break; @@ -249,9 +299,10 @@ public class LogCLIHelpers implements Configurable { continue; } if (!thisNodeFile.getPath().getName() - .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { + .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = - new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath()); + new AggregatedLogFormat.LogReader(getConf(), + thisNodeFile.getPath()); try { DataInputStream valueStream; @@ -261,13 +312,14 @@ public class LogCLIHelpers implements Configurable { while (valueStream != null) { String containerString = - "\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName(); + "\n\nContainer: " + key + " on " + + thisNodeFile.getPath().getName(); out.println(containerString); out.println(StringUtils.repeat("=", containerString.length())); while (true) { try { LogReader.readAContainerLogsForALogType(valueStream, out, - thisNodeFile.getModificationTime()); + thisNodeFile.getModificationTime()); foundAnyLogs = true; } catch (EOFException eof) { break; @@ -283,7 +335,7 @@ public class LogCLIHelpers implements Configurable { } } } - if (! foundAnyLogs) { + if (!foundAnyLogs) { emptyLogDir(getRemoteAppLogDir(appId, appOwner).toString()); return -1; } @@ -398,6 +450,9 @@ public class LogCLIHelpers implements Configurable { getConf()).listStatus(remoteAppLogDir); } catch (FileNotFoundException fnf) { logDirNotExist(remoteAppLogDir.toString()); + } catch (AccessControlException | AccessDeniedException ace) { + logDirNoAccessPermission(remoteAppLogDir.toString(), appOwner, + ace.getMessage()); } return nodeFiles; } @@ -426,7 +481,7 @@ public class LogCLIHelpers implements Configurable { private static void containerLogNotFound(String containerId) { System.err.println("Logs for container " + containerId - + " are not present in this log-file."); + + " are not present in this log-file."); } private static void logDirNotExist(String remoteAppLogDir) { @@ -437,4 +492,13 @@ public class LogCLIHelpers implements Configurable { private static void emptyLogDir(String remoteAppLogDir) { System.err.println(remoteAppLogDir + " does not have any log files."); } + + private static void logDirNoAccessPermission(String remoteAppLogDir, + String appOwner, String errorMessage) throws IOException { + System.err.println("Guessed logs' owner is " + appOwner + + " and current user " + + UserGroupInformation.getCurrentUser().getUserName() + " does not " + + "have permission to access " + remoteAppLogDir + + ". Error message found: " + errorMessage); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
