This is an automated email from the ASF dual-hosted git repository.
eyang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 49e1292 YARN-9524. Fixed TestAHSWebService and TestLogsCLI unit
tests. Contributed by Prabhu Joseph
49e1292 is described below
commit 49e1292ea3e4d00ab0b0191bd8c4ea4d2afed671
Author: Eric Yang <ey...@apache.org>
AuthorDate: Mon May 6 19:48:45 2019 -0400
YARN-9524. Fixed TestAHSWebService and TestLogsCLI unit tests.
Contributed by Prabhu Joseph
---
.../apache/hadoop/yarn/client/cli/TestLogsCLI.java | 27 +++++++++---
.../yarn/logaggregation/LogAggregationUtils.java | 48 ++++++++++++++++------
.../hadoop/yarn/logaggregation/LogCLIHelpers.java | 34 ++++++++++++++-
.../LogAggregationFileController.java | 2 +-
.../ifile/LogAggregationIndexedFileController.java | 2 +-
5 files changed, 92 insertions(+), 21 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index f9061eb..801cf40 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -407,7 +407,7 @@ public class TestLogsCLI {
Path path =
new Path(remoteLogRootDir + ugi.getShortUserName()
- + "/logs/application_0_0001");
+ + "/bucket_logs/0001/application_0_0001");
if (fs.exists(path)) {
fs.delete(path, true);
}
@@ -925,7 +925,6 @@ public class TestLogsCLI {
public void testFetchApplictionLogsAsAnotherUser() throws Exception {
String remoteLogRootDir = "target/logs/";
String rootLogDir = "target/LocalLogs";
-
String testUser = "test";
UserGroupInformation testUgi = UserGroupInformation
.createRemoteUser(testUser);
@@ -966,9 +965,9 @@ public class TestLogsCLI {
// create container logs in localLogDir for app
createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes);
- // create the remote app dir for app
- // but for a different user testUser"
- Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
+ // create the remote app dir for app but for a different user testUser
+ Path path = new Path(remoteLogRootDir + testUser + "/bucket_logs/0001/"
+ + appId);
if (fs.exists(path)) {
fs.delete(path, true);
}
@@ -1016,6 +1015,22 @@ public class TestLogsCLI {
logMessage(containerId, "syslog")));
sysOutStream.reset();
+ // Verify appOwner guessed correctly with older log dir dtructure
+ path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
+ if (fs.exists(path)) {
+ fs.delete(path, true);
+ }
+ assertTrue(fs.mkdirs(path));
+ uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs,
+ nodeId, containerId, path, fs);
+
+ exitCode = cli.run(new String[] {
+ "-applicationId", appId.toString()});
+ assertTrue(exitCode == 0);
+ assertTrue(sysOutStream.toString().contains(
+ logMessage(containerId, "syslog")));
+ sysOutStream.reset();
+
// Verify that we could get the err message "Can not find the appOwner"
// if we do not specify the appOwner, can not get appReport, and
// the app does not exist in remote dir.
@@ -1034,7 +1049,7 @@ public class TestLogsCLI {
System.currentTimeMillis(), 1000);
String priorityUser = "priority";
Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser
- + "/logs/" + appTest);
+ + "/bucket_logs/1000/" + appTest);
if (fs.exists(pathWithoutPerm)) {
fs.delete(pathWithoutPerm, true);
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
index 3f5151b..deff2aa 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
@@ -83,6 +83,30 @@ public class LogAggregationUtils {
suffix), appId.toString());
}
+ public static Path getOlderRemoteAppLogDir(Configuration conf,
+ ApplicationId appId, String user, Path remoteRootLogDir, String suffix)
+ throws IOException {
+ org.apache.hadoop.fs.Path remoteAppDir = null;
+ if (user == null) {
+ org.apache.hadoop.fs.Path qualifiedRemoteRootLogDir =
+ FileContext.getFileContext(conf).makeQualified(remoteRootLogDir);
+ FileContext fc = FileContext.getFileContext(
+ qualifiedRemoteRootLogDir.toUri(), conf);
+ org.apache.hadoop.fs.Path toMatch = LogAggregationUtils
+ .getOlderRemoteAppLogDir(appId, "*", remoteRootLogDir, suffix);
+ FileStatus[] matching = fc.util().globStatus(toMatch);
+ if (matching == null || matching.length != 1) {
+ throw new IOException("Can not find remote application directory for "
+ + "the application:" + appId);
+ }
+ remoteAppDir = matching[0].getPath();
+ } else {
+ remoteAppDir = LogAggregationUtils.getOlderRemoteAppLogDir(
+ appId, user, remoteRootLogDir, suffix);
+ }
+ return remoteAppDir;
+ }
+
/**
* Gets the remote suffixed log dir for the user.
* @param remoteRootLogDir the aggregated log remote root log dir
@@ -274,9 +298,9 @@ public class LogAggregationUtils {
StringBuilder diagnosticsMsg = new StringBuilder();
// Get Node Files from new app log dir
- Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner,
- remoteRootLogDir, suffix);
try {
+ Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner,
+ remoteRootLogDir, suffix);
nodeFilesCur = getNodeFiles(conf, remoteAppLogDir, appId, appOwner);
} catch (IOException ex) {
diagnosticsMsg.append(ex.getMessage() + "\n");
@@ -284,9 +308,9 @@ public class LogAggregationUtils {
// Get Node Files from old app log dir
if (isOlderPathEnabled(conf)) {
- remoteAppLogDir = getOlderRemoteAppLogDir(appId, appOwner,
- remoteRootLogDir, suffix);
try {
+ Path remoteAppLogDir = getOlderRemoteAppLogDir(conf, appId, appOwner,
+ remoteRootLogDir, suffix);
nodeFilesPrev = getNodeFiles(conf,
remoteAppLogDir, appId, appOwner);
} catch (IOException ex) {
@@ -338,14 +362,14 @@ public class LogAggregationUtils {
org.apache.hadoop.fs.Path remoteRootLogDir, String suffix)
throws IOException {
StringBuilder diagnosticsMsg = new StringBuilder();
- Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner,
- remoteRootLogDir, suffix);
List<FileStatus> nodeFiles = new ArrayList<>();
- Path qualifiedLogDir =
- FileContext.getFileContext(conf).makeQualified(remoteAppLogDir);
// Get Node Files from new app log dir
try {
+ Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner,
+ remoteRootLogDir, suffix);
+ Path qualifiedLogDir =
+ FileContext.getFileContext(conf).makeQualified(remoteAppLogDir);
nodeFiles.addAll(Arrays.asList(FileContext.getFileContext(
qualifiedLogDir.toUri(), conf).util().listStatus(remoteAppLogDir)));
} catch (IOException ex) {
@@ -354,11 +378,11 @@ public class LogAggregationUtils {
// Get Node Files from old app log dir
if (isOlderPathEnabled(conf)) {
- remoteAppLogDir = getOlderRemoteAppLogDir(appId, appOwner,
- remoteRootLogDir, suffix);
- qualifiedLogDir = FileContext.getFileContext(conf).
- makeQualified(remoteAppLogDir);
try {
+ Path remoteAppLogDir = getOlderRemoteAppLogDir(conf, appId, appOwner,
+ remoteRootLogDir, suffix);
+ Path qualifiedLogDir = FileContext.getFileContext(conf).
+ makeQualified(remoteAppLogDir);
nodeFiles.addAll(Arrays.asList(FileContext.getFileContext(
qualifiedLogDir.toUri(),
conf).util().listStatus(remoteAppLogDir)));
} catch (IOException ex) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index 9dae7b9..8a72d80 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -88,6 +88,7 @@ public class LogCLIHelpers implements Configurable {
String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf);
Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir,
appId, bestGuess, suffix);
+
FileContext fc =
FileContext.getFileContext(remoteRootLogDir.toUri(), conf);
String pathAccess = fullPath.toString();
@@ -95,19 +96,50 @@ public class LogCLIHelpers implements Configurable {
if (fc.util().exists(fullPath)) {
return bestGuess;
}
+
+ boolean scanOldPath = LogAggregationUtils.isOlderPathEnabled(conf);
+ if (scanOldPath) {
+ Path olderAppPath = LogAggregationUtils.getOlderRemoteAppLogDir(appId,
+ bestGuess, remoteRootLogDir, suffix);
+ if (fc.util().exists(olderAppPath)) {
+ return bestGuess;
+ }
+ }
+
Path toMatch = LogAggregationUtils.
getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix);
+
pathAccess = toMatch.toString();
FileStatus[] matching = fc.util().globStatus(toMatch);
if (matching == null || matching.length != 1) {
+ if (scanOldPath) {
+ toMatch = LogAggregationUtils.getOlderRemoteAppLogDir(appId, "*",
+ remoteRootLogDir, suffix);
+ try {
+ matching = fc.util().globStatus(toMatch);
+ if (matching != null && matching.length == 1) {
+ //fetch the user from the old path /app-logs/user[/suffix]/app_id
+ Path parent = matching[0].getPath().getParent();
+ //skip the suffix too
+ if (suffix != null && !StringUtils.isEmpty(suffix)) {
+ parent = parent.getParent();
+ }
+ return parent.getName();
+ }
+ } catch (IOException e) {
+ // Ignore IOException from accessing older app log dir
+ }
+ }
return null;
}
- //fetch the user from the full path /app-logs/user[/suffix]/app_id
+ //fetch the user from the full path /app-logs/user[/suffix]/bucket/app_id
Path parent = matching[0].getPath().getParent();
//skip the suffix too
if (suffix != null && !StringUtils.isEmpty(suffix)) {
parent = parent.getParent();
}
+ //skip the bucket
+ parent = parent.getParent();
return parent.getName();
} catch (AccessControlException | AccessDeniedException ex) {
logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage());
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index 053a563..2547d95 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -470,7 +470,7 @@ public abstract class LogAggregationFileController {
*/
public Path getOlderRemoteAppLogDir(ApplicationId appId, String appOwner)
throws IOException {
- return LogAggregationUtils.getOlderRemoteAppLogDir(appId, appOwner,
+ return LogAggregationUtils.getOlderRemoteAppLogDir(conf, appId, appOwner,
this.remoteRootLogDir, this.remoteRootLogDirSuffix);
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
index 9bb4f9d..96e0637 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
@@ -822,7 +822,7 @@ public class LogAggregationIndexedFileController
@Override
public Path getOlderRemoteAppLogDir(ApplicationId appId, String user)
throws IOException {
- return LogAggregationUtils.getOlderRemoteAppLogDir(appId, user,
+ return LogAggregationUtils.getOlderRemoteAppLogDir(conf, appId, user,
this.remoteRootLogDir, this.remoteRootLogDirSuffix);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
