Author: sseth
Date: Thu Feb 2 01:43:49 2012
New Revision: 1239433
URL: http://svn.apache.org/viewvc?rev=1239433&view=rev
Log:
MAPREDUCE-3744. Fix the yarn logs command line. Improve error messages for
mapred job -logs. (Contributed by Jason Lowe)
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1239433&r1=1239432&r2=1239433&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Thu Feb 2
01:43:49 2012
@@ -654,6 +654,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3752. Modified application limits to include queue max-capacities
besides the usual user limits. (Arun C Murthy via vinodkv)
+ MAPREDUCE-3744. Fix the yarn logs command line. Improve error messages for
+ mapred job -logs. (Jason Lowe via sseth)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java?rev=1239433&r1=1239432&r2=1239433&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
Thu Feb 2 01:43:49 2012
@@ -345,7 +345,7 @@ public class CLI extends Configured impl
LogParams logParams = cluster.getLogParams(jobID, taskAttemptID);
LogDumper logDumper = new LogDumper();
logDumper.setConf(getConf());
- logDumper.dumpAContainersLogs(logParams.getApplicationId(),
+ exitCode = logDumper.dumpAContainersLogs(logParams.getApplicationId(),
logParams.getContainerId(), logParams.getNodeId(),
logParams.getOwner());
} catch (IOException e) {
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java?rev=1239433&r1=1239432&r2=1239433&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogDumper.java
Thu Feb 2 01:43:49 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.logaggreg
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
+import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.commons.cli.CommandLine;
@@ -103,14 +104,15 @@ public class LogDumper extends Configure
if (appOwner == null || appOwner.isEmpty()) {
appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
}
+ int resultCode = 0;
if (containerIdStr == null && nodeAddress == null) {
- dumpAllContainersLogs(appId, appOwner, out);
+ resultCode = dumpAllContainersLogs(appId, appOwner, out);
} else if ((containerIdStr == null && nodeAddress != null)
|| (containerIdStr != null && nodeAddress == null)) {
System.out.println("ContainerId or NodeAddress cannot be null!");
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("general options are: ", opts);
- return -1;
+ resultCode = -1;
} else {
Path remoteRootLogDir =
new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
@@ -122,27 +124,33 @@ public class LogDumper extends Configure
appId,
appOwner,
ConverterUtils.toNodeId(nodeAddress),
- getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX,
-
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX)));
- return dumpAContainerLogs(containerIdStr, reader, out);
+ LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf())));
+ resultCode = dumpAContainerLogs(containerIdStr, reader, out);
}
- return 0;
+ return resultCode;
}
- public void dumpAContainersLogs(String appId, String containerId,
+ public int dumpAContainersLogs(String appId, String containerId,
String nodeId, String jobOwner) throws IOException {
Path remoteRootLogDir =
new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf());
- AggregatedLogFormat.LogReader reader =
- new AggregatedLogFormat.LogReader(getConf(),
- LogAggregationUtils.getRemoteNodeLogFileForApp(remoteRootLogDir,
- ConverterUtils.toApplicationId(appId), jobOwner,
- ConverterUtils.toNodeId(nodeId), suffix));
+ Path logPath = LogAggregationUtils.getRemoteNodeLogFileForApp(
+ remoteRootLogDir, ConverterUtils.toApplicationId(appId), jobOwner,
+ ConverterUtils.toNodeId(nodeId), suffix);
+ AggregatedLogFormat.LogReader reader;
+ try {
+ reader = new AggregatedLogFormat.LogReader(getConf(), logPath);
+ } catch (FileNotFoundException fnfe) {
+ System.out.println("Logs not available at " + logPath.toString());
+ System.out.println(
+ "Log aggregation has not completed or is not enabled.");
+ return -1;
+ }
DataOutputStream out = new DataOutputStream(System.out);
- dumpAContainerLogs(containerId, reader, out);
+ return dumpAContainerLogs(containerId, reader, out);
}
private int dumpAContainerLogs(String containerIdStr,
@@ -174,21 +182,28 @@ public class LogDumper extends Configure
return 0;
}
- private void dumpAllContainersLogs(ApplicationId appId, String appOwner,
+ private int dumpAllContainersLogs(ApplicationId appId, String appOwner,
DataOutputStream out) throws IOException {
Path remoteRootLogDir =
new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
String user = appOwner;
String logDirSuffix =
- getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
- YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX);
+ LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf());
//TODO Change this to get a list of files from the LAS.
Path remoteAppLogDir =
LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, appId, user,
logDirSuffix);
- RemoteIterator<FileStatus> nodeFiles =
- FileContext.getFileContext().listStatus(remoteAppLogDir);
+ RemoteIterator<FileStatus> nodeFiles;
+ try {
+ nodeFiles = FileContext.getFileContext().listStatus(remoteAppLogDir);
+ } catch (FileNotFoundException fnf) {
+ System.out.println("Logs not available at "
+ + remoteAppLogDir.toString());
+ System.out.println(
+ "Log aggregation has not completed or is not enabled.");
+ return -1;
+ }
while (nodeFiles.hasNext()) {
FileStatus thisNodeFile = nodeFiles.next();
AggregatedLogFormat.LogReader reader =
@@ -217,12 +232,14 @@ public class LogDumper extends Configure
reader.close();
}
}
+ return 0;
}
public static void main(String[] args) throws Exception {
Configuration conf = new YarnConfiguration();
LogDumper logDumper = new LogDumper();
logDumper.setConf(conf);
- logDumper.run(args);
+ int exitCode = logDumper.run(args);
+ System.exit(exitCode);
}
}
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java?rev=1239433&view=auto
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java
(added)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java
Thu Feb 2 01:43:49 2012
@@ -0,0 +1,47 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.logaggregation;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Test;
+
+public class TestLogDumper {
+ @Test
+ public void testFailResultCodes() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
+ LogDumper dumper = new LogDumper();
+ dumper.setConf(conf);
+
+ // verify dumping a non-existent application's logs returns a failure code
+ int exitCode = dumper.run( new String[] {
+ "-applicationId", "application_0_0" } );
+ assertTrue("Should return an error code", exitCode != 0);
+
+ // verify dumping a non-existent container log is a failure code
+ exitCode = dumper.dumpAContainersLogs("application_0_0", "container_0_0",
+ "nonexistentnode:1234", "nobody");
+ assertTrue("Should return an error code", exitCode != 0);
+ }
+}
