Author: zjshen
Date: Sun Jul 27 01:43:21 2014
New Revision: 1613745
URL: http://svn.apache.org/r1613745
Log:
MAPREDUCE-6002. Made MR task avoid reporting error to AM when the task process
is shutting down. Contributed by Wangda Tan.
svn merge --ignore-ancestry -c 1613743 ../../trunk/
Modified:
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
Modified: hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/CHANGES.txt?rev=1613745&r1=1613744&r2=1613745&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/CHANGES.txt Sun
Jul 27 01:43:21 2014
@@ -147,6 +147,9 @@ Release 2.5.0 - UNRELEASED
MAPREDUCE-5952. LocalContainerLauncher#renameMapOutputForReduce
incorrectly
assumes a single dir for mapOutIndex. (Gera Shegalov via kasha)
+ MAPREDUCE-6002. Made MR task avoid reporting error to AM when the task
process
+ is shutting down. (Wangda Tan via zjshen)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java?rev=1613745&r1=1613744&r2=1613745&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
(original)
+++
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
Sun Jul 27 01:43:21 2014
@@ -31,6 +31,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import com.google.common.annotations.VisibleForTesting;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSError;
@@ -57,6 +58,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -406,7 +408,9 @@ public class LocalContainerLauncher exte
} catch (FSError e) {
LOG.fatal("FSError from child", e);
// umbilical: MRAppMaster creates (taskAttemptListener), passes to us
- umbilical.fsError(classicAttemptID, e.getMessage());
+ if (!ShutdownHookManager.get().isShutdownInProgress()) {
+ umbilical.fsError(classicAttemptID, e.getMessage());
+ }
throw new RuntimeException();
} catch (Exception exception) {
@@ -429,11 +433,13 @@ public class LocalContainerLauncher exte
} catch (Throwable throwable) {
LOG.fatal("Error running local (uberized) 'child' : "
+ StringUtils.stringifyException(throwable));
- Throwable tCause = throwable.getCause();
- String cause = (tCause == null)
- ? throwable.getMessage()
- : StringUtils.stringifyException(tCause);
- umbilical.fatalError(classicAttemptID, cause);
+ if (!ShutdownHookManager.get().isShutdownInProgress()) {
+ Throwable tCause = throwable.getCause();
+ String cause =
+ (tCause == null) ? throwable.getMessage() : StringUtils
+ .stringifyException(tCause);
+ umbilical.fatalError(classicAttemptID, cause);
+ }
throw new RuntimeException();
}
}
Modified:
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1613745&r1=1613744&r2=1613745&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
(original)
+++
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
Sun Jul 27 01:43:21 2014
@@ -56,6 +56,7 @@ import org.apache.hadoop.security.Securi
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -170,7 +171,9 @@ class YarnChild {
});
} catch (FSError e) {
LOG.fatal("FSError from child", e);
- umbilical.fsError(taskid, e.getMessage());
+ if (!ShutdownHookManager.get().isShutdownInProgress()) {
+ umbilical.fsError(taskid, e.getMessage());
+ }
} catch (Exception exception) {
LOG.warn("Exception running child : "
+ StringUtils.stringifyException(exception));
@@ -195,17 +198,22 @@ class YarnChild {
}
// Report back any failures, for diagnostic purposes
if (taskid != null) {
- umbilical.fatalError(taskid,
StringUtils.stringifyException(exception));
+ if (!ShutdownHookManager.get().isShutdownInProgress()) {
+ umbilical.fatalError(taskid,
+ StringUtils.stringifyException(exception));
+ }
}
} catch (Throwable throwable) {
LOG.fatal("Error running child : "
+ StringUtils.stringifyException(throwable));
if (taskid != null) {
- Throwable tCause = throwable.getCause();
- String cause = tCause == null
- ? throwable.getMessage()
- : StringUtils.stringifyException(tCause);
- umbilical.fatalError(taskid, cause);
+ if (!ShutdownHookManager.get().isShutdownInProgress()) {
+ Throwable tCause = throwable.getCause();
+ String cause =
+ tCause == null ? throwable.getMessage() : StringUtils
+ .stringifyException(tCause);
+ umbilical.fatalError(taskid, cause);
+ }
}
} finally {
RPC.stopProxy(umbilical);
Modified:
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java?rev=1613745&r1=1613744&r2=1613745&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
(original)
+++
hadoop/common/branches/branch-2.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
Sun Jul 27 01:43:21 2014
@@ -66,6 +66,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
@@ -321,6 +322,11 @@ abstract public class Task implements Wr
protected void reportFatalError(TaskAttemptID id, Throwable throwable,
String logMsg) {
LOG.fatal(logMsg);
+
+ if (ShutdownHookManager.get().isShutdownInProgress()) {
+ return;
+ }
+
Throwable tCause = throwable.getCause();
String cause = tCause == null
? StringUtils.stringifyException(throwable)
