Author: jeagles
Date: Tue Mar 19 16:01:04 2013
New Revision: 1458368
URL: http://svn.apache.org/r1458368
Log:
MAPREDUCE-5053. java.lang.InternalError from decompression codec cause reducer
to fail (Robert Parker via jeagles)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1458368&r1=1458367&r2=1458368&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue
Mar 19 16:01:04 2013
@@ -100,6 +100,9 @@ Release 0.23.7 - UNRELEASED
MAPREDUCE-5042. Reducer unable to fetch for a map task that was recovered
(Jason Lowe via bobby)
+ MAPREDUCE-5053. java.lang.InternalError from decompression codec cause
+ reducer to fail (Robert Parker via jeagles)
+
Release 0.23.6 - 2013-02-06
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java?rev=1458368&r1=1458367&r2=1458368&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
Tue Mar 19 16:01:04 2013
@@ -347,18 +347,23 @@ class Fetcher<K,V> extends Thread {
return EMPTY_ATTEMPT_ID_ARRAY;
}
- // Go!
- LOG.info("fetcher#" + id + " about to shuffle output of map " +
- mapOutput.getMapId() + " decomp: " +
- decompressedLength + " len: " + compressedLength + " to " +
- mapOutput.getType());
- if (mapOutput.getType() == Type.MEMORY) {
- shuffleToMemory(host, mapOutput, input,
- (int) decompressedLength, (int) compressedLength);
- } else {
- shuffleToDisk(host, mapOutput, input, compressedLength);
+ // lz*, snappy, etc. throw java.lang.InternalError when there is a
decompression
+ // error, catching and throwing IOException to trigger fetch failure
logic
+ try {
+ // Go!
+ LOG.info("fetcher#" + id + " about to shuffle output of map "
+ + mapOutput.getMapId() + " decomp: " + decompressedLength
+ + " len: " + compressedLength + " to " + mapOutput.getType());
+ if (mapOutput.getType() == Type.MEMORY) {
+ shuffleToMemory(host, mapOutput, input, (int) decompressedLength,
+ (int) compressedLength);
+ } else {
+ shuffleToDisk(host, mapOutput, input, compressedLength);
+ }
+ } catch (java.lang.InternalError e) {
+ LOG.warn("Failed to shuffle for fetcher#"+id, e);
+ throw new IOException(e);
}
-
// Inform the shuffle scheduler
long endTime = System.currentTimeMillis();
scheduler.copySucceeded(mapId, host, compressedLength,
