Repository: hadoop Updated Branches: refs/heads/branch-2.7 e768f8b77 -> 7f43eb954
HDFS-9812. Streamer threads leak if failure happens when closing DFSOutputStream. Contributed by Lin Yiqun. (cherry picked from commit 352d299cf8ebe330d24117df98d1e6a64ae38c26) (cherry picked from commit fe0009a2bd17e26f9e9364ec3f772e9a3f138de8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7f43eb95 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7f43eb95 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7f43eb95 Branch: refs/heads/branch-2.7 Commit: 7f43eb9547fb0470b77df26eeea411dc2ab81644 Parents: e768f8b Author: Akira Ajisaka <aajis...@apache.org> Authored: Tue Mar 8 10:43:17 2016 +0900 Committer: Akira Ajisaka <aajis...@apache.org> Committed: Tue Mar 8 11:15:27 2016 +0900 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f43eb95/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9878c90..7842890 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -138,6 +138,9 @@ Release 2.7.3 - UNRELEASED HDFS-9865. TestBlockReplacement fails intermittently in trunk (Lin Yiqun via iwasakims) + HDFS-9812. Streamer threads leak if failure happens when closing + DFSOutputStream. (Lin Yiqun via aajisaka) + Release 2.7.2 - 2016-01-25 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f43eb95/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index b8de7c4..ebeaacb 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -2262,7 +2262,6 @@ public class DFSOutputStream extends FSOutputSummer flushInternal(); // flush all data to Datanodes // get last block before destroying the streamer ExtendedBlock lastBlock = streamer.getBlock(); - closeThreads(false); TraceScope scope = Trace.startSpan("completeFile", Sampler.NEVER); try { completeFile(lastBlock); @@ -2271,7 +2270,12 @@ public class DFSOutputStream extends FSOutputSummer } } catch (ClosedChannelException e) { } finally { - setClosed(); + // Failures may happen when flushing data. + // Streamers may keep waiting for the new block information. + // Thus need to force closing these threads. + // Don't need to call setClosed() because closeThreads(true) + // calls setClosed() in the finally block. + closeThreads(true); } }
