Repository: hadoop Updated Branches: refs/heads/trunk 588baab16 -> 07ecdb877
HDFS-9261. Erasure Coding: Skip encoding the data cells if all the parity data streamers are failed for the current block group. (Rakesh R via umamahesh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07ecdb87 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07ecdb87 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07ecdb87 Branch: refs/heads/trunk Commit: 07ecdb877dee0aa76a3269e37ac4783a55ab6bf6 Parents: 588baab Author: Uma Mahesh <umamah...@apache.org> Authored: Wed Oct 28 23:33:11 2015 -0700 Committer: Uma Mahesh <umamah...@apache.org> Committed: Wed Oct 28 23:33:11 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hdfs/DFSStripedOutputStream.java | 18 ++++++++++++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ 2 files changed, 21 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/07ecdb87/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 884fa42..c595026 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -844,6 +844,11 @@ public class DFSStripedOutputStream extends DFSOutputStream { void writeParityCells() throws IOException { final ByteBuffer[] buffers = cellBuffers.getBuffers(); + // Skips encoding and writing parity cells if there are no healthy parity + // data streamers + if (!checkAnyParityStreamerIsHealthy()) { + return; + } //encode the data cells encode(encoder, numDataBlocks, buffers); for (int i = numDataBlocks; i < numAllBlocks; i++) { @@ -852,6 +857,19 @@ public class DFSStripedOutputStream extends DFSOutputStream { cellBuffers.clear(); } + private boolean checkAnyParityStreamerIsHealthy() { + for (int i = numDataBlocks; i < numAllBlocks; i++) { + if (streamers.get(i).isHealthy()) { + return true; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Skips encoding and writing parity cells as there are " + + "no healthy parity data streamers: " + streamers); + } + return false; + } + void writeParity(int index, ByteBuffer buffer, byte[] checksumBuf) throws IOException { final StripedDataStreamer current = setCurrentStreamer(index); http://git-wip-us.apache.org/repos/asf/hadoop/blob/07ecdb87/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index aa26088..a87fe76 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -184,6 +184,9 @@ Trunk (Unreleased) HDFS-9070. Allow fsck display pending replica location information for being-written blocks. (GAO Rui via jing9) + HDFS-9261. Erasure Coding: Skip encoding the data cells if all the parity data + streamers are failed for the current block group. (Rakesh R via umamahesh) + OPTIMIZATIONS BUG FIXES
