HDFS-8744. Erasure Coding: the number of chunks in packet is not updated when writing parity data. Contributed by Li Bo
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/436454a8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/436454a8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/436454a8 Branch: refs/heads/HDFS-7285-REBASE Commit: 436454a8927f14bc1fcf7231eb4cb9ce01c2c395 Parents: 61b228a Author: boli2 <bo.b...@intel.com> Authored: Mon Jul 13 00:41:36 2015 -0400 Committer: Vinayakumar B <vinayakum...@apache.org> Committed: Thu Aug 13 17:19:54 2015 +0530 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++ .../src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 2 +- .../src/main/java/org/apache/hadoop/hdfs/DFSPacket.java | 6 +++--- .../java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java | 4 +++- 4 files changed, 10 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/436454a8/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 90f6732..7b6d165 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -341,3 +341,6 @@ HDFS-8484. Erasure coding: Two contiguous blocks occupy IDs belong to same striped group. (Walter Su via jing9) + + HDFS-8744. Erasure Coding: the number of chunks in packet is not updated + when writing parity data. (Li Bo) http://git-wip-us.apache.org/repos/asf/hadoop/blob/436454a8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 088f6dd..0f02049 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -418,7 +418,7 @@ public class DFSOutputStream extends FSOutputSummer currentPacket.writeChecksum(checksum, ckoff, cklen); currentPacket.writeData(b, offset, len); - currentPacket.incNumChunks(); + currentPacket.incNumChunks(1); getStreamer().incBytesCurBlock(len); // If packet is full, enqueue it for transmission http://git-wip-us.apache.org/repos/asf/hadoop/blob/436454a8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java index a26e35e..2698de3 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java @@ -259,10 +259,10 @@ public class DFSPacket { } /** - * increase the number of chunks by one + * increase the number of chunks by n */ - synchronized void incNumChunks() { - numChunks++; + synchronized void incNumChunks(int n) { + numChunks += n; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/436454a8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 5e80b46..5ad1db8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -389,11 +389,13 @@ public class DFSStripedOutputStream extends DFSOutputStream { int maxBytesToPacket = p.getMaxChunks() * bytesPerChecksum; int toWrite = byteBuffer.remaining() > maxBytesToPacket ? maxBytesToPacket: byteBuffer.remaining(); - int ckLen = ((toWrite - 1) / bytesPerChecksum + 1) * getChecksumSize(); + int chunks = (toWrite - 1) / bytesPerChecksum + 1; + int ckLen = chunks * getChecksumSize(); p.writeChecksum(checksumBuf, ckOff, ckLen); ckOff += ckLen; p.writeData(byteBuffer, toWrite); getCurrentStreamer().incBytesCurBlock(toWrite); + p.incNumChunks(chunks); packets.add(p); } return packets;
