HDFS-8719. Erasure Coding: client generates too many small packets when writing parity data. Contributed by Li Bo.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48f3830f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48f3830f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48f3830f Branch: refs/heads/HDFS-7285-merge Commit: 48f3830f2171420d65e74d7a44a6e7905012fe0d Parents: 42a1d0d Author: Walter Su <waltersu4...@apache.org> Authored: Thu Jul 9 16:18:05 2015 +0800 Committer: Walter Su <waltersu4...@apache.org> Committed: Thu Jul 9 16:18:05 2015 +0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++ .../src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 2 +- .../main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/48f3830f/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 58b91b6..cdbef05 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -332,3 +332,6 @@ HDFS-8461. Erasure coding: fix priority level of UnderReplicatedBlocks for striped block. (Walter Su via jing9) + + HDFS-8719. Erasure Coding: client generates too many small packets when + writing parity data. (Li Bo via waltersu4549) http://git-wip-us.apache.org/repos/asf/hadoop/blob/48f3830f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index ea1ea26..9e201ad 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -459,7 +459,7 @@ public class DFSOutputStream extends FSOutputSummer * write filled up its partial chunk. Tell the summer to generate full * crc chunks from now on. */ - private void adjustChunkBoundary() { + protected void adjustChunkBoundary() { if (streamer.getAppendChunk() && streamer.getBytesCurBlock() % bytesPerChecksum == 0) { streamer.setAppendChunk(false); http://git-wip-us.apache.org/repos/asf/hadoop/blob/48f3830f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index cd7f6d4..4234351 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -318,6 +318,7 @@ public class DFSStripedOutputStream extends DFSOutputStream { private synchronized StripedDataStreamer setCurrentStreamer(int i) { streamer = streamers.get(i); + adjustChunkBoundary(); return getCurrentStreamer(); }
