Repository: hadoop Updated Branches: refs/heads/trunk 656c8f952 -> 5eca6dece
HADOOP-12327. Initialize output buffers with ZERO bytes in erasure coder. Contributed by Kai Zheng. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5eca6dec Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5eca6dec Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5eca6dec Branch: refs/heads/trunk Commit: 5eca6dece67620f990f3306b6caaf09f317b38f6 Parents: 656c8f9 Author: Walter Su <waltersu4...@apache.org> Authored: Thu Oct 29 15:02:06 2015 +0800 Committer: Walter Su <waltersu4...@apache.org> Committed: Thu Oct 29 15:04:34 2015 +0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++ .../rawcoder/AbstractRawErasureCoder.java | 47 +++++++++++++++----- .../rawcoder/AbstractRawErasureDecoder.java | 8 ++-- .../rawcoder/AbstractRawErasureEncoder.java | 8 ++-- .../io/erasurecode/rawcoder/RSRawDecoder.java | 4 +- .../io/erasurecode/rawcoder/XORRawDecoder.java | 1 - .../io/erasurecode/rawcoder/XORRawEncoder.java | 1 - .../hadoop/hdfs/DFSStripedOutputStream.java | 3 -- .../erasurecode/ErasureCodingWorker.java | 7 +-- 9 files changed, 49 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 49f3d94..6b33a2c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -603,6 +603,9 @@ Trunk (Unreleased) HADOOP-11921. Enhance tests for erasure coders. (Kai Zheng) + HADOOP-12327. Initialize output buffers with ZERO bytes in erasure coder. + (Kai Zheng via waltersu4549) + Release 2.8.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java index 35e9492..d8a57eb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java @@ -32,6 +32,7 @@ import java.nio.ByteBuffer; public abstract class AbstractRawErasureCoder extends Configured implements RawErasureCoder { + private static byte[] emptyChunk = new byte[4096]; private final int numDataUnits; private final int numParityUnits; private final int numAllUnits; @@ -42,6 +43,23 @@ public abstract class AbstractRawErasureCoder this.numAllUnits = numDataUnits + numParityUnits; } + /** + * Make sure to return an empty chunk buffer for the desired length. + * @param leastLength + * @return empty chunk of zero bytes + */ + protected static byte[] getEmptyChunk(int leastLength) { + if (emptyChunk.length >= leastLength) { + return emptyChunk; // In most time + } + + synchronized (AbstractRawErasureCoder.class) { + emptyChunk = new byte[leastLength]; + } + + return emptyChunk; + } + @Override public int getNumDataUnits() { return numDataUnits; @@ -73,11 +91,9 @@ public abstract class AbstractRawErasureCoder * @return the buffer itself, with ZERO bytes written, the position and limit * are not changed after the call */ - protected ByteBuffer resetBuffer(ByteBuffer buffer) { + protected ByteBuffer resetBuffer(ByteBuffer buffer, int len) { int pos = buffer.position(); - for (int i = pos; i < buffer.limit(); ++i) { - buffer.put((byte) 0); - } + buffer.put(getEmptyChunk(len), 0, len); buffer.position(pos); return buffer; @@ -90,9 +106,8 @@ public abstract class AbstractRawErasureCoder * @return the buffer itself */ protected byte[] resetBuffer(byte[] buffer, int offset, int len) { - for (int i = offset; i < len; ++i) { - buffer[i] = (byte) 0; - } + byte[] empty = getEmptyChunk(len); + System.arraycopy(empty, 0, buffer, offset, len); return buffer; } @@ -104,9 +119,10 @@ public abstract class AbstractRawErasureCoder * @param allowNull whether to allow any element to be null or not * @param dataLen the length of data available in the buffer to ensure with * @param isDirectBuffer is direct buffer or not to ensure with + * @param isOutputs is output buffer or not */ - protected void ensureLengthAndType(ByteBuffer[] buffers, boolean allowNull, - int dataLen, boolean isDirectBuffer) { + protected void checkParameterBuffers(ByteBuffer[] buffers, boolean + allowNull, int dataLen, boolean isDirectBuffer, boolean isOutputs) { for (ByteBuffer buffer : buffers) { if (buffer == null && !allowNull) { throw new HadoopIllegalArgumentException( @@ -120,18 +136,23 @@ public abstract class AbstractRawErasureCoder throw new HadoopIllegalArgumentException( "Invalid buffer, isDirect should be " + isDirectBuffer); } + if (isOutputs) { + resetBuffer(buffer, dataLen); + } } } } /** - * Check and ensure the buffers are of the length specified by dataLen. + * Check and ensure the buffers are of the length specified by dataLen. If is + * output buffers, ensure they will be ZEROed. * @param buffers the buffers to check * @param allowNull whether to allow any element to be null or not * @param dataLen the length of data available in the buffer to ensure with + * @param isOutputs is output buffer or not */ - protected void ensureLength(byte[][] buffers, - boolean allowNull, int dataLen) { + protected void checkParameterBuffers(byte[][] buffers, boolean allowNull, + int dataLen, boolean isOutputs) { for (byte[] buffer : buffers) { if (buffer == null && !allowNull) { throw new HadoopIllegalArgumentException( @@ -139,6 +160,8 @@ public abstract class AbstractRawErasureCoder } else if (buffer != null && buffer.length != dataLen) { throw new HadoopIllegalArgumentException( "Invalid buffer not of length " + dataLen); + } else if (isOutputs) { + resetBuffer(buffer, 0, dataLen); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java index a99730d..2cfb57c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java @@ -48,8 +48,8 @@ public abstract class AbstractRawErasureDecoder extends AbstractRawErasureCoder if (dataLen == 0) { return; } - ensureLengthAndType(inputs, true, dataLen, usingDirectBuffer); - ensureLengthAndType(outputs, false, dataLen, usingDirectBuffer); + checkParameterBuffers(inputs, true, dataLen, usingDirectBuffer, false); + checkParameterBuffers(outputs, false, dataLen, usingDirectBuffer, true); if (usingDirectBuffer) { doDecode(inputs, erasedIndexes, outputs); @@ -106,8 +106,8 @@ public abstract class AbstractRawErasureDecoder extends AbstractRawErasureCoder if (dataLen == 0) { return; } - ensureLength(inputs, true, dataLen); - ensureLength(outputs, false, dataLen); + checkParameterBuffers(inputs, true, dataLen, false); + checkParameterBuffers(outputs, false, dataLen, true); int[] inputOffsets = new int[inputs.length]; // ALL ZERO int[] outputOffsets = new int[outputs.length]; // ALL ZERO http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java index 99c754e..13c895c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java @@ -45,8 +45,8 @@ public abstract class AbstractRawErasureEncoder extends AbstractRawErasureCoder if (dataLen == 0) { return; } - ensureLengthAndType(inputs, false, dataLen, usingDirectBuffer); - ensureLengthAndType(outputs, false, dataLen, usingDirectBuffer); + checkParameterBuffers(inputs, false, dataLen, usingDirectBuffer, false); + checkParameterBuffers(outputs, false, dataLen, usingDirectBuffer, true); if (usingDirectBuffer) { doEncode(inputs, outputs); @@ -93,8 +93,8 @@ public abstract class AbstractRawErasureEncoder extends AbstractRawErasureCoder if (dataLen == 0) { return; } - ensureLength(inputs, false, dataLen); - ensureLength(outputs, false, dataLen); + checkParameterBuffers(inputs, false, dataLen, false); + checkParameterBuffers(outputs, false, dataLen, true); int[] inputOffsets = new int[inputs.length]; // ALL ZERO int[] outputOffsets = new int[outputs.length]; // ALL ZERO http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java index 1acaab9..87347c0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java @@ -206,7 +206,7 @@ public class RSRawDecoder extends AbstractRawErasureDecoder { if (erasedIndexes[i] == erasedOrNotToReadIndexes[j]) { found = true; adjustedDirectBufferOutputsParameter[j] = - resetBuffer(outputs[outputIdx++]); + resetBuffer(outputs[outputIdx++], dataLen); } } if (!found) { @@ -220,7 +220,7 @@ public class RSRawDecoder extends AbstractRawErasureDecoder { ByteBuffer buffer = checkGetDirectBuffer(bufferIdx, dataLen); buffer.position(0); buffer.limit(dataLen); - adjustedDirectBufferOutputsParameter[i] = resetBuffer(buffer); + adjustedDirectBufferOutputsParameter[i] = resetBuffer(buffer, dataLen); bufferIdx++; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java index f11dd9f..61017dd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java @@ -39,7 +39,6 @@ public class XORRawDecoder extends AbstractRawErasureDecoder { protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes, ByteBuffer[] outputs) { ByteBuffer output = outputs[0]; - resetBuffer(output); int erasedIdx = erasedIndexes[0]; http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java index bc1ae90..646fc17 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java @@ -37,7 +37,6 @@ public class XORRawEncoder extends AbstractRawErasureEncoder { protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) { ByteBuffer output = outputs[0]; - resetBuffer(output); // Get the first buffer's data. int iIdx, oIdx; http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index c595026..f5bae2a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -221,9 +221,6 @@ public class DFSStripedOutputStream extends DFSOutputStream { private void clear() { for (int i = 0; i< numAllBlocks; i++) { buffers[i].clear(); - if (i >= numDataBlocks) { - Arrays.fill(buffers[i].array(), (byte) 0); - } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java index 64afcd0..834bd07 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java @@ -907,15 +907,10 @@ public final class ErasureCodingWorker { for (int i = 0; i < targetBuffers.length; i++) { if (targetBuffers[i] != null) { - cleanBuffer(targetBuffers[i]); + targetBuffers[i].clear(); } } } - - private ByteBuffer cleanBuffer(ByteBuffer buffer) { - Arrays.fill(buffer.array(), (byte) 0); - return (ByteBuffer)buffer.clear(); - } // send an empty packet to mark the end of the block private void endTargetBlocks(boolean[] targetsStatus) {
