Repository: hadoop Updated Branches: refs/heads/HDFS-7285 850d7fac9 -> 3140bc0f0
HADOOP-11921. Enhance tests for erasure coders. Contributed by Kai Zheng Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3140bc0f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3140bc0f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3140bc0f Branch: refs/heads/HDFS-7285 Commit: 3140bc0f00cc12633ab5ff31613942c1ce9c90d2 Parents: 850d7fa Author: Kai Zheng <kai.zh...@intel.com> Authored: Thu May 7 06:07:51 2015 +0800 Committer: Kai Zheng <kai.zh...@intel.com> Committed: Thu May 7 06:07:51 2015 +0800 ---------------------------------------------------------------------- .../hadoop-common/CHANGES-HDFS-EC-7285.txt | 2 + .../hadoop/fs/CommonConfigurationKeys.java | 4 -- .../apache/hadoop/io/erasurecode/ECChunk.java | 20 ++++++-- .../erasurecode/coder/AbstractErasureCoder.java | 6 +-- .../io/erasurecode/coder/RSErasureDecoder.java | 40 +-------------- .../rawcoder/AbstractRawErasureCoder.java | 35 +++++++++++++- .../rawcoder/AbstractRawErasureDecoder.java | 51 ++++++++++++-------- .../rawcoder/AbstractRawErasureEncoder.java | 48 +++++++++--------- .../erasurecode/rawcoder/RawErasureCoder.java | 8 +-- .../erasurecode/rawcoder/RawErasureDecoder.java | 24 +++++---- .../io/erasurecode/rawcoder/XORRawDecoder.java | 24 +++------ .../io/erasurecode/rawcoder/XORRawEncoder.java | 6 ++- .../hadoop/io/erasurecode/TestCoderBase.java | 4 +- .../erasurecode/coder/TestRSErasureCoder.java | 4 +- 14 files changed, 146 insertions(+), 130 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index 9749270..bfbdcb9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -44,3 +44,5 @@ HADOOP-11818. Minor improvements for erasurecode classes. (Rakesh R via Kai Zheng) HADOOP-11841. Remove unused ecschema-def.xml files. (szetszwo) + + HADOOP-11920 Enhance tests for erasure coders. (Kai Zheng) http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index bd2a24b..3f2871b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -143,10 +143,6 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { /** Supported erasure codec classes */ public static final String IO_ERASURECODE_CODECS_KEY = "io.erasurecode.codecs"; - /** Use XOR raw coder when possible for the RS codec */ - public static final String IO_ERASURECODE_CODEC_RS_USEXOR_KEY = - "io.erasurecode.codec.rs.usexor"; - /** Raw coder factory for the RS codec */ public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY = "io.erasurecode.codec.rs.rawcoder"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java index 01e8f35..34dd90b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java @@ -58,8 +58,15 @@ public class ECChunk { public static ByteBuffer[] toBuffers(ECChunk[] chunks) { ByteBuffer[] buffers = new ByteBuffer[chunks.length]; + ECChunk chunk; for (int i = 0; i < chunks.length; i++) { - buffers[i] = chunks[i].getBuffer(); + chunk = chunks[i]; + if (chunk == null) { + buffers[i] = null; + continue; + } + + buffers[i] = chunk.getBuffer(); } return buffers; @@ -71,12 +78,19 @@ public class ECChunk { * @param chunks * @return an array of byte array */ - public static byte[][] toArray(ECChunk[] chunks) { + public static byte[][] toArrays(ECChunk[] chunks) { byte[][] bytesArr = new byte[chunks.length][]; ByteBuffer buffer; + ECChunk chunk; for (int i = 0; i < chunks.length; i++) { - buffer = chunks[i].getBuffer(); + chunk = chunks[i]; + if (chunk == null) { + bytesArr[i] = null; + continue; + } + + buffer = chunk.getBuffer(); if (buffer.hasArray()) { bytesArr[i] = buffer.array(); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java index 7403e35..d491570 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java @@ -90,11 +90,7 @@ public abstract class AbstractErasureCoder throw new RuntimeException("Failed to create raw coder", e); } - if (fact != null) { - return isEncoder ? fact.createEncoder() : fact.createDecoder(); - } - - return null; + return isEncoder ? fact.createEncoder() : fact.createDecoder(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java index fc664a5..ec7cbb5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java @@ -17,13 +17,11 @@ */ package org.apache.hadoop.io.erasurecode.coder; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECBlockGroup; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; -import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder; /** * Reed-Solomon erasure decoder that decodes a block group. @@ -32,38 +30,14 @@ import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder; */ public class RSErasureDecoder extends AbstractErasureDecoder { private RawErasureDecoder rsRawDecoder; - private RawErasureDecoder xorRawDecoder; - private boolean useXorWhenPossible = true; @Override - public void setConf(Configuration conf) { - super.setConf(conf); - - if (conf != null) { - this.useXorWhenPossible = conf.getBoolean( - CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_USEXOR_KEY, true); - } - } - - @Override protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) { - RawErasureDecoder rawDecoder; - ECBlock[] inputBlocks = getInputBlocks(blockGroup); ECBlock[] outputBlocks = getOutputBlocks(blockGroup); - /** - * Optimization: according to some benchmark, when only one block is erased - * and to be recovering, the most simple XOR scheme can be much efficient. - * We will have benchmark tests to verify this opt is effect or not. - */ - if (outputBlocks.length == 1 && useXorWhenPossible) { - rawDecoder = checkCreateXorRawDecoder(); - } else { - rawDecoder = checkCreateRSRawDecoder(); - } - + RawErasureDecoder rawDecoder = checkCreateRSRawDecoder(); return new ErasureDecodingStep(inputBlocks, getErasedIndexes(inputBlocks), outputBlocks, rawDecoder); } @@ -81,19 +55,9 @@ public class RSErasureDecoder extends AbstractErasureDecoder { return rsRawDecoder; } - private RawErasureDecoder checkCreateXorRawDecoder() { - if (xorRawDecoder == null) { - xorRawDecoder = new XORRawDecoder(); - xorRawDecoder.initialize(getNumDataUnits(), 1, getChunkSize()); - } - return xorRawDecoder; - } - @Override public void release() { - if (xorRawDecoder != null) { - xorRawDecoder.release(); - } else if (rsRawDecoder != null) { + if (rsRawDecoder != null) { rsRawDecoder.release(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java index e6f3d92..eed9035 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java @@ -19,6 +19,9 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.conf.Configured; +import java.nio.ByteBuffer; +import java.util.Arrays; + /** * A common class of basic facilities to be shared by encoder and decoder * @@ -27,6 +30,9 @@ import org.apache.hadoop.conf.Configured; public abstract class AbstractRawErasureCoder extends Configured implements RawErasureCoder { + // Hope to reset coding buffers a little faster using it + protected byte[] zeroChunkBytes; + private int numDataUnits; private int numParityUnits; private int chunkSize; @@ -37,6 +43,8 @@ public abstract class AbstractRawErasureCoder this.numDataUnits = numDataUnits; this.numParityUnits = numParityUnits; this.chunkSize = chunkSize; + + zeroChunkBytes = new byte[chunkSize]; // With ZERO by default } @Override @@ -55,7 +63,7 @@ public abstract class AbstractRawErasureCoder } @Override - public boolean preferNativeBuffer() { + public boolean preferDirectBuffer() { return false; } @@ -63,4 +71,29 @@ public abstract class AbstractRawErasureCoder public void release() { // Nothing to do by default } + /** + * Ensure the buffer (either input or output) ready to read or write with ZERO + * bytes fully in chunkSize. + * @param buffer + * @return the buffer itself + */ + protected ByteBuffer resetDirectBuffer(ByteBuffer buffer) { + buffer.clear(); + buffer.put(zeroChunkBytes); + buffer.position(0); + + return buffer; + } + + /** + * Ensure the buffer (either input or output) ready to read or write with ZERO + * bytes fully in chunkSize. + * @param buffer bytes array buffer + * @return the buffer itself + */ + protected byte[] resetArrayBuffer(byte[] buffer) { + System.arraycopy(zeroChunkBytes, 0, buffer, 0, buffer.length); + + return buffer; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java index 4613b25..17f867a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java @@ -32,9 +32,7 @@ public abstract class AbstractRawErasureDecoder extends AbstractRawErasureCoder @Override public void decode(ByteBuffer[] inputs, int[] erasedIndexes, ByteBuffer[] outputs) { - if (erasedIndexes.length == 0) { - return; - } + checkParameters(inputs, erasedIndexes, outputs); doDecode(inputs, erasedIndexes, outputs); } @@ -50,9 +48,7 @@ public abstract class AbstractRawErasureDecoder extends AbstractRawErasureCoder @Override public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs) { - if (erasedIndexes.length == 0) { - return; - } + checkParameters(inputs, erasedIndexes, outputs); doDecode(inputs, erasedIndexes, outputs); } @@ -69,20 +65,12 @@ public abstract class AbstractRawErasureDecoder extends AbstractRawErasureCoder @Override public void decode(ECChunk[] inputs, int[] erasedIndexes, ECChunk[] outputs) { - doDecode(inputs, erasedIndexes, outputs); - } + checkParameters(inputs, erasedIndexes, outputs); - /** - * Perform the real decoding using chunks - * @param inputs - * @param erasedIndexes - * @param outputs - */ - protected void doDecode(ECChunk[] inputs, int[] erasedIndexes, - ECChunk[] outputs) { - if (inputs[0].getBuffer().hasArray()) { - byte[][] inputBytesArr = ECChunk.toArray(inputs); - byte[][] outputBytesArr = ECChunk.toArray(outputs); + boolean hasArray = inputs[0].getBuffer().hasArray(); + if (hasArray) { + byte[][] inputBytesArr = ECChunk.toArrays(inputs); + byte[][] outputBytesArr = ECChunk.toArrays(outputs); doDecode(inputBytesArr, erasedIndexes, outputBytesArr); } else { ByteBuffer[] inputBuffers = ECChunk.toBuffers(inputs); @@ -90,4 +78,29 @@ public abstract class AbstractRawErasureDecoder extends AbstractRawErasureCoder doDecode(inputBuffers, erasedIndexes, outputBuffers); } } + /** + * Check and validate decoding parameters, throw exception accordingly. The + * checking assumes it's a MDS code. Other code can override this. + * @param inputs + * @param erasedIndexes + * @param outputs + */ + protected void checkParameters(Object[] inputs, int[] erasedIndexes, + Object[] outputs) { + if (inputs.length != getNumParityUnits() + getNumDataUnits()) { + throw new IllegalArgumentException("Invalid inputs length"); + } + + if (erasedIndexes.length != outputs.length) { + throw new IllegalArgumentException( + "erasedIndexes and outputs mismatch in length"); + } + + if (erasedIndexes.length > getNumParityUnits()) { + throw new IllegalArgumentException( + "Too many erased, not recoverable"); + } + + + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java index 4feaf39..51695e8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java @@ -31,8 +31,7 @@ public abstract class AbstractRawErasureEncoder extends AbstractRawErasureCoder @Override public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs) { - assert (inputs.length == getNumDataUnits()); - assert (outputs.length == getNumParityUnits()); + checkParameters(inputs, outputs); doEncode(inputs, outputs); } @@ -46,8 +45,7 @@ public abstract class AbstractRawErasureEncoder extends AbstractRawErasureCoder @Override public void encode(byte[][] inputs, byte[][] outputs) { - assert (inputs.length == getNumDataUnits()); - assert (outputs.length == getNumParityUnits()); + checkParameters(inputs, outputs); doEncode(inputs, outputs); } @@ -61,33 +59,37 @@ public abstract class AbstractRawErasureEncoder extends AbstractRawErasureCoder @Override public void encode(ECChunk[] inputs, ECChunk[] outputs) { - assert (inputs.length == getNumDataUnits()); - assert (outputs.length == getNumParityUnits()); + checkParameters(inputs, outputs); - doEncode(inputs, outputs); - } - - /** - * Perform the real encoding work using chunks. - * @param inputs - * @param outputs - */ - protected void doEncode(ECChunk[] inputs, ECChunk[] outputs) { /** - * Note callers may pass byte array, or ByteBuffer via ECChunk according + * Note callers may pass byte array, or direct buffer via ECChunk according * to how ECChunk is created. Some implementations of coder use byte array - * (ex: pure Java), some use native ByteBuffer (ex: ISA-L), all for the - * better performance. + * (ex: pure Java), some use direct buffer (ex: ISA-L), all for the better + * performance. */ - if (inputs[0].getBuffer().hasArray()) { - byte[][] inputBytesArr = ECChunk.toArray(inputs); - byte[][] outputBytesArr = ECChunk.toArray(outputs); - doEncode(inputBytesArr, outputBytesArr); + boolean hasArray = inputs[0].getBuffer().hasArray(); + if (hasArray) { + byte[][] inputBytesArr = ECChunk.toArrays(inputs); + byte[][] outputBytesArr = ECChunk.toArrays(outputs); + encode(inputBytesArr, outputBytesArr); } else { ByteBuffer[] inputBuffers = ECChunk.toBuffers(inputs); ByteBuffer[] outputBuffers = ECChunk.toBuffers(outputs); - doEncode(inputBuffers, outputBuffers); + encode(inputBuffers, outputBuffers); } } + /** + * Check and validate decoding parameters, throw exception accordingly. + * @param inputs + * @param outputs + */ + protected void checkParameters(Object[] inputs, Object[] outputs) { + if (inputs.length != getNumDataUnits()) { + throw new IllegalArgumentException("Invalid inputs length"); + } + if (outputs.length != getNumParityUnits()) { + throw new IllegalArgumentException("Invalid outputs length"); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java index 9af5b6c..3fb211f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java @@ -64,13 +64,13 @@ public interface RawErasureCoder extends Configurable { public int getChunkSize(); /** - * Tell if native or off-heap buffer is preferred or not. It's for callers to - * decide how to allocate coding chunk buffers, either on heap or off heap. - * It will return false by default. + * Tell if direct buffer is preferred or not. It's for callers to + * decide how to allocate coding chunk buffers, using DirectByteBuffer or + * bytes array. It will return false by default. * @return true if native buffer is preferred for performance consideration, * otherwise false. */ - public boolean preferNativeBuffer(); + public boolean preferDirectBuffer(); /** * Should be called when release this coder. Good chance to release encoding http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java index 1358b7d..1807da7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java @@ -31,24 +31,30 @@ import java.nio.ByteBuffer; public interface RawErasureDecoder extends RawErasureCoder { /** - * Decode with inputs and erasedIndexes, generates outputs - * @param inputs - * @param outputs + * Decode with inputs and erasedIndexes, generates outputs. + * @param inputs inputs to read data from + * @param erasedIndexes indexes of erased units in the inputs array + * @param outputs outputs to write into for data generated according to + * erasedIndexes */ public void decode(ByteBuffer[] inputs, int[] erasedIndexes, ByteBuffer[] outputs); /** - * Decode with inputs and erasedIndexes, generates outputs - * @param inputs - * @param outputs + * Decode with inputs and erasedIndexes, generates outputs. + * @param inputs inputs to read data from + * @param erasedIndexes indexes of erased units in the inputs array + * @param outputs outputs to write into for data generated according to + * erasedIndexes */ public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs); /** - * Decode with inputs and erasedIndexes, generates outputs - * @param inputs - * @param outputs + * Decode with inputs and erasedIndexes, generates outputs. + * @param inputs inputs to read data from + * @param erasedIndexes indexes of erased units in the inputs array + * @param outputs outputs to write into for data generated according to + * erasedIndexes */ public void decode(ECChunk[] inputs, int[] erasedIndexes, ECChunk[] outputs); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java index b6b1633..7c11681 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java @@ -27,17 +27,11 @@ public class XORRawDecoder extends AbstractRawErasureDecoder { @Override protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes, ByteBuffer[] outputs) { - assert(erasedIndexes.length == outputs.length); - assert(erasedIndexes.length <= 1); + resetDirectBuffer(outputs[0]); - int bufSize = inputs[0].remaining(); + int bufSize = getChunkSize(); int erasedIdx = erasedIndexes[0]; - // Set the output to zeros. - for (int j = 0; j < bufSize; j++) { - outputs[0].put(j, (byte) 0); - } - // Process the inputs. for (int i = 0; i < inputs.length; i++) { // Skip the erased location. @@ -52,19 +46,13 @@ public class XORRawDecoder extends AbstractRawErasureDecoder { } @Override - protected void doDecode(byte[][] inputs, int[] erasedIndexes, - byte[][] outputs) { - assert(erasedIndexes.length == outputs.length); - assert(erasedIndexes.length <= 1); + protected void doDecode(byte[][] inputs, + int[] erasedIndexes, byte[][] outputs) { + resetArrayBuffer(outputs[0]); - int bufSize = inputs[0].length; + int bufSize = getChunkSize(); int erasedIdx = erasedIndexes[0]; - // Set the output to zeros. - for (int j = 0; j < bufSize; j++) { - outputs[0][j] = 0; - } - // Process the inputs. for (int i = 0; i < inputs.length; i++) { // Skip the erased location. http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java index dbfab5d..c751a73 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java @@ -26,8 +26,9 @@ public class XORRawEncoder extends AbstractRawErasureEncoder { @Override protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) { - int bufSize = inputs[0].remaining(); + resetDirectBuffer(outputs[0]); + int bufSize = getChunkSize(); // Get the first buffer's data. for (int j = 0; j < bufSize; j++) { outputs[0].put(j, inputs[0].get(j)); @@ -43,8 +44,9 @@ public class XORRawEncoder extends AbstractRawErasureEncoder { @Override protected void doEncode(byte[][] inputs, byte[][] outputs) { - int bufSize = inputs[0].length; + resetArrayBuffer(outputs[0]); + int bufSize = getChunkSize(); // Get the first buffer's data. for (int j = 0; j < bufSize; j++) { outputs[0][j] = inputs[0][j]; http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java index 22fd98d..9c88665 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java @@ -75,8 +75,8 @@ public abstract class TestCoderBase { */ protected void compareAndVerify(ECChunk[] erasedChunks, ECChunk[] recoveredChunks) { - byte[][] erased = ECChunk.toArray(erasedChunks); - byte[][] recovered = ECChunk.toArray(recoveredChunks); + byte[][] erased = ECChunk.toArrays(erasedChunks); + byte[][] recovered = ECChunk.toArrays(recoveredChunks); boolean result = Arrays.deepEquals(erased, recovered); assertTrue("Decoding and comparing failed.", result); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3140bc0f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java index 3507dd2..6b3e742 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java @@ -59,9 +59,9 @@ public class TestRSErasureCoder extends TestErasureCoderBase { Configuration conf = new Configuration(); conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, RSRawErasureCoderFactory.class.getCanonicalName()); - conf.setBoolean( - CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_USEXOR_KEY, false); + prepare(conf, 10, 4, null); + testCoding(true); }
