Repository: hadoop Updated Branches: refs/heads/HDFS-7285 24d0fbe60 -> 631322517
HADOOP-12013 Generate fixed data to perform erasure coder test. Contributed by Kai Zheng Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/63132251 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/63132251 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/63132251 Branch: refs/heads/HDFS-7285 Commit: 6313225177d3895b2bdf96cebea5ef86f2b925b7 Parents: 24d0fbe Author: Kai Zheng <kai.zh...@intel.com> Authored: Fri May 22 23:47:52 2015 +0800 Committer: Kai Zheng <kai.zh...@intel.com> Committed: Fri May 22 23:47:52 2015 +0800 ---------------------------------------------------------------------- .../hadoop-common/CHANGES-HDFS-EC-7285.txt | 2 + .../hadoop/io/erasurecode/TestCoderBase.java | 87 +++++++++++++++++++- .../io/erasurecode/rawcoder/TestRSRawCoder.java | 8 +- .../rawcoder/TestRSRawCoderBase.java | 21 +++-- 4 files changed, 110 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/63132251/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index c799b4f..531b8d5 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -54,3 +54,5 @@ HADOOP-11938. Enhance ByteBuffer version encode/decode API of raw erasure coder. (Kai Zheng via Zhe Zhang) + + HADOOP-12013. Generate fixed data to perform erasure coder test. (Kai Zheng) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/63132251/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java index cc3617c..3686695 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java @@ -52,6 +52,12 @@ public abstract class TestCoderBase { // may go to different coding implementations. protected boolean usingDirectBuffer = true; + protected boolean usingFixedData = true; + // Using this the generated data can be repeatable across multiple calls to + // encode(), in order for troubleshooting. + private static int FIXED_DATA_GENERATOR = 0; + protected byte[][] fixedData; + protected int getChunkSize() { return chunkSize; } @@ -63,13 +69,17 @@ public abstract class TestCoderBase { /** * Prepare before running the case. + * @param conf * @param numDataUnits * @param numParityUnits * @param erasedDataIndexes + * @param erasedParityIndexes + * @param usingFixedData Using fixed or pre-generated data to test instead of + * generating data */ protected void prepare(Configuration conf, int numDataUnits, int numParityUnits, int[] erasedDataIndexes, - int[] erasedParityIndexes) { + int[] erasedParityIndexes, boolean usingFixedData) { this.conf = conf; this.numDataUnits = numDataUnits; this.numParityUnits = numParityUnits; @@ -77,6 +87,38 @@ public abstract class TestCoderBase { erasedDataIndexes : new int[] {0}; this.erasedParityIndexes = erasedParityIndexes != null ? erasedParityIndexes : new int[] {0}; + this.usingFixedData = usingFixedData; + if (usingFixedData) { + prepareFixedData(); + } + } + + /** + * Prepare before running the case. + * @param conf + * @param numDataUnits + * @param numParityUnits + * @param erasedDataIndexes + * @param erasedParityIndexes + */ + protected void prepare(Configuration conf, int numDataUnits, + int numParityUnits, int[] erasedDataIndexes, + int[] erasedParityIndexes) { + prepare(conf, numDataUnits, numParityUnits, erasedDataIndexes, + erasedParityIndexes, false); + } + + /** + * Prepare before running the case. + * @param numDataUnits + * @param numParityUnits + * @param erasedDataIndexes + * @param erasedParityIndexes + */ + protected void prepare(int numDataUnits, int numParityUnits, + int[] erasedDataIndexes, int[] erasedParityIndexes) { + prepare(null, numDataUnits, numParityUnits, erasedDataIndexes, + erasedParityIndexes, false); } /** @@ -278,6 +320,29 @@ public abstract class TestCoderBase { * @return */ protected ECChunk[] prepareDataChunksForEncoding() { + if (usingFixedData) { + ECChunk[] chunks = new ECChunk[numDataUnits]; + for (int i = 0; i < chunks.length; i++) { + chunks[i] = makeChunkUsingData(fixedData[i]); + } + return chunks; + } + + return generateDataChunks(); + } + + private ECChunk makeChunkUsingData(byte[] data) { + ECChunk chunk = allocateOutputChunk(); + ByteBuffer buffer = chunk.getBuffer(); + int pos = buffer.position(); + buffer.put(data, 0, chunkSize); + buffer.flip(); + buffer.position(pos); + + return chunk; + } + + private ECChunk[] generateDataChunks() { ECChunk[] chunks = new ECChunk[numDataUnits]; for (int i = 0; i < chunks.length; i++) { chunks[i] = generateDataChunk(); @@ -286,6 +351,15 @@ public abstract class TestCoderBase { return chunks; } + private void prepareFixedData() { + // We may load test data from a resource, or just generate randomly. + // The generated data will be used across subsequent encode/decode calls. + this.fixedData = new byte[numDataUnits][]; + for (int i = 0; i < numDataUnits; i++) { + fixedData[i] = generateFixedData(baseChunkSize * 2); + } + } + /** * Generate data chunk by making random data. * @return @@ -319,6 +393,17 @@ public abstract class TestCoderBase { return buffer; } + protected byte[] generateFixedData(int len) { + byte[] buffer = new byte[len]; + for (int i = 0; i < buffer.length; i++) { + buffer[i] = (byte) FIXED_DATA_GENERATOR++; + if (FIXED_DATA_GENERATOR == 256) { + FIXED_DATA_GENERATOR = 0; + } + } + return buffer; + } + /** * Prepare parity chunks for encoding, each chunk for each parity unit. * @return http://git-wip-us.apache.org/repos/asf/hadoop/blob/63132251/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java index 02b9eea..80ec04d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java @@ -80,7 +80,13 @@ public class TestRSRawCoder extends TestRSRawCoderBase { @Test public void testCodingDirectBuffer_10x4_erasure_of_d2_d4_p0() { - prepare(null, 10, 4, new int[] {2, 4}, new int[] {0}); + prepare(null, 10, 4, new int[]{2, 4}, new int[]{0}); + testCoding(true); + } + + @Test + public void testCodingDirectBuffer_usingFixedData_10x4_erasure_of_d2_d4_p0() { + prepare(null, 10, 4, new int[] {2, 4}, new int[] {0}, true); testCoding(true); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/63132251/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java index c06aded..efde332 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java @@ -17,12 +17,7 @@ */ package org.apache.hadoop.io.erasurecode.rawcoder; -import org.apache.hadoop.io.erasurecode.ECChunk; import org.apache.hadoop.io.erasurecode.rawcoder.util.RSUtil; -import org.junit.Before; -import org.junit.Test; - -import java.nio.ByteBuffer; /** * Test base for raw Reed-solomon coders. @@ -32,6 +27,8 @@ public abstract class TestRSRawCoderBase extends TestRawCoderBase { private static int symbolSize = 0; private static int symbolMax = 0; + private static int RS_FIXED_DATA_GENERATOR = 0; + static { symbolSize = (int) Math.round(Math.log( RSUtil.GF.getFieldSize()) / Math.log(2)); @@ -41,9 +38,21 @@ public abstract class TestRSRawCoderBase extends TestRawCoderBase { @Override protected byte[] generateData(int len) { byte[] buffer = new byte[len]; - for (int i = 0; i < len; i++) { + for (int i = 0; i < buffer.length; i++) { buffer[i] = (byte) RAND.nextInt(symbolMax); } return buffer; } + + @Override + protected byte[] generateFixedData(int len) { + byte[] buffer = new byte[len]; + for (int i = 0; i < buffer.length; i++) { + buffer[i] = (byte) RS_FIXED_DATA_GENERATOR++; + if (RS_FIXED_DATA_GENERATOR == symbolMax) { + RS_FIXED_DATA_GENERATOR = 0; + } + } + return buffer; + } }
