hadoop git commit: HADOOP-11847 Enhance raw coder allowing to read least required inputs in decoding. Contributed by Kai Zheng
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 f56e19286 - 48513749a
HADOOP-11847 Enhance raw coder allowing to read least required inputs in
decoding. Contributed by Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48513749
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48513749
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48513749
Branch: refs/heads/HDFS-7285
Commit: 48513749a6ee20e7e2b29f78257e6ed05f38859e
Parents: f56e192
Author: Kai Zheng kai.zh...@intel.com
Authored: Tue May 26 22:45:19 2015 +0800
Committer: Kai Zheng kai.zh...@intel.com
Committed: Tue May 26 22:45:19 2015 +0800
--
.../hadoop-common/CHANGES-HDFS-EC-7285.txt | 3 +
.../rawcoder/AbstractRawErasureCoder.java | 27 +++-
.../rawcoder/AbstractRawErasureDecoder.java | 75 +++--
.../rawcoder/AbstractRawErasureEncoder.java | 8 +-
.../io/erasurecode/rawcoder/RSRawDecoder.java | 162 ++-
.../erasurecode/rawcoder/RawErasureDecoder.java | 20 ++-
.../io/erasurecode/rawcoder/XORRawDecoder.java | 2 +-
.../io/erasurecode/rawcoder/XORRawEncoder.java | 2 +-
.../erasurecode/rawcoder/util/GaloisField.java | 12 +-
.../hadoop/io/erasurecode/TestCoderBase.java| 39 ++---
.../erasurecode/coder/TestErasureCoderBase.java | 1 -
.../io/erasurecode/rawcoder/TestRSRawCoder.java | 101 ++--
.../erasurecode/rawcoder/TestRawCoderBase.java | 54 +++
.../erasurecode/rawcoder/TestXORRawCoder.java | 45 ++
14 files changed, 397 insertions(+), 154 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48513749/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
index c9b80d3..0c24473 100644
--- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
@@ -59,3 +59,6 @@
HADOOP-12029. Remove chunkSize from ECSchema as its not required for coders
(vinayakumarb)
+
+HADOOP-11847. Enhance raw coder allowing to read least required inputs in
decoding.
+(Kai Zheng)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48513749/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
index 06ae660..e6a1542 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
@@ -60,12 +60,13 @@ public abstract class AbstractRawErasureCoder
}
/**
- * Ensure output buffer filled with ZERO bytes fully in chunkSize.
- * @param buffer a buffer ready to write chunk size bytes
+ * Ensure a buffer filled with ZERO bytes from current readable/writable
+ * position.
+ * @param buffer a buffer ready to read / write certain size bytes
* @return the buffer itself, with ZERO bytes written, the position and limit
* are not changed after the call
*/
- protected ByteBuffer resetOutputBuffer(ByteBuffer buffer) {
+ protected ByteBuffer resetBuffer(ByteBuffer buffer) {
int pos = buffer.position();
for (int i = pos; i buffer.limit(); ++i) {
buffer.put((byte) 0);
@@ -77,7 +78,7 @@ public abstract class AbstractRawErasureCoder
/**
* Ensure the buffer (either input or output) ready to read or write with
ZERO
- * bytes fully in chunkSize.
+ * bytes fully in specified length of len.
* @param buffer bytes array buffer
* @return the buffer itself
*/
@@ -92,11 +93,16 @@ public abstract class AbstractRawErasureCoder
/**
* Check and ensure the buffers are of the length specified by dataLen.
* @param buffers
+ * @param allowNull
* @param dataLen
*/
- protected void ensureLength(ByteBuffer[] buffers, int dataLen) {
+ protected void ensureLength(ByteBuffer[] buffers,
+ boolean allowNull, int dataLen) {
for (int i = 0; i buffers.length; ++i) {
- if (buffers[i].remaining() != dataLen) {
+ if (buffers[i] == null !allowNull) {
+throw new HadoopIllegalArgumentException(
+Invalid
[49/50] [abbrv] hadoop git commit: HADOOP-11847 Enhance raw coder allowing to read least required inputs in decoding. Contributed by Kai Zheng
HADOOP-11847 Enhance raw coder allowing to read least required inputs in
decoding. Contributed by Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ad48488
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ad48488
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ad48488
Branch: refs/heads/HDFS-7285
Commit: 4ad484883f773c702a1874fc12816ef1a4a54136
Parents: 5a391e1
Author: Kai Zheng kai.zh...@intel.com
Authored: Tue May 26 22:45:19 2015 +0800
Committer: Zhe Zhang zhezh...@cloudera.com
Committed: Tue May 26 12:07:16 2015 -0700
--
.../hadoop-common/CHANGES-HDFS-EC-7285.txt | 3 +
.../rawcoder/AbstractRawErasureCoder.java | 27 +++-
.../rawcoder/AbstractRawErasureDecoder.java | 75 +++--
.../rawcoder/AbstractRawErasureEncoder.java | 8 +-
.../io/erasurecode/rawcoder/RSRawDecoder.java | 162 ++-
.../erasurecode/rawcoder/RawErasureDecoder.java | 20 ++-
.../io/erasurecode/rawcoder/XORRawDecoder.java | 2 +-
.../io/erasurecode/rawcoder/XORRawEncoder.java | 2 +-
.../erasurecode/rawcoder/util/GaloisField.java | 12 +-
.../hadoop/io/erasurecode/TestCoderBase.java| 39 ++---
.../erasurecode/coder/TestErasureCoderBase.java | 1 -
.../io/erasurecode/rawcoder/TestRSRawCoder.java | 101 ++--
.../erasurecode/rawcoder/TestRawCoderBase.java | 54 +++
.../erasurecode/rawcoder/TestXORRawCoder.java | 45 ++
14 files changed, 397 insertions(+), 154 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ad48488/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
index c9b80d3..0c24473 100644
--- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
@@ -59,3 +59,6 @@
HADOOP-12029. Remove chunkSize from ECSchema as its not required for coders
(vinayakumarb)
+
+HADOOP-11847. Enhance raw coder allowing to read least required inputs in
decoding.
+(Kai Zheng)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ad48488/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
index 06ae660..e6a1542 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
@@ -60,12 +60,13 @@ public abstract class AbstractRawErasureCoder
}
/**
- * Ensure output buffer filled with ZERO bytes fully in chunkSize.
- * @param buffer a buffer ready to write chunk size bytes
+ * Ensure a buffer filled with ZERO bytes from current readable/writable
+ * position.
+ * @param buffer a buffer ready to read / write certain size bytes
* @return the buffer itself, with ZERO bytes written, the position and limit
* are not changed after the call
*/
- protected ByteBuffer resetOutputBuffer(ByteBuffer buffer) {
+ protected ByteBuffer resetBuffer(ByteBuffer buffer) {
int pos = buffer.position();
for (int i = pos; i buffer.limit(); ++i) {
buffer.put((byte) 0);
@@ -77,7 +78,7 @@ public abstract class AbstractRawErasureCoder
/**
* Ensure the buffer (either input or output) ready to read or write with
ZERO
- * bytes fully in chunkSize.
+ * bytes fully in specified length of len.
* @param buffer bytes array buffer
* @return the buffer itself
*/
@@ -92,11 +93,16 @@ public abstract class AbstractRawErasureCoder
/**
* Check and ensure the buffers are of the length specified by dataLen.
* @param buffers
+ * @param allowNull
* @param dataLen
*/
- protected void ensureLength(ByteBuffer[] buffers, int dataLen) {
+ protected void ensureLength(ByteBuffer[] buffers,
+ boolean allowNull, int dataLen) {
for (int i = 0; i buffers.length; ++i) {
- if (buffers[i].remaining() != dataLen) {
+ if (buffers[i] == null !allowNull) {
+throw new HadoopIllegalArgumentException(
+Invalid buffer found, not allowing null);
+ } else if (buffers[i] != null
