hadoop git commit: HDFS-8744. Erasure Coding: the number of chunks in packet is not updated when writing parity data. Contributed by Li Bo
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 e692c7dd9 -> f4098dfae
HDFS-8744. Erasure Coding: the number of chunks in packet is not updated when
writing parity data. Contributed by Li Bo
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4098dfa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4098dfa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4098dfa
Branch: refs/heads/HDFS-7285
Commit: f4098dfae44b6dd287a3054f2c04658773b4f466
Parents: e692c7d
Author: boli2
Authored: Mon Jul 13 00:41:36 2015 -0400
Committer: boli2
Committed: Mon Jul 13 00:41:36 2015 -0400
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++
.../src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 2 +-
.../src/main/java/org/apache/hadoop/hdfs/DFSPacket.java| 6 +++---
.../java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java| 4 +++-
4 files changed, 10 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4098dfa/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 90f6732..7b6d165 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -341,3 +341,6 @@
HDFS-8484. Erasure coding: Two contiguous blocks occupy IDs belong to same
striped group. (Walter Su via jing9)
+
+HDFS-8744. Erasure Coding: the number of chunks in packet is not updated
+when writing parity data. (Li Bo)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4098dfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 9e201ad..f41044b 100755
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -419,7 +419,7 @@ public class DFSOutputStream extends FSOutputSummer
currentPacket.writeChecksum(checksum, ckoff, cklen);
currentPacket.writeData(b, offset, len);
-currentPacket.incNumChunks();
+currentPacket.incNumChunks(1);
streamer.incBytesCurBlock(len);
// If packet is full, enqueue it for transmission
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4098dfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
index a26e35e..2698de3 100755
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
@@ -259,10 +259,10 @@ public class DFSPacket {
}
/**
- * increase the number of chunks by one
+ * increase the number of chunks by n
*/
- synchronized void incNumChunks() {
-numChunks++;
+ synchronized void incNumChunks(int n) {
+numChunks += n;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4098dfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 4234351..e6de714 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -389,11 +389,13 @@ public class DFSStripedOutputStream extends
DFSOutputStream {
int maxBytesToPacket = p.getMaxChunks() * bytesPerChecksum;
int toWrite = byteBuffer.remaining() > maxBytesToPacket ?
maxBytesToPacket: byteBuffer.remaining();
- int ckLen = ((toWrite - 1) / bytesPerChecksum + 1) * getChecksumSize();
+ int chunks = (toWrite - 1) / bytesPerChecksum + 1;
+ int ckLen = chunks * getChecksumSize();
p.writeChecksum(checksumBuf, ckOff, ckLen);
ckOff += ckLen;
hadoop git commit: HDFS-8857. Erasure Coding: Fix ArrayIndexOutOfBoundsException in TestWriteStripedFileWithFailure. Contributed by Li Bo
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 4de48211c -> 8799363db
HDFS-8857. Erasure Coding: Fix ArrayIndexOutOfBoundsException in
TestWriteStripedFileWithFailure. Contributed by Li Bo
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8799363d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8799363d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8799363d
Branch: refs/heads/HDFS-7285
Commit: 8799363db1c0e0ce0abd4ab68b780092e7dc5263
Parents: 4de4821
Author: boli2
Authored: Wed Aug 5 22:11:50 2015 -0400
Committer: boli2
Committed: Wed Aug 5 22:11:50 2015 -0400
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++
.../org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8799363d/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 8e54c76..5ad084b 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -388,3 +388,6 @@
HDFS-8399. Erasure Coding: unit test the behaviour of BlockManager recovery
work for the deleted blocks. (Rakesh R via zhz)
+
+HDFS-8857. Erasure Coding: Fix ArrayIndexOutOfBoundsException in
+TestWriteStripedFileWithFailure. (Li Bo)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8799363d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
index 5448773..3679c5f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
@@ -37,7 +37,7 @@ import static
org.apache.hadoop.hdfs.StripedFileTestUtil.parityBlocks;
public class TestWriteStripedFileWithFailure {
public static final Log LOG = LogFactory
- .getLog(TestReadStripedFileWithMissingBlocks.class);
+ .getLog(TestWriteStripedFileWithFailure.class);
private static MiniDFSCluster cluster;
private static FileSystem fs;
private static Configuration conf = new HdfsConfiguration();
@@ -114,7 +114,7 @@ public class TestWriteStripedFileWithFailure {
dataDNFailureNum);
Assert.assertNotNull(dataDNFailureIndices);
int[] parityDNFailureIndices = StripedFileTestUtil.randomArray(dataBlocks,
-dataBlocks + parityBlocks, dataDNFailureNum);
+dataBlocks + parityBlocks, parityDNFailureNum);
Assert.assertNotNull(parityDNFailureIndices);
int[] failedDataNodes = new int[dataDNFailureNum + parityDNFailureNum];
hadoop git commit: HDFS-8366. Erasure Coding: Make the timeout parameter of polling blocking queue configurable in DFSStripedOutputStream. Contributed by Li Bo
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 12d030bef -> 3cf3398f3
HDFS-8366. Erasure Coding: Make the timeout parameter of polling blocking queue
configurable in DFSStripedOutputStream. Contributed by Li Bo
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3cf3398f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3cf3398f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3cf3398f
Branch: refs/heads/HDFS-7285
Commit: 3cf3398f3c23b9e9ca421cfa66fdf15081fc86da
Parents: 12d030b
Author: boli2
Authored: Tue May 19 02:14:46 2015 -0400
Committer: boli2
Committed: Tue May 19 02:14:46 2015 -0400
--
.../hdfs/client/HdfsClientConfigKeys.java | 10
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 +++
.../hadoop/hdfs/DFSStripedOutputStream.java | 19
.../hadoop/hdfs/client/impl/DfsClientConf.java | 24
4 files changed, 51 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cf3398f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 6006d71..9373e98 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -189,6 +189,16 @@ public interface HdfsClientConfigKeys {
int THREADPOOL_SIZE_DEFAULT = 18;
}
+ /** dfs.client.write.striped configuration properties */
+ interface StripedWrite {
+String PREFIX = Write.PREFIX + "striped.";
+
+String MAX_SECONDS_GET_STRIPED_BLOCK_KEY = PREFIX +
"max-seconds-get-striped-block";
+int MAX_SECONDS_GET_STRIPED_BLOCK_DEFAULT = 90;
+String MAX_SECONDS_GET_ENDED_BLOCK_KEY = PREFIX +
"max-seconds-get-ended-block";
+int MAX_SECONDS_GET_ENDED_BLOCK_DEFAULT = 60;
+ }
+
/** dfs.http.client configuration properties */
interface HttpClient {
String PREFIX = "dfs.http.client.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cf3398f/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
old mode 100644
new mode 100755
index 3170e9b..939ba89
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -225,3 +225,6 @@
(Yi Liu via jing9)
HDFS-8320. Erasure coding: consolidate striping-related terminologies.
(zhz)
+
+ HDFS-8366. Erasure Coding: Make the timeout parameter of polling
blocking queue
+ configurable in DFSStripedOutputStream. (Li Bo)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cf3398f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index b99afab..a648023 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -33,6 +33,8 @@ import java.util.concurrent.TimeUnit;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -61,11 +63,14 @@ import com.google.common.base.Preconditions;
public class DFSStripedOutputStream extends DFSOutputStream {
/** Coordinate the communication between the streamers. */
static class Coordinator {
+private final DfsClientConf conf;
private final List> endBlocks;
private final List> stripedBlocks;
private volatile boolean shouldLocateFollowingBlock = false;
-Coordinator(final int numDataBlocks, final int numAllBlocks
