Author: arp
Date: Thu Aug 21 01:14:14 2014
New Revision: 1619276
URL: http://svn.apache.org/r1619276
Log:
HDFS-6758: Merging r1619275 from trunk to branch-2.
Added:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
- copied unchanged from r1619275,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1619276&r1=1619275&r2=1619276&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Thu Aug 21 01:14:14 2014
@@ -154,7 +154,10 @@ Release 2.6.0 - UNRELEASED
hadoop.security.saslproperties.resolver.class. (Benoy Antony via cnauroth)
HDFS-6878. Change MiniDFSCluster to support StorageType configuration
- for individual directories (Arpit Agarwal)
+ for individual directories. (Arpit Agarwal)
+
+ HDFS-6758. block writer should pass the expected block size to
+ DataXceiverServer. (Arpit Agarwal)
OPTIMIZATIONS
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1619276&r1=1619275&r2=1619276&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
Thu Aug 21 01:14:14 2014
@@ -1339,8 +1339,14 @@ public class DFSOutputStream extends FSO
//
BlockConstructionStage bcs = recoveryFlag? stage.getRecoveryStage():
stage;
+
+ // We cannot change the block length in 'block' as it counts the
number
+ // of bytes ack'ed.
+ ExtendedBlock blockCopy = new ExtendedBlock(block);
+ blockCopy.setNumBytes(blockSize);
+
// send the request
- new Sender(out).writeBlock(block, nodeStorageTypes[0], accessToken,
+ new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0],
accessToken,
dfsClient.clientName, nodes, nodeStorageTypes, null, bcs,
nodes.length, block.getNumBytes(), bytesSent, newGS, checksum,
cachingStrategy.get());
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java?rev=1619276&r1=1619275&r2=1619276&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
Thu Aug 21 01:14:14 2014
@@ -576,7 +576,9 @@ class DataXceiver extends Receiver imple
// forward the original version of the block to downstream mirrors, so
// make a copy here.
final ExtendedBlock originalBlock = new ExtendedBlock(block);
- block.setNumBytes(dataXceiverServer.estimateBlockSize);
+ if (block.getNumBytes() == 0) {
+ block.setNumBytes(dataXceiverServer.estimateBlockSize);
+ }
LOG.info("Receiving " + block + " src: " + remoteAddress + " dest: "
+ localAddress);
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java?rev=1619276&r1=1619275&r2=1619276&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
Thu Aug 21 01:14:14 2014
@@ -100,11 +100,8 @@ class DataXceiverServer implements Runna
/**
* We need an estimate for block size to check if the disk partition has
- * enough space. For now we set it to be the default block size set
- * in the server side configuration, which is not ideal because the
- * default block size should be a client-size configuration.
- * A better solution is to include in the header the estimated block size,
- * i.e. either the actual block size or the default block size.
+ * enough space. Newer clients pass the expected block size to the DataNode.
+ * For older clients we just use the server-side default block size.
*/
final long estimateBlockSize;
