Author: arp
Date: Tue Feb 25 02:20:51 2014
New Revision: 1571546
URL: http://svn.apache.org/r1571546
Log:
HDFS-5922. Merging r1571543 from branch-2 to branch-2.4
Added:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
- copied unchanged from r1571543,
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1571546&r1=1571545&r2=1571546&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue Feb 25 02:20:51 2014
@@ -272,6 +272,8 @@ Release 2.4.0 - UNRELEASED
HDFS-5981. PBImageXmlWriter generates malformed XML.
(Haohui Mai via cnauroth)
+ HDFS-5922. DN heartbeat thread can get stuck in tight loop. (Arpit Agarwal)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1571546&r1=1571545&r2=1571546&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
Tue Feb 25 02:20:51 2014
@@ -1113,6 +1113,7 @@ public class DFSOutputStream extends FSO
excluded.length > 0 ? excluded : null);
block = lb.getBlock();
block.setNumBytes(0);
+ bytesSent = 0;
accessToken = lb.getBlockToken();
nodes = lb.getLocations();
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java?rev=1571546&r1=1571545&r2=1571546&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
Tue Feb 25 02:20:51 2014
@@ -101,7 +101,10 @@ class BPServiceActor implements Runnable
private final Map<String, PerStoragePendingIncrementalBR>
pendingIncrementalBRperStorage = Maps.newHashMap();
- private volatile int pendingReceivedRequests = 0;
+ // IBR = Incremental Block Report. If this flag is set then an IBR will be
+ // sent immediately by the actor thread without waiting for the IBR timer
+ // to elapse.
+ private volatile boolean sendImmediateIBR = false;
private volatile boolean shouldServiceRun = true;
private final DataNode dn;
private final DNConf dnConf;
@@ -283,12 +286,10 @@ class BPServiceActor implements Runnable
if (perStorageMap.getBlockInfoCount() > 0) {
// Send newly-received and deleted blockids to namenode
ReceivedDeletedBlockInfo[] rdbi = perStorageMap.dequeueBlockInfos();
- pendingReceivedRequests =
- (pendingReceivedRequests > rdbi.length ?
- (pendingReceivedRequests - rdbi.length) : 0);
reports.add(new StorageReceivedDeletedBlocks(storageUuid, rdbi));
}
}
+ sendImmediateIBR = false;
}
if (reports.size() == 0) {
@@ -312,8 +313,8 @@ class BPServiceActor implements Runnable
// didn't put something newer in the meantime.
PerStoragePendingIncrementalBR perStorageMap =
pendingIncrementalBRperStorage.get(report.getStorageID());
- pendingReceivedRequests +=
- perStorageMap.putMissingBlockInfos(report.getBlocks());
+ perStorageMap.putMissingBlockInfos(report.getBlocks());
+ sendImmediateIBR = true;
}
}
}
@@ -371,7 +372,7 @@ class BPServiceActor implements Runnable
ReceivedDeletedBlockInfo bInfo, String storageUuid) {
synchronized (pendingIncrementalBRperStorage) {
addPendingReplicationBlockInfo(bInfo, storageUuid);
- pendingReceivedRequests++;
+ sendImmediateIBR = true;
pendingIncrementalBRperStorage.notifyAll();
}
}
@@ -433,6 +434,11 @@ class BPServiceActor implements Runnable
}
}
+ @VisibleForTesting
+ boolean hasPendingIBR() {
+ return sendImmediateIBR;
+ }
+
/**
* Report the list blocks to the Namenode
* @return DatanodeCommands returned by the NN. May be null.
@@ -676,8 +682,8 @@ class BPServiceActor implements Runnable
}
}
}
- if (pendingReceivedRequests > 0
- || (startTime - lastDeletedReport > dnConf.deleteReportInterval)) {
+ if (sendImmediateIBR ||
+ (startTime - lastDeletedReport > dnConf.deleteReportInterval)) {
reportReceivedDeletedBlocks();
lastDeletedReport = startTime;
}
@@ -701,7 +707,7 @@ class BPServiceActor implements Runnable
long waitTime = dnConf.heartBeatInterval -
(Time.now() - lastHeartbeat);
synchronized(pendingIncrementalBRperStorage) {
- if (waitTime > 0 && pendingReceivedRequests == 0) {
+ if (waitTime > 0 && !sendImmediateIBR) {
try {
pendingIncrementalBRperStorage.wait(waitTime);
} catch (InterruptedException ie) {
