Repository: hadoop Updated Branches: refs/heads/branch-2.6 f86c9c6c7 -> 0280e9aee
HDFS-7217. Better batching of IBRs. Contributed by Kihwal Lee. (cherry picked from commit db71bb54bcc75b71c5841b25ceb03fb0218c6d4f) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0280e9ae Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0280e9ae Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0280e9ae Branch: refs/heads/branch-2.6 Commit: 0280e9aee07cd50936345a0f79d799a7e6ca9dd9 Parents: f86c9c6 Author: Kihwal Lee <kih...@apache.org> Authored: Thu Oct 9 12:11:42 2014 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Thu Oct 9 12:11:42 2014 -0500 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hadoop/hdfs/server/datanode/BPOfferService.java | 4 ++-- .../hadoop/hdfs/server/datanode/BPServiceActor.java | 10 +++++++--- .../hdfs/server/datanode/TestIncrementalBlockReports.java | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0280e9ae/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b42d1cd..85a2ad9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -278,6 +278,8 @@ Release 2.6.0 - UNRELEASED HDFS-6894. Add XDR parser method for each NFS response. (Brandon Li via wheat9) + HDFS-7217. Better batching of IBRs. (kihwal) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/0280e9ae/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index ae64d8a..2eab87f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -237,7 +237,7 @@ class BPOfferService { delHint); for (BPServiceActor actor : bpServices) { - actor.notifyNamenodeBlockImmediately(bInfo, storageUuid); + actor.notifyNamenodeBlock(bInfo, storageUuid, true); } } @@ -270,7 +270,7 @@ class BPOfferService { block.getLocalBlock(), BlockStatus.RECEIVING_BLOCK, null); for (BPServiceActor actor : bpServices) { - actor.notifyNamenodeBlockImmediately(bInfo, storageUuid); + actor.notifyNamenodeBlock(bInfo, storageUuid, false); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0280e9ae/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 7d30688..2961698 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -368,13 +368,17 @@ class BPServiceActor implements Runnable { * till namenode is informed before responding with success to the * client? For now we don't. */ - void notifyNamenodeBlockImmediately( - ReceivedDeletedBlockInfo bInfo, String storageUuid) { + void notifyNamenodeBlock(ReceivedDeletedBlockInfo bInfo, + String storageUuid, boolean now) { synchronized (pendingIncrementalBRperStorage) { addPendingReplicationBlockInfo( bInfo, dn.getFSDataset().getStorage(storageUuid)); sendImmediateIBR = true; - pendingIncrementalBRperStorage.notifyAll(); + // If now is true, the report is sent right away. + // Otherwise, it will be sent out in the next heartbeat. + if (now) { + pendingIncrementalBRperStorage.notifyAll(); + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0280e9ae/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java index 81dc13c..b5aa93f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java @@ -84,7 +84,7 @@ public class TestIncrementalBlockReports { private void injectBlockReceived() { ReceivedDeletedBlockInfo rdbi = new ReceivedDeletedBlockInfo( getDummyBlock(), BlockStatus.RECEIVED_BLOCK, null); - actor.notifyNamenodeBlockImmediately(rdbi, storageUuid); + actor.notifyNamenodeBlock(rdbi, storageUuid, true); } /**