[07/50] [abbrv] hadoop git commit: HDFS-4937. ReplicationMonitor can infinite-loop in BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.

2015-11-12 Thread wheat9 
HDFS-4937. ReplicationMonitor can infinite-loop in 
BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff47f35d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff47f35d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff47f35d

Branch: refs/heads/HDFS-8707
Commit: ff47f35deed14ba6463cba76f0e6a6c15abb3eca
Parents: c563401
Author: Kihwal Lee 
Authored: Thu Nov 5 09:25:20 2015 -0600
Committer: Kihwal Lee 
Committed: Thu Nov 5 09:25:20 2015 -0600

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
 .../server/blockmanagement/BlockPlacementPolicyDefault.java | 9 +
 2 files changed, 12 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff47f35d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 910753a..f5c6f0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2256,6 +2256,9 @@ Release 2.7.3 - UNRELEASED
 HDFS-9289. Make DataStreamer#block thread safe and verify genStamp in
 commitBlock. (Chang Li via zhz)
 
+HDFS-4937. ReplicationMonitor can infinite-loop in 
+BlockPlacementPolicyDefault#chooseRandom(). (kihwal)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff47f35d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 56ebc35..d94179b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -659,6 +659,7 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 
 int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(
 scope, excludedNodes);
+int refreshCounter = numOfAvailableNodes;
 StringBuilder builder = null;
 if (LOG.isDebugEnabled()) {
   builder = debugLoggingBuilder.get();
@@ -708,6 +709,14 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 // If no candidate storage was found on this DN then set badTarget.
 badTarget = (storage == null);
   }
+  // Refresh the node count. If the live node count became smaller,
+  // but it is not reflected in this loop, it may loop forever in case
+  // the replicas/rack cannot be satisfied.
+  if (--refreshCounter == 0) {
+numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(scope,
+excludedNodes);
+refreshCounter = numOfAvailableNodes;
+  }
 }
   
 if (numOfReplicas>0) {

hadoop git commit: HDFS-4937. ReplicationMonitor can infinite-loop in BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.

2015-11-05 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/trunk c56340179 -> ff47f35de


HDFS-4937. ReplicationMonitor can infinite-loop in 
BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff47f35d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff47f35d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff47f35d

Branch: refs/heads/trunk
Commit: ff47f35deed14ba6463cba76f0e6a6c15abb3eca
Parents: c563401
Author: Kihwal Lee 
Authored: Thu Nov 5 09:25:20 2015 -0600
Committer: Kihwal Lee 
Committed: Thu Nov 5 09:25:20 2015 -0600

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
 .../server/blockmanagement/BlockPlacementPolicyDefault.java | 9 +
 2 files changed, 12 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff47f35d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 910753a..f5c6f0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2256,6 +2256,9 @@ Release 2.7.3 - UNRELEASED
 HDFS-9289. Make DataStreamer#block thread safe and verify genStamp in
 commitBlock. (Chang Li via zhz)
 
+HDFS-4937. ReplicationMonitor can infinite-loop in 
+BlockPlacementPolicyDefault#chooseRandom(). (kihwal)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff47f35d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 56ebc35..d94179b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -659,6 +659,7 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 
 int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(
 scope, excludedNodes);
+int refreshCounter = numOfAvailableNodes;
 StringBuilder builder = null;
 if (LOG.isDebugEnabled()) {
   builder = debugLoggingBuilder.get();
@@ -708,6 +709,14 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 // If no candidate storage was found on this DN then set badTarget.
 badTarget = (storage == null);
   }
+  // Refresh the node count. If the live node count became smaller,
+  // but it is not reflected in this loop, it may loop forever in case
+  // the replicas/rack cannot be satisfied.
+  if (--refreshCounter == 0) {
+numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(scope,
+excludedNodes);
+refreshCounter = numOfAvailableNodes;
+  }
 }
   
 if (numOfReplicas>0) {

[06/50] [abbrv] hadoop git commit: HDFS-4937. ReplicationMonitor can infinite-loop in BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.

2015-11-04 Thread aengineer 
HDFS-4937. ReplicationMonitor can infinite-loop in 
BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43539b5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43539b5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43539b5f

Branch: refs/heads/HDFS-7240
Commit: 43539b5ff4ac0874a8a454dc93a2a782b0e0ea8f
Parents: ce31b22
Author: Kihwal Lee 
Authored: Fri Oct 30 09:27:21 2015 -0500
Committer: Kihwal Lee 
Committed: Fri Oct 30 09:29:13 2015 -0500

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 +++
 .../blockmanagement/BlockPlacementPolicyDefault.java| 12 
 2 files changed, 15 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/43539b5f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 38b9e55..f6a22a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2201,6 +2201,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-9332. Fix Precondition failures from NameNodeEditLogRoller while
 saving namespace. (wang)
 
+HDFS-4937. ReplicationMonitor can infinite-loop in
+BlockPlacementPolicyDefault#chooseRandom() (kihwal)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/43539b5f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d9b8d60..f610574 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -659,6 +659,7 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 
 int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(
 scope, excludedNodes);
+int refreshCounter = numOfAvailableNodes;
 StringBuilder builder = null;
 if (LOG.isDebugEnabled()) {
   builder = debugLoggingBuilder.get();
@@ -708,6 +709,17 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 // If no candidate storage was found on this DN then set badTarget.
 badTarget = (storage == null);
   }
+  // Refresh the node count. If the live node count became smaller,
+  // but it is not reflected in this loop, it may loop forever in case
+  // the replicas/rack cannot be satisfied.
+  if (--refreshCounter == 0) {
+refreshCounter = clusterMap.countNumOfAvailableNodes(scope,
+excludedNodes);
+// It has already gone through enough number of nodes.
+if (refreshCounter <= excludedNodes.size()) {
+  break;
+}
+  }
 }
   
 if (numOfReplicas>0) {

[20/50] hadoop git commit: HDFS-4937. ReplicationMonitor can infinite-loop in BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.

2015-11-03 Thread jing9 
HDFS-4937. ReplicationMonitor can infinite-loop in 
BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43539b5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43539b5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43539b5f

Branch: refs/heads/HDFS-8966
Commit: 43539b5ff4ac0874a8a454dc93a2a782b0e0ea8f
Parents: ce31b22
Author: Kihwal Lee 
Authored: Fri Oct 30 09:27:21 2015 -0500
Committer: Kihwal Lee 
Committed: Fri Oct 30 09:29:13 2015 -0500

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 +++
 .../blockmanagement/BlockPlacementPolicyDefault.java| 12 
 2 files changed, 15 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/43539b5f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 38b9e55..f6a22a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2201,6 +2201,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-9332. Fix Precondition failures from NameNodeEditLogRoller while
 saving namespace. (wang)
 
+HDFS-4937. ReplicationMonitor can infinite-loop in
+BlockPlacementPolicyDefault#chooseRandom() (kihwal)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/43539b5f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d9b8d60..f610574 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -659,6 +659,7 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 
 int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(
 scope, excludedNodes);
+int refreshCounter = numOfAvailableNodes;
 StringBuilder builder = null;
 if (LOG.isDebugEnabled()) {
   builder = debugLoggingBuilder.get();
@@ -708,6 +709,17 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 // If no candidate storage was found on this DN then set badTarget.
 badTarget = (storage == null);
   }
+  // Refresh the node count. If the live node count became smaller,
+  // but it is not reflected in this loop, it may loop forever in case
+  // the replicas/rack cannot be satisfied.
+  if (--refreshCounter == 0) {
+refreshCounter = clusterMap.countNumOfAvailableNodes(scope,
+excludedNodes);
+// It has already gone through enough number of nodes.
+if (refreshCounter <= excludedNodes.size()) {
+  break;
+}
+  }
 }
   
 if (numOfReplicas>0) {

hadoop git commit: HDFS-4937. ReplicationMonitor can infinite-loop in BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.

2015-10-30 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/trunk ce31b2273 -> 43539b5ff


HDFS-4937. ReplicationMonitor can infinite-loop in 
BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43539b5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43539b5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43539b5f

Branch: refs/heads/trunk
Commit: 43539b5ff4ac0874a8a454dc93a2a782b0e0ea8f
Parents: ce31b22
Author: Kihwal Lee 
Authored: Fri Oct 30 09:27:21 2015 -0500
Committer: Kihwal Lee 
Committed: Fri Oct 30 09:29:13 2015 -0500

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 +++
 .../blockmanagement/BlockPlacementPolicyDefault.java| 12 
 2 files changed, 15 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/43539b5f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 38b9e55..f6a22a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2201,6 +2201,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-9332. Fix Precondition failures from NameNodeEditLogRoller while
 saving namespace. (wang)
 
+HDFS-4937. ReplicationMonitor can infinite-loop in
+BlockPlacementPolicyDefault#chooseRandom() (kihwal)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/43539b5f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d9b8d60..f610574 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -659,6 +659,7 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 
 int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(
 scope, excludedNodes);
+int refreshCounter = numOfAvailableNodes;
 StringBuilder builder = null;
 if (LOG.isDebugEnabled()) {
   builder = debugLoggingBuilder.get();
@@ -708,6 +709,17 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 // If no candidate storage was found on this DN then set badTarget.
 badTarget = (storage == null);
   }
+  // Refresh the node count. If the live node count became smaller,
+  // but it is not reflected in this loop, it may loop forever in case
+  // the replicas/rack cannot be satisfied.
+  if (--refreshCounter == 0) {
+refreshCounter = clusterMap.countNumOfAvailableNodes(scope,
+excludedNodes);
+// It has already gone through enough number of nodes.
+if (refreshCounter <= excludedNodes.size()) {
+  break;
+}
+  }
 }
   
 if (numOfReplicas>0) {

hadoop git commit: HDFS-4937. ReplicationMonitor can infinite-loop in BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee. (cherry picked from commit 43539b5ff4ac0874a8a454dc93a2a782

2015-10-30 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 1c80473a4 -> 0c7fd2630


HDFS-4937. ReplicationMonitor can infinite-loop in 
BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.
(cherry picked from commit 43539b5ff4ac0874a8a454dc93a2a782b0e0ea8f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c7fd263
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c7fd263
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c7fd263

Branch: refs/heads/branch-2
Commit: 0c7fd26302109273a07b6087a9b92c9dff080816
Parents: 1c80473
Author: Kihwal Lee 
Authored: Fri Oct 30 09:30:11 2015 -0500
Committer: Kihwal Lee 
Committed: Fri Oct 30 09:30:11 2015 -0500

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 +++
 .../blockmanagement/BlockPlacementPolicyDefault.java| 12 
 2 files changed, 15 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c7fd263/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 52775a8..fffdc91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1364,6 +1364,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-9332. Fix Precondition failures from NameNodeEditLogRoller while
 saving namespace. (wang)
 
+HDFS-4937. ReplicationMonitor can infinite-loop in
+BlockPlacementPolicyDefault#chooseRandom() (kihwal)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c7fd263/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d9b8d60..f610574 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -659,6 +659,7 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 
 int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(
 scope, excludedNodes);
+int refreshCounter = numOfAvailableNodes;
 StringBuilder builder = null;
 if (LOG.isDebugEnabled()) {
   builder = debugLoggingBuilder.get();
@@ -708,6 +709,17 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 // If no candidate storage was found on this DN then set badTarget.
 badTarget = (storage == null);
   }
+  // Refresh the node count. If the live node count became smaller,
+  // but it is not reflected in this loop, it may loop forever in case
+  // the replicas/rack cannot be satisfied.
+  if (--refreshCounter == 0) {
+refreshCounter = clusterMap.countNumOfAvailableNodes(scope,
+excludedNodes);
+// It has already gone through enough number of nodes.
+if (refreshCounter <= excludedNodes.size()) {
+  break;
+}
+  }
 }
   
 if (numOfReplicas>0) {

hadoop git commit: HDFS-4937. ReplicationMonitor can infinite-loop in BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee. (cherry picked from commit 43539b5ff4ac0874a8a454dc93a2a782

2015-10-30 Thread kihwal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 c84ec0657 -> c250b21c2


HDFS-4937. ReplicationMonitor can infinite-loop in 
BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.
(cherry picked from commit 43539b5ff4ac0874a8a454dc93a2a782b0e0ea8f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c250b21c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c250b21c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c250b21c

Branch: refs/heads/branch-2.7
Commit: c250b21c23945ce2c580186c224cc65ab2b501fc
Parents: c84ec06
Author: Kihwal Lee 
Authored: Fri Oct 30 09:39:49 2015 -0500
Committer: Kihwal Lee 
Committed: Fri Oct 30 09:39:49 2015 -0500

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 +++
 .../blockmanagement/BlockPlacementPolicyDefault.java| 12 
 2 files changed, 15 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c250b21c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index bd6e9f0..59a1717 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -89,6 +89,9 @@ Release 2.7.2 - UNRELEASED
 HDFS-9317. Document fsck -blockId and -storagepolicy options in branch-2.7.
 (aajisaka)
 
+HDFS-4937. ReplicationMonitor can infinite-loop in
+BlockPlacementPolicyDefault#chooseRandom() (kihwal)
+
 Release 2.7.1 - 2015-07-06
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c250b21c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 97ea782..086abca 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -622,6 +622,7 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
   
 int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(
 scope, excludedNodes);
+int refreshCounter = numOfAvailableNodes;
 StringBuilder builder = null;
 if (LOG.isDebugEnabled()) {
   builder = debugLoggingBuilder.get();
@@ -675,6 +676,17 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 // If no candidate storage was found on this DN then set badTarget.
 badTarget = (i == storages.length);
   }
+  // Refresh the node count. If the live node count became smaller,
+  // but it is not reflected in this loop, it may loop forever in case
+  // the replicas/rack cannot be satisfied.
+  if (--refreshCounter == 0) {
+refreshCounter = clusterMap.countNumOfAvailableNodes(scope,
+excludedNodes);
+// It has already gone through enough number of nodes.
+if (refreshCounter <= excludedNodes.size()) {
+  break;
+}
+  }
 }
   
 if (numOfReplicas>0) {