[12/50] [abbrv] hadoop git commit: HDFS-13833. Improve BlockPlacementPolicyDefault's consider load logic. Contributed by Shweta.

2018-09-20 Thread xyao 
HDFS-13833. Improve BlockPlacementPolicyDefault's consider load logic. 
Contributed by Shweta.

Signed-off-by: Xiao Chen 


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27978bcb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27978bcb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27978bcb

Branch: refs/heads/HDDS-4
Commit: 27978bcb66a9130cbf26d37ec454c0b7fcdc2530
Parents: 3929653
Author: Shweta 
Authored: Tue Sep 18 20:22:25 2018 -0700
Committer: Xiao Chen 
Committed: Tue Sep 18 20:23:50 2018 -0700

--
 .../BlockPlacementPolicyDefault.java| 29 ++--
 .../blockmanagement/TestReplicationPolicy.java  | 28 +++
 2 files changed, 49 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d00f961..d396845 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -913,6 +913,24 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
   }
 
   /**
+   * Determine if a datanode should be chosen based on current workload.
+   *
+   * @param node The target datanode
+   * @return Return true if the datanode should be excluded, otherwise false
+   */
+  boolean excludeNodeByLoad(DatanodeDescriptor node){
+final double maxLoad = considerLoadFactor *
+stats.getInServiceXceiverAverage();
+final int nodeLoad = node.getXceiverCount();
+if ((nodeLoad > maxLoad) && (maxLoad > 0)) {
+  logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
+  "(load: " + nodeLoad + " > " + maxLoad + ")");
+  return true;
+}
+return false;
+  }
+
+  /**
* Determine if a datanode is good for placing block.
*
* @param node The target datanode
@@ -923,7 +941,7 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
* @param results A list containing currently chosen nodes. Used to check if
*too many nodes has been chosen in the target rack.
* @param avoidStaleNodes Whether or not to avoid choosing stale nodes
-   * @return Reture true if the datanode is good candidate, otherwise false
+   * @return Return true if the datanode is good candidate, otherwise false
*/
   boolean isGoodDatanode(DatanodeDescriptor node,
  int maxTargetPerRack, boolean considerLoad,
@@ -943,13 +961,8 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 }
 
 // check the communication traffic of the target machine
-if (considerLoad) {
-  final double maxLoad = considerLoadFactor *
-  stats.getInServiceXceiverAverage();
-  final int nodeLoad = node.getXceiverCount();
-  if (nodeLoad > maxLoad) {
-logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
-"(load: " + nodeLoad + " > " + maxLoad + ")");
+if(considerLoad){
+  if(excludeNodeByLoad(node)){
 return false;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 27dcbf1..f08fa13 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.server.namenode.Namesystem;
 import org.apache.hadoop.hdfs.server.namenode.TestINodeFile;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.net.Node;
+import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.apache.log4j.spi.LoggingEvent;
@@

[28/50] [abbrv] hadoop git commit: HDFS-13833. Improve BlockPlacementPolicyDefault's consider load logic. Contributed by Shweta.

2018-09-20 Thread xkrogen 
HDFS-13833. Improve BlockPlacementPolicyDefault's consider load logic. 
Contributed by Shweta.

Signed-off-by: Xiao Chen 


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27978bcb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27978bcb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27978bcb

Branch: refs/heads/HDFS-12943
Commit: 27978bcb66a9130cbf26d37ec454c0b7fcdc2530
Parents: 3929653
Author: Shweta 
Authored: Tue Sep 18 20:22:25 2018 -0700
Committer: Xiao Chen 
Committed: Tue Sep 18 20:23:50 2018 -0700

--
 .../BlockPlacementPolicyDefault.java| 29 ++--
 .../blockmanagement/TestReplicationPolicy.java  | 28 +++
 2 files changed, 49 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d00f961..d396845 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -913,6 +913,24 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
   }
 
   /**
+   * Determine if a datanode should be chosen based on current workload.
+   *
+   * @param node The target datanode
+   * @return Return true if the datanode should be excluded, otherwise false
+   */
+  boolean excludeNodeByLoad(DatanodeDescriptor node){
+final double maxLoad = considerLoadFactor *
+stats.getInServiceXceiverAverage();
+final int nodeLoad = node.getXceiverCount();
+if ((nodeLoad > maxLoad) && (maxLoad > 0)) {
+  logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
+  "(load: " + nodeLoad + " > " + maxLoad + ")");
+  return true;
+}
+return false;
+  }
+
+  /**
* Determine if a datanode is good for placing block.
*
* @param node The target datanode
@@ -923,7 +941,7 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
* @param results A list containing currently chosen nodes. Used to check if
*too many nodes has been chosen in the target rack.
* @param avoidStaleNodes Whether or not to avoid choosing stale nodes
-   * @return Reture true if the datanode is good candidate, otherwise false
+   * @return Return true if the datanode is good candidate, otherwise false
*/
   boolean isGoodDatanode(DatanodeDescriptor node,
  int maxTargetPerRack, boolean considerLoad,
@@ -943,13 +961,8 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 }
 
 // check the communication traffic of the target machine
-if (considerLoad) {
-  final double maxLoad = considerLoadFactor *
-  stats.getInServiceXceiverAverage();
-  final int nodeLoad = node.getXceiverCount();
-  if (nodeLoad > maxLoad) {
-logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
-"(load: " + nodeLoad + " > " + maxLoad + ")");
+if(considerLoad){
+  if(excludeNodeByLoad(node)){
 return false;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 27dcbf1..f08fa13 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.server.namenode.Namesystem;
 import org.apache.hadoop.hdfs.server.namenode.TestINodeFile;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.net.Node;
+import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.apache.log4j.spi.LoggingEvent;

[24/50] [abbrv] hadoop git commit: HDFS-13833. Improve BlockPlacementPolicyDefault's consider load logic. Contributed by Shweta.

2018-09-19 Thread botong 
HDFS-13833. Improve BlockPlacementPolicyDefault's consider load logic. 
Contributed by Shweta.

Signed-off-by: Xiao Chen 


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27978bcb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27978bcb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27978bcb

Branch: refs/heads/YARN-7402
Commit: 27978bcb66a9130cbf26d37ec454c0b7fcdc2530
Parents: 3929653
Author: Shweta 
Authored: Tue Sep 18 20:22:25 2018 -0700
Committer: Xiao Chen 
Committed: Tue Sep 18 20:23:50 2018 -0700

--
 .../BlockPlacementPolicyDefault.java| 29 ++--
 .../blockmanagement/TestReplicationPolicy.java  | 28 +++
 2 files changed, 49 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d00f961..d396845 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -913,6 +913,24 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
   }
 
   /**
+   * Determine if a datanode should be chosen based on current workload.
+   *
+   * @param node The target datanode
+   * @return Return true if the datanode should be excluded, otherwise false
+   */
+  boolean excludeNodeByLoad(DatanodeDescriptor node){
+final double maxLoad = considerLoadFactor *
+stats.getInServiceXceiverAverage();
+final int nodeLoad = node.getXceiverCount();
+if ((nodeLoad > maxLoad) && (maxLoad > 0)) {
+  logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
+  "(load: " + nodeLoad + " > " + maxLoad + ")");
+  return true;
+}
+return false;
+  }
+
+  /**
* Determine if a datanode is good for placing block.
*
* @param node The target datanode
@@ -923,7 +941,7 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
* @param results A list containing currently chosen nodes. Used to check if
*too many nodes has been chosen in the target rack.
* @param avoidStaleNodes Whether or not to avoid choosing stale nodes
-   * @return Reture true if the datanode is good candidate, otherwise false
+   * @return Return true if the datanode is good candidate, otherwise false
*/
   boolean isGoodDatanode(DatanodeDescriptor node,
  int maxTargetPerRack, boolean considerLoad,
@@ -943,13 +961,8 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 }
 
 // check the communication traffic of the target machine
-if (considerLoad) {
-  final double maxLoad = considerLoadFactor *
-  stats.getInServiceXceiverAverage();
-  final int nodeLoad = node.getXceiverCount();
-  if (nodeLoad > maxLoad) {
-logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
-"(load: " + nodeLoad + " > " + maxLoad + ")");
+if(considerLoad){
+  if(excludeNodeByLoad(node)){
 return false;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 27dcbf1..f08fa13 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.server.namenode.Namesystem;
 import org.apache.hadoop.hdfs.server.namenode.TestINodeFile;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.net.Node;
+import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.apache.log4j.spi.LoggingEvent;
@@

hadoop git commit: HDFS-13833. Improve BlockPlacementPolicyDefault's consider load logic. Contributed by Shweta.

2018-09-18 Thread xiao 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 392965370 -> 27978bcb6


HDFS-13833. Improve BlockPlacementPolicyDefault's consider load logic. 
Contributed by Shweta.

Signed-off-by: Xiao Chen 


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27978bcb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27978bcb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27978bcb

Branch: refs/heads/trunk
Commit: 27978bcb66a9130cbf26d37ec454c0b7fcdc2530
Parents: 3929653
Author: Shweta 
Authored: Tue Sep 18 20:22:25 2018 -0700
Committer: Xiao Chen 
Committed: Tue Sep 18 20:23:50 2018 -0700

--
 .../BlockPlacementPolicyDefault.java| 29 ++--
 .../blockmanagement/TestReplicationPolicy.java  | 28 +++
 2 files changed, 49 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d00f961..d396845 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -913,6 +913,24 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
   }
 
   /**
+   * Determine if a datanode should be chosen based on current workload.
+   *
+   * @param node The target datanode
+   * @return Return true if the datanode should be excluded, otherwise false
+   */
+  boolean excludeNodeByLoad(DatanodeDescriptor node){
+final double maxLoad = considerLoadFactor *
+stats.getInServiceXceiverAverage();
+final int nodeLoad = node.getXceiverCount();
+if ((nodeLoad > maxLoad) && (maxLoad > 0)) {
+  logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
+  "(load: " + nodeLoad + " > " + maxLoad + ")");
+  return true;
+}
+return false;
+  }
+
+  /**
* Determine if a datanode is good for placing block.
*
* @param node The target datanode
@@ -923,7 +941,7 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
* @param results A list containing currently chosen nodes. Used to check if
*too many nodes has been chosen in the target rack.
* @param avoidStaleNodes Whether or not to avoid choosing stale nodes
-   * @return Reture true if the datanode is good candidate, otherwise false
+   * @return Return true if the datanode is good candidate, otherwise false
*/
   boolean isGoodDatanode(DatanodeDescriptor node,
  int maxTargetPerRack, boolean considerLoad,
@@ -943,13 +961,8 @@ public class BlockPlacementPolicyDefault extends 
BlockPlacementPolicy {
 }
 
 // check the communication traffic of the target machine
-if (considerLoad) {
-  final double maxLoad = considerLoadFactor *
-  stats.getInServiceXceiverAverage();
-  final int nodeLoad = node.getXceiverCount();
-  if (nodeLoad > maxLoad) {
-logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
-"(load: " + nodeLoad + " > " + maxLoad + ")");
+if(considerLoad){
+  if(excludeNodeByLoad(node)){
 return false;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 27dcbf1..f08fa13 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.server.namenode.Namesystem;
 import org.apache.hadoop.hdfs.server.namenode.TestINodeFile;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.net.Node;
+import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.log4j.Level;