subject:"hadoop git commit\: HDFS\-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang."

[09/50] [abbrv] hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

2016-11-22 Thread subru

HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source 
DataNode. Contributed by  Mark Wagner and Zhe Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49a09179
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49a09179
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49a09179

Branch: refs/heads/YARN-2915
Commit: 49a09179e3fadae090126261be0a7fe0aa48798e
Parents: f922067
Author: Kihwal Lee 
Authored: Mon Nov 21 10:13:24 2016 -0600
Committer: Kihwal Lee 
Committed: Mon Nov 21 10:13:24 2016 -0600

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  4 +++
 .../hadoop/hdfs/server/balancer/Balancer.java   |  5 ++-
 .../hadoop/hdfs/server/balancer/Dispatcher.java | 34 
 .../apache/hadoop/hdfs/server/mover/Mover.java  |  5 ++-
 .../src/main/resources/hdfs-default.xml | 20 
 .../hdfs/server/balancer/TestBalancer.java  |  2 ++
 6 files changed, 55 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a09179/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b1fb3f4..b9fd939 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -502,6 +502,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_BALANCER_KERBEROS_PRINCIPAL_KEY = 
"dfs.balancer.kerberos.principal";
   public static final String  DFS_BALANCER_BLOCK_MOVE_TIMEOUT = 
"dfs.balancer.block-move.timeout";
   public static final int DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT = 0;
+  public static final String  DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.balancer.max-no-move-interval";
+  public static final intDFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT = 
60*1000; // One minute
 
 
   public static final String  DFS_MOVER_MOVEDWINWIDTH_KEY = 
"dfs.mover.movedWinWidth";
@@ -519,6 +521,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   "dfs.mover.keytab.file";
   public static final String  DFS_MOVER_KERBEROS_PRINCIPAL_KEY =
   "dfs.mover.kerberos.principal";
+  public static final String  DFS_MOVER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.mover.max-no-move-interval";
+  public static final intDFS_MOVER_MAX_NO_MOVE_INTERVAL_DEFAULT = 60*1000; 
// One minute
 
   public static final String  DFS_DATANODE_ADDRESS_KEY = 
"dfs.datanode.address";
   public static final int DFS_DATANODE_DEFAULT_PORT = 9866;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a09179/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 583ade3..61352f3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -285,13 +285,16 @@ public class Balancer {
 final int blockMoveTimeout = conf.getInt(
 DFSConfigKeys.DFS_BALANCER_BLOCK_MOVE_TIMEOUT,
 DFSConfigKeys.DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT);
+final int maxNoMoveInterval = conf.getInt(
+DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY,
+DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT);
 
 this.nnc = theblockpool;
 this.dispatcher =
 new Dispatcher(theblockpool, p.getIncludedNodes(),
 p.getExcludedNodes(), movedWinWidth, moverThreads,
 dispatcherThreads, maxConcurrentMovesPerNode, getBlocksSize,
-getBlocksMinBlockSize, blockMoveTimeout, conf);
+getBlocksMinBlockSize, blockMoveTimeout, maxNoMoveInterval, conf);
 this.threshold = p.getThreshold();
 this.policy = p.getBalancingPolicy();
 this.sourceNodes = p.getSourceNodes();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a09179/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
--
diff --git

[37/50] [abbrv] hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

2016-11-22 Thread liuml07

HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source 
DataNode. Contributed by  Mark Wagner and Zhe Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49a09179
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49a09179
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49a09179

Branch: refs/heads/HADOOP-13345
Commit: 49a09179e3fadae090126261be0a7fe0aa48798e
Parents: f922067
Author: Kihwal Lee 
Authored: Mon Nov 21 10:13:24 2016 -0600
Committer: Kihwal Lee 
Committed: Mon Nov 21 10:13:24 2016 -0600

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  4 +++
 .../hadoop/hdfs/server/balancer/Balancer.java   |  5 ++-
 .../hadoop/hdfs/server/balancer/Dispatcher.java | 34 
 .../apache/hadoop/hdfs/server/mover/Mover.java  |  5 ++-
 .../src/main/resources/hdfs-default.xml | 20 
 .../hdfs/server/balancer/TestBalancer.java  |  2 ++
 6 files changed, 55 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a09179/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b1fb3f4..b9fd939 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -502,6 +502,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_BALANCER_KERBEROS_PRINCIPAL_KEY = 
"dfs.balancer.kerberos.principal";
   public static final String  DFS_BALANCER_BLOCK_MOVE_TIMEOUT = 
"dfs.balancer.block-move.timeout";
   public static final int DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT = 0;
+  public static final String  DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.balancer.max-no-move-interval";
+  public static final intDFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT = 
60*1000; // One minute
 
 
   public static final String  DFS_MOVER_MOVEDWINWIDTH_KEY = 
"dfs.mover.movedWinWidth";
@@ -519,6 +521,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   "dfs.mover.keytab.file";
   public static final String  DFS_MOVER_KERBEROS_PRINCIPAL_KEY =
   "dfs.mover.kerberos.principal";
+  public static final String  DFS_MOVER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.mover.max-no-move-interval";
+  public static final intDFS_MOVER_MAX_NO_MOVE_INTERVAL_DEFAULT = 60*1000; 
// One minute
 
   public static final String  DFS_DATANODE_ADDRESS_KEY = 
"dfs.datanode.address";
   public static final int DFS_DATANODE_DEFAULT_PORT = 9866;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a09179/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 583ade3..61352f3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -285,13 +285,16 @@ public class Balancer {
 final int blockMoveTimeout = conf.getInt(
 DFSConfigKeys.DFS_BALANCER_BLOCK_MOVE_TIMEOUT,
 DFSConfigKeys.DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT);
+final int maxNoMoveInterval = conf.getInt(
+DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY,
+DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT);
 
 this.nnc = theblockpool;
 this.dispatcher =
 new Dispatcher(theblockpool, p.getIncludedNodes(),
 p.getExcludedNodes(), movedWinWidth, moverThreads,
 dispatcherThreads, maxConcurrentMovesPerNode, getBlocksSize,
-getBlocksMinBlockSize, blockMoveTimeout, conf);
+getBlocksMinBlockSize, blockMoveTimeout, maxNoMoveInterval, conf);
 this.threshold = p.getThreshold();
 this.policy = p.getBalancingPolicy();
 this.sourceNodes = p.getSourceNodes();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a09179/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
--
diff --git

[40/50] [abbrv] hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

2016-11-22 Thread sjlee

HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source 
DataNode. Contributed by  Mark Wagner and Zhe Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49a09179
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49a09179
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49a09179

Branch: refs/heads/HADOOP-13070
Commit: 49a09179e3fadae090126261be0a7fe0aa48798e
Parents: f922067
Author: Kihwal Lee 
Authored: Mon Nov 21 10:13:24 2016 -0600
Committer: Kihwal Lee 
Committed: Mon Nov 21 10:13:24 2016 -0600

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  4 +++
 .../hadoop/hdfs/server/balancer/Balancer.java   |  5 ++-
 .../hadoop/hdfs/server/balancer/Dispatcher.java | 34 
 .../apache/hadoop/hdfs/server/mover/Mover.java  |  5 ++-
 .../src/main/resources/hdfs-default.xml | 20 
 .../hdfs/server/balancer/TestBalancer.java  |  2 ++
 6 files changed, 55 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a09179/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b1fb3f4..b9fd939 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -502,6 +502,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_BALANCER_KERBEROS_PRINCIPAL_KEY = 
"dfs.balancer.kerberos.principal";
   public static final String  DFS_BALANCER_BLOCK_MOVE_TIMEOUT = 
"dfs.balancer.block-move.timeout";
   public static final int DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT = 0;
+  public static final String  DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.balancer.max-no-move-interval";
+  public static final intDFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT = 
60*1000; // One minute
 
 
   public static final String  DFS_MOVER_MOVEDWINWIDTH_KEY = 
"dfs.mover.movedWinWidth";
@@ -519,6 +521,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   "dfs.mover.keytab.file";
   public static final String  DFS_MOVER_KERBEROS_PRINCIPAL_KEY =
   "dfs.mover.kerberos.principal";
+  public static final String  DFS_MOVER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.mover.max-no-move-interval";
+  public static final intDFS_MOVER_MAX_NO_MOVE_INTERVAL_DEFAULT = 60*1000; 
// One minute
 
   public static final String  DFS_DATANODE_ADDRESS_KEY = 
"dfs.datanode.address";
   public static final int DFS_DATANODE_DEFAULT_PORT = 9866;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a09179/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 583ade3..61352f3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -285,13 +285,16 @@ public class Balancer {
 final int blockMoveTimeout = conf.getInt(
 DFSConfigKeys.DFS_BALANCER_BLOCK_MOVE_TIMEOUT,
 DFSConfigKeys.DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT);
+final int maxNoMoveInterval = conf.getInt(
+DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY,
+DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT);
 
 this.nnc = theblockpool;
 this.dispatcher =
 new Dispatcher(theblockpool, p.getIncludedNodes(),
 p.getExcludedNodes(), movedWinWidth, moverThreads,
 dispatcherThreads, maxConcurrentMovesPerNode, getBlocksSize,
-getBlocksMinBlockSize, blockMoveTimeout, conf);
+getBlocksMinBlockSize, blockMoveTimeout, maxNoMoveInterval, conf);
 this.threshold = p.getThreshold();
 this.policy = p.getBalancingPolicy();
 this.sourceNodes = p.getSourceNodes();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a09179/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
--
diff --git

hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

2016-11-21 Thread kihwal

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 0da0bf0de -> 01ad9dee0


HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source 
DataNode. Contributed by Mark Wagner and Zhe Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01ad9dee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01ad9dee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01ad9dee

Branch: refs/heads/branch-2.7
Commit: 01ad9dee087cab5d1c472d4896c22f7de5098a9e
Parents: 0da0bf0
Author: Kihwal Lee 
Authored: Mon Nov 21 16:58:12 2016 -0600
Committer: Kihwal Lee 
Committed: Mon Nov 21 16:58:12 2016 -0600

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  4 +++
 .../hadoop/hdfs/server/balancer/Balancer.java   |  5 ++-
 .../hadoop/hdfs/server/balancer/Dispatcher.java | 34 
 .../apache/hadoop/hdfs/server/mover/Mover.java  |  5 ++-
 .../src/main/resources/hdfs-default.xml | 20 
 .../hdfs/server/balancer/TestBalancer.java  |  2 ++
 7 files changed, 58 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/01ad9dee/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 99113a4..6bbd0d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -64,6 +64,9 @@ Release 2.7.4 - UNRELEASED
 HDFS-10872. Add MutableRate metrics for FSNamesystemLock operations.
 (Erik Krogen via zhz)
 
+HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source
+DataNode (Mark Wagner and zhz via kihwal)
+
   OPTIMIZATIONS
 
 HDFS-10896. Move lock logging logic from FSNamesystem into 
FSNamesystemLock.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/01ad9dee/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index dc5ad0a..83f882b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -460,6 +460,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final longDFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_DEFAULT = 
10L*1024*1024; // 10MB
   public static final String  DFS_BALANCER_BLOCK_MOVE_TIMEOUT = 
"dfs.balancer.block-move.timeout";
   public static final int DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT = 0;
+  public static final String  DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.balancer.max-no-move-interval";
+  public static final intDFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT = 
60*1000; // One minute
 
   public static final String  DFS_MOVER_MOVEDWINWIDTH_KEY = 
"dfs.mover.movedWinWidth";
   public static final longDFS_MOVER_MOVEDWINWIDTH_DEFAULT = 5400*1000L;
@@ -467,6 +469,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final int DFS_MOVER_MOVERTHREADS_DEFAULT = 1000;
   public static final String  DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY = 
"dfs.mover.retry.max.attempts";
   public static final int DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT = 10;
+  public static final String  DFS_MOVER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.mover.max-no-move-interval";
+  public static final intDFS_MOVER_MAX_NO_MOVE_INTERVAL_DEFAULT = 60*1000; 
// One minute
 
   public static final String  DFS_DATANODE_ADDRESS_KEY = 
"dfs.datanode.address";
   public static final int DFS_DATANODE_DEFAULT_PORT = 50010;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/01ad9dee/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 2f6aba8..4fb6f0e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -258,11 +258,14 @@ public class Balancer {
 final int blockMoveTimeout = conf.getInt(

hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

2016-11-21 Thread kihwal

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 74503ed6b -> f5410f081


HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source 
DataNode. Contributed by Mark Wagner and Zhe Zhang.

(cherry picked from commit 774b0cd845a67bc0e2ef99b415ce1e5432769551)

Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5410f08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5410f08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5410f08

Branch: refs/heads/branch-2.8
Commit: f5410f08140850c7ad27558f79a13733f3ff1034
Parents: 74503ed
Author: Kihwal Lee 
Authored: Mon Nov 21 10:52:22 2016 -0600
Committer: Kihwal Lee 
Committed: Mon Nov 21 10:52:42 2016 -0600

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  4 +++
 .../hadoop/hdfs/server/balancer/Balancer.java   |  5 ++-
 .../hadoop/hdfs/server/balancer/Dispatcher.java | 34 
 .../apache/hadoop/hdfs/server/mover/Mover.java  |  5 ++-
 .../src/main/resources/hdfs-default.xml | 20 
 .../hdfs/server/balancer/TestBalancer.java  |  2 ++
 6 files changed, 55 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5410f08/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index c289f50..12573c5 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -469,6 +469,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_BALANCER_KERBEROS_PRINCIPAL_KEY = 
"dfs.balancer.kerberos.principal";
   public static final String  DFS_BALANCER_BLOCK_MOVE_TIMEOUT = 
"dfs.balancer.block-move.timeout";
   public static final int DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT = 0;
+  public static final String  DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.balancer.max-no-move-interval";
+  public static final intDFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT = 
60*1000; // One minute
 
 
   public static final String  DFS_MOVER_MOVEDWINWIDTH_KEY = 
"dfs.mover.movedWinWidth";
@@ -477,6 +479,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final int DFS_MOVER_MOVERTHREADS_DEFAULT = 1000;
   public static final String  DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY = 
"dfs.mover.retry.max.attempts";
   public static final int DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT = 10;
+  public static final String  DFS_MOVER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.mover.max-no-move-interval";
+  public static final intDFS_MOVER_MAX_NO_MOVE_INTERVAL_DEFAULT = 60*1000; 
// One minute
 
   public static final String  DFS_DATANODE_ADDRESS_KEY = 
"dfs.datanode.address";
   public static final int DFS_DATANODE_DEFAULT_PORT = 50010;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5410f08/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 68c27b6..94f84a5 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -283,13 +283,16 @@ public class Balancer {
 final int blockMoveTimeout = conf.getInt(
 DFSConfigKeys.DFS_BALANCER_BLOCK_MOVE_TIMEOUT,
 DFSConfigKeys.DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT);
+final int maxNoMoveInterval = conf.getInt(
+DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY,
+DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT);
 
 this.nnc = theblockpool;
 this.dispatcher =
 new Dispatcher(theblockpool, p.getIncludedNodes(),
 p.getExcludedNodes(), movedWinWidth, moverThreads,
 dispatcherThreads, maxConcurrentMovesPerNode, getBlocksSize,
-getBlocksMinBlockSize, blockMoveTimeout, conf);
+getBlocksMinBlockSize, blockMoveTimeout, maxNoMoveInterval, conf);
 this.threshold = p.getThreshold();
 this.policy = p.getBalancingPolicy();

hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

2016-11-21 Thread kihwal

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 38024eb8f -> 774b0cd84


HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source 
DataNode. Contributed by Mark Wagner and Zhe Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/774b0cd8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/774b0cd8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/774b0cd8

Branch: refs/heads/branch-2
Commit: 774b0cd845a67bc0e2ef99b415ce1e5432769551
Parents: 38024eb
Author: Kihwal Lee 
Authored: Mon Nov 21 10:38:03 2016 -0600
Committer: Kihwal Lee 
Committed: Mon Nov 21 10:39:04 2016 -0600

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  4 +++
 .../hadoop/hdfs/server/balancer/Balancer.java   |  5 ++-
 .../hadoop/hdfs/server/balancer/Dispatcher.java | 34 
 .../apache/hadoop/hdfs/server/mover/Mover.java  |  5 ++-
 .../src/main/resources/hdfs-default.xml | 20 
 .../hdfs/server/balancer/TestBalancer.java  |  2 ++
 6 files changed, 55 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/774b0cd8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index c1e6de6..a3b9712 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -478,6 +478,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_BALANCER_KERBEROS_PRINCIPAL_KEY = 
"dfs.balancer.kerberos.principal";
   public static final String  DFS_BALANCER_BLOCK_MOVE_TIMEOUT = 
"dfs.balancer.block-move.timeout";
   public static final int DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT = 0;
+  public static final String  DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.balancer.max-no-move-interval";
+  public static final intDFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT = 
60*1000; // One minute
 
 
   public static final String  DFS_MOVER_MOVEDWINWIDTH_KEY = 
"dfs.mover.movedWinWidth";
@@ -486,6 +488,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final int DFS_MOVER_MOVERTHREADS_DEFAULT = 1000;
   public static final String  DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY = 
"dfs.mover.retry.max.attempts";
   public static final int DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT = 10;
+  public static final String  DFS_MOVER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.mover.max-no-move-interval";
+  public static final intDFS_MOVER_MAX_NO_MOVE_INTERVAL_DEFAULT = 60*1000; 
// One minute
 
   public static final String  DFS_DATANODE_ADDRESS_KEY = 
"dfs.datanode.address";
   public static final int DFS_DATANODE_DEFAULT_PORT = 50010;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/774b0cd8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 68c27b6..94f84a5 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -283,13 +283,16 @@ public class Balancer {
 final int blockMoveTimeout = conf.getInt(
 DFSConfigKeys.DFS_BALANCER_BLOCK_MOVE_TIMEOUT,
 DFSConfigKeys.DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT);
+final int maxNoMoveInterval = conf.getInt(
+DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY,
+DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT);
 
 this.nnc = theblockpool;
 this.dispatcher =
 new Dispatcher(theblockpool, p.getIncludedNodes(),
 p.getExcludedNodes(), movedWinWidth, moverThreads,
 dispatcherThreads, maxConcurrentMovesPerNode, getBlocksSize,
-getBlocksMinBlockSize, blockMoveTimeout, conf);
+getBlocksMinBlockSize, blockMoveTimeout, maxNoMoveInterval, conf);
 this.threshold = p.getThreshold();
 this.policy = p.getBalancingPolicy();
 this.sourceNodes = p.getSourceNodes();

hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

2016-11-21 Thread kihwal

Repository: hadoop
Updated Branches:
  refs/heads/trunk f92206774 -> 49a09179e


HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source 
DataNode. Contributed by  Mark Wagner and Zhe Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49a09179
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49a09179
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49a09179

Branch: refs/heads/trunk
Commit: 49a09179e3fadae090126261be0a7fe0aa48798e
Parents: f922067
Author: Kihwal Lee 
Authored: Mon Nov 21 10:13:24 2016 -0600
Committer: Kihwal Lee 
Committed: Mon Nov 21 10:13:24 2016 -0600

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  4 +++
 .../hadoop/hdfs/server/balancer/Balancer.java   |  5 ++-
 .../hadoop/hdfs/server/balancer/Dispatcher.java | 34 
 .../apache/hadoop/hdfs/server/mover/Mover.java  |  5 ++-
 .../src/main/resources/hdfs-default.xml | 20 
 .../hdfs/server/balancer/TestBalancer.java  |  2 ++
 6 files changed, 55 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a09179/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b1fb3f4..b9fd939 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -502,6 +502,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_BALANCER_KERBEROS_PRINCIPAL_KEY = 
"dfs.balancer.kerberos.principal";
   public static final String  DFS_BALANCER_BLOCK_MOVE_TIMEOUT = 
"dfs.balancer.block-move.timeout";
   public static final int DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT = 0;
+  public static final String  DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.balancer.max-no-move-interval";
+  public static final intDFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT = 
60*1000; // One minute
 
 
   public static final String  DFS_MOVER_MOVEDWINWIDTH_KEY = 
"dfs.mover.movedWinWidth";
@@ -519,6 +521,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   "dfs.mover.keytab.file";
   public static final String  DFS_MOVER_KERBEROS_PRINCIPAL_KEY =
   "dfs.mover.kerberos.principal";
+  public static final String  DFS_MOVER_MAX_NO_MOVE_INTERVAL_KEY = 
"dfs.mover.max-no-move-interval";
+  public static final intDFS_MOVER_MAX_NO_MOVE_INTERVAL_DEFAULT = 60*1000; 
// One minute
 
   public static final String  DFS_DATANODE_ADDRESS_KEY = 
"dfs.datanode.address";
   public static final int DFS_DATANODE_DEFAULT_PORT = 9866;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a09179/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 583ade3..61352f3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -285,13 +285,16 @@ public class Balancer {
 final int blockMoveTimeout = conf.getInt(
 DFSConfigKeys.DFS_BALANCER_BLOCK_MOVE_TIMEOUT,
 DFSConfigKeys.DFS_BALANCER_BLOCK_MOVE_TIMEOUT_DEFAULT);
+final int maxNoMoveInterval = conf.getInt(
+DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_KEY,
+DFSConfigKeys.DFS_BALANCER_MAX_NO_MOVE_INTERVAL_DEFAULT);
 
 this.nnc = theblockpool;
 this.dispatcher =
 new Dispatcher(theblockpool, p.getIncludedNodes(),
 p.getExcludedNodes(), movedWinWidth, moverThreads,
 dispatcherThreads, maxConcurrentMovesPerNode, getBlocksSize,
-getBlocksMinBlockSize, blockMoveTimeout, conf);
+getBlocksMinBlockSize, blockMoveTimeout, maxNoMoveInterval, conf);
 this.threshold = p.getThreshold();
 this.policy = p.getBalancingPolicy();
 this.sourceNodes = p.getSourceNodes();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a09179/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
--

[09/50] [abbrv] hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

[37/50] [abbrv] hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

[40/50] [abbrv] hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

hadoop git commit: HDFS-10966. Enhance Dispatcher logic on deciding when to give up a source DataNode. Contributed by Mark Wagner and Zhe Zhang.

7 matches

Site Navigation

Mail list logo

Footer information