hadoop git commit: HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz Wo Nicholas Sze.
Repository: hadoop
Updated Branches:
refs/heads/trunk a3839a9fb -> ae71a671a
HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz
Wo Nicholas Sze.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae71a671
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae71a671
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae71a671
Branch: refs/heads/trunk
Commit: ae71a671a3b4b454aa393c2974b6f1f16dd61405
Parents: a3839a9
Author: Jing Zhao
Authored: Thu Nov 6 17:48:36 2014 -0800
Committer: Jing Zhao
Committed: Thu Nov 6 17:48:36 2014 -0800
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/server/balancer/Balancer.java | 81 ---
.../hadoop/hdfs/server/balancer/Dispatcher.java | 13 ++-
.../hdfs/server/balancer/NameNodeConnector.java | 6 ++
.../hdfs/server/balancer/TestBalancer.java | 84 ++--
5 files changed, 142 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae71a671/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1a41251..5a7dac0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1042,6 +1042,9 @@ Release 2.6.0 - UNRELEASED
file descriptors when SASL is enabled on DataTransferProtocol.
(Chris Nauroth via wheat9)
+HDFS-7364. Balancer always shows zero Bytes Already Moved.
+(Tsz Wo Nicholas Sze via jing9)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae71a671/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 2a19537..dba1e2d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -27,7 +27,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
-import java.util.Formatter;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -162,7 +161,7 @@ import com.google.common.base.Preconditions;
public class Balancer {
static final Log LOG = LogFactory.getLog(Balancer.class);
- private static final Path BALANCER_ID_PATH = new Path("/system/balancer.id");
+ static final Path BALANCER_ID_PATH = new Path("/system/balancer.id");
private static final long GB = 1L << 30; //1GB
private static final long MAX_SIZE_TO_MOVE = 10*GB;
@@ -459,7 +458,7 @@ public class Balancer {
}
/* reset all fields in a balancer preparing for the next iteration */
- private void resetData(Configuration conf) {
+ void resetData(Configuration conf) {
this.overUtilized.clear();
this.aboveAvgUtilized.clear();
this.belowAvgUtilized.clear();
@@ -467,16 +466,47 @@ public class Balancer {
this.policy.reset();
dispatcher.reset(conf);;
}
-
+
+ static class Result {
+final ExitStatus exitStatus;
+final long bytesLeftToMove;
+final long bytesBeingMoved;
+final long bytesAlreadyMoved;
+
+Result(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved,
+long bytesAlreadyMoved) {
+ this.exitStatus = exitStatus;
+ this.bytesLeftToMove = bytesLeftToMove;
+ this.bytesBeingMoved = bytesBeingMoved;
+ this.bytesAlreadyMoved = bytesAlreadyMoved;
+}
+
+void print(int iteration, PrintStream out) {
+ out.printf("%-24s %10d %19s %18s %17s%n",
+ DateFormat.getDateTimeInstance().format(new Date()), iteration,
+ StringUtils.byteDesc(bytesAlreadyMoved),
+ StringUtils.byteDesc(bytesLeftToMove),
+ StringUtils.byteDesc(bytesBeingMoved));
+}
+ }
+
+ Result newResult(ExitStatus exitStatus, long bytesLeftToMove, long
bytesBeingMoved) {
+return new Result(exitStatus, bytesLeftToMove, bytesBeingMoved,
+dispatcher.getBytesMoved());
+ }
+
+ Result newResult(ExitStatus exitStatus) {
+return new Result(exitStatus, -1, -1, dispatcher.getBytesMoved());
+ }
+
/** Run an iteration for all datanodes. */
- private ExitStatus run(int iteration, Formatter formatter,
-
hadoop git commit: HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz Wo Nicholas Sze.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 52e57a95d -> 10fe37522
HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz
Wo Nicholas Sze.
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10fe3752
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10fe3752
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10fe3752
Branch: refs/heads/branch-2
Commit: 10fe37522c943274d14263fc2f67c0720c26b2ad
Parents: 52e57a9
Author: Jing Zhao
Authored: Thu Nov 6 17:48:36 2014 -0800
Committer: Jing Zhao
Committed: Thu Nov 6 17:55:49 2014 -0800
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/server/balancer/Balancer.java | 81 ---
.../hadoop/hdfs/server/balancer/Dispatcher.java | 13 ++-
.../hdfs/server/balancer/NameNodeConnector.java | 7 ++
.../hdfs/server/balancer/TestBalancer.java | 84 ++--
5 files changed, 143 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10fe3752/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 29211b9..7676030 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -897,6 +897,9 @@ Release 2.6.0 - UNRELEASED
file descriptors when SASL is enabled on DataTransferProtocol.
(Chris Nauroth via wheat9)
+HDFS-7364. Balancer always shows zero Bytes Already Moved.
+(Tsz Wo Nicholas Sze via jing9)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10fe3752/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 87348b3..eeac6ee 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -27,7 +27,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
-import java.util.Formatter;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -163,7 +162,7 @@ import com.google.common.base.Preconditions;
public class Balancer {
static final Log LOG = LogFactory.getLog(Balancer.class);
- private static final Path BALANCER_ID_PATH = new Path("/system/balancer.id");
+ static final Path BALANCER_ID_PATH = new Path("/system/balancer.id");
private static final long GB = 1L << 30; //1GB
private static final long MAX_SIZE_TO_MOVE = 10*GB;
@@ -460,7 +459,7 @@ public class Balancer {
}
/* reset all fields in a balancer preparing for the next iteration */
- private void resetData(Configuration conf) {
+ void resetData(Configuration conf) {
this.overUtilized.clear();
this.aboveAvgUtilized.clear();
this.belowAvgUtilized.clear();
@@ -468,16 +467,47 @@ public class Balancer {
this.policy.reset();
dispatcher.reset(conf);;
}
-
+
+ static class Result {
+final ExitStatus exitStatus;
+final long bytesLeftToMove;
+final long bytesBeingMoved;
+final long bytesAlreadyMoved;
+
+Result(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved,
+long bytesAlreadyMoved) {
+ this.exitStatus = exitStatus;
+ this.bytesLeftToMove = bytesLeftToMove;
+ this.bytesBeingMoved = bytesBeingMoved;
+ this.bytesAlreadyMoved = bytesAlreadyMoved;
+}
+
+void print(int iteration, PrintStream out) {
+ out.printf("%-24s %10d %19s %18s %17s%n",
+ DateFormat.getDateTimeInstance().format(new Date()), iteration,
+ StringUtils.byteDesc(bytesAlreadyMoved),
+ StringUtils.byteDesc(bytesLeftToMove),
+ StringUtils.byteDesc(bytesBeingMoved));
+}
+ }
+
+ Result newResult(ExitStatus exitStatus, long bytesLeftToMove, long
bytesBeingMoved) {
+return new Result(exitStatus, bytesLeftToMove, bytesBeingMoved,
+dispatcher.getBytesMoved());
+ }
+
+ Result newResult(ExitStatus exitStatus) {
+return new Result(exitStatus, -1, -1, dispatcher.getB
hadoop git commit: HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz Wo Nicholas Sze.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 21ef5afaf -> a9b043613
HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz
Wo Nicholas Sze.
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9b04361
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9b04361
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9b04361
Branch: refs/heads/branch-2.6
Commit: a9b04361329c93f09eb7616fc92b4e8140716f15
Parents: 21ef5af
Author: Jing Zhao
Authored: Thu Nov 6 17:48:36 2014 -0800
Committer: Jing Zhao
Committed: Thu Nov 6 17:56:15 2014 -0800
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/server/balancer/Balancer.java | 81 ---
.../hadoop/hdfs/server/balancer/Dispatcher.java | 13 ++-
.../hdfs/server/balancer/NameNodeConnector.java | 7 ++
.../hdfs/server/balancer/TestBalancer.java | 84 ++--
5 files changed, 143 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9b04361/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 44b99ca..87825b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -733,6 +733,9 @@ Release 2.6.0 - UNRELEASED
file descriptors when SASL is enabled on DataTransferProtocol.
(Chris Nauroth via wheat9)
+HDFS-7364. Balancer always shows zero Bytes Already Moved.
+(Tsz Wo Nicholas Sze via jing9)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9b04361/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 87348b3..eeac6ee 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -27,7 +27,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
-import java.util.Formatter;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -163,7 +162,7 @@ import com.google.common.base.Preconditions;
public class Balancer {
static final Log LOG = LogFactory.getLog(Balancer.class);
- private static final Path BALANCER_ID_PATH = new Path("/system/balancer.id");
+ static final Path BALANCER_ID_PATH = new Path("/system/balancer.id");
private static final long GB = 1L << 30; //1GB
private static final long MAX_SIZE_TO_MOVE = 10*GB;
@@ -460,7 +459,7 @@ public class Balancer {
}
/* reset all fields in a balancer preparing for the next iteration */
- private void resetData(Configuration conf) {
+ void resetData(Configuration conf) {
this.overUtilized.clear();
this.aboveAvgUtilized.clear();
this.belowAvgUtilized.clear();
@@ -468,16 +467,47 @@ public class Balancer {
this.policy.reset();
dispatcher.reset(conf);;
}
-
+
+ static class Result {
+final ExitStatus exitStatus;
+final long bytesLeftToMove;
+final long bytesBeingMoved;
+final long bytesAlreadyMoved;
+
+Result(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved,
+long bytesAlreadyMoved) {
+ this.exitStatus = exitStatus;
+ this.bytesLeftToMove = bytesLeftToMove;
+ this.bytesBeingMoved = bytesBeingMoved;
+ this.bytesAlreadyMoved = bytesAlreadyMoved;
+}
+
+void print(int iteration, PrintStream out) {
+ out.printf("%-24s %10d %19s %18s %17s%n",
+ DateFormat.getDateTimeInstance().format(new Date()), iteration,
+ StringUtils.byteDesc(bytesAlreadyMoved),
+ StringUtils.byteDesc(bytesLeftToMove),
+ StringUtils.byteDesc(bytesBeingMoved));
+}
+ }
+
+ Result newResult(ExitStatus exitStatus, long bytesLeftToMove, long
bytesBeingMoved) {
+return new Result(exitStatus, bytesLeftToMove, bytesBeingMoved,
+dispatcher.getBytesMoved());
+ }
+
+ Result newResult(ExitStatus exitStatus) {
+return new Result(exitStatus, -1, -1, dispatcher.
[08/30] hadoop git commit: HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz Wo Nicholas Sze.
HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz
Wo Nicholas Sze.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae71a671
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae71a671
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae71a671
Branch: refs/heads/HDFS-EC
Commit: ae71a671a3b4b454aa393c2974b6f1f16dd61405
Parents: a3839a9
Author: Jing Zhao
Authored: Thu Nov 6 17:48:36 2014 -0800
Committer: Jing Zhao
Committed: Thu Nov 6 17:48:36 2014 -0800
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/server/balancer/Balancer.java | 81 ---
.../hadoop/hdfs/server/balancer/Dispatcher.java | 13 ++-
.../hdfs/server/balancer/NameNodeConnector.java | 6 ++
.../hdfs/server/balancer/TestBalancer.java | 84 ++--
5 files changed, 142 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae71a671/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1a41251..5a7dac0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1042,6 +1042,9 @@ Release 2.6.0 - UNRELEASED
file descriptors when SASL is enabled on DataTransferProtocol.
(Chris Nauroth via wheat9)
+HDFS-7364. Balancer always shows zero Bytes Already Moved.
+(Tsz Wo Nicholas Sze via jing9)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae71a671/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 2a19537..dba1e2d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -27,7 +27,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
-import java.util.Formatter;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -162,7 +161,7 @@ import com.google.common.base.Preconditions;
public class Balancer {
static final Log LOG = LogFactory.getLog(Balancer.class);
- private static final Path BALANCER_ID_PATH = new Path("/system/balancer.id");
+ static final Path BALANCER_ID_PATH = new Path("/system/balancer.id");
private static final long GB = 1L << 30; //1GB
private static final long MAX_SIZE_TO_MOVE = 10*GB;
@@ -459,7 +458,7 @@ public class Balancer {
}
/* reset all fields in a balancer preparing for the next iteration */
- private void resetData(Configuration conf) {
+ void resetData(Configuration conf) {
this.overUtilized.clear();
this.aboveAvgUtilized.clear();
this.belowAvgUtilized.clear();
@@ -467,16 +466,47 @@ public class Balancer {
this.policy.reset();
dispatcher.reset(conf);;
}
-
+
+ static class Result {
+final ExitStatus exitStatus;
+final long bytesLeftToMove;
+final long bytesBeingMoved;
+final long bytesAlreadyMoved;
+
+Result(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved,
+long bytesAlreadyMoved) {
+ this.exitStatus = exitStatus;
+ this.bytesLeftToMove = bytesLeftToMove;
+ this.bytesBeingMoved = bytesBeingMoved;
+ this.bytesAlreadyMoved = bytesAlreadyMoved;
+}
+
+void print(int iteration, PrintStream out) {
+ out.printf("%-24s %10d %19s %18s %17s%n",
+ DateFormat.getDateTimeInstance().format(new Date()), iteration,
+ StringUtils.byteDesc(bytesAlreadyMoved),
+ StringUtils.byteDesc(bytesLeftToMove),
+ StringUtils.byteDesc(bytesBeingMoved));
+}
+ }
+
+ Result newResult(ExitStatus exitStatus, long bytesLeftToMove, long
bytesBeingMoved) {
+return new Result(exitStatus, bytesLeftToMove, bytesBeingMoved,
+dispatcher.getBytesMoved());
+ }
+
+ Result newResult(ExitStatus exitStatus) {
+return new Result(exitStatus, -1, -1, dispatcher.getBytesMoved());
+ }
+
/** Run an iteration for all datanodes. */
- private ExitStatus run(int iteration, Formatter formatter,
- Configuration conf) {
+ Result runOneIteration() {
try {
final
