hadoop git commit: HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz Wo Nicholas Sze.
Repository: hadoop Updated Branches: refs/heads/trunk a3839a9fb -> ae71a671a HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz Wo Nicholas Sze. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae71a671 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae71a671 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae71a671 Branch: refs/heads/trunk Commit: ae71a671a3b4b454aa393c2974b6f1f16dd61405 Parents: a3839a9 Author: Jing Zhao Authored: Thu Nov 6 17:48:36 2014 -0800 Committer: Jing Zhao Committed: Thu Nov 6 17:48:36 2014 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/hdfs/server/balancer/Balancer.java | 81 --- .../hadoop/hdfs/server/balancer/Dispatcher.java | 13 ++- .../hdfs/server/balancer/NameNodeConnector.java | 6 ++ .../hdfs/server/balancer/TestBalancer.java | 84 ++-- 5 files changed, 142 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae71a671/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1a41251..5a7dac0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1042,6 +1042,9 @@ Release 2.6.0 - UNRELEASED file descriptors when SASL is enabled on DataTransferProtocol. (Chris Nauroth via wheat9) +HDFS-7364. Balancer always shows zero Bytes Already Moved. +(Tsz Wo Nicholas Sze via jing9) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae71a671/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 2a19537..dba1e2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -27,7 +27,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; -import java.util.Formatter; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -162,7 +161,7 @@ import com.google.common.base.Preconditions; public class Balancer { static final Log LOG = LogFactory.getLog(Balancer.class); - private static final Path BALANCER_ID_PATH = new Path("/system/balancer.id"); + static final Path BALANCER_ID_PATH = new Path("/system/balancer.id"); private static final long GB = 1L << 30; //1GB private static final long MAX_SIZE_TO_MOVE = 10*GB; @@ -459,7 +458,7 @@ public class Balancer { } /* reset all fields in a balancer preparing for the next iteration */ - private void resetData(Configuration conf) { + void resetData(Configuration conf) { this.overUtilized.clear(); this.aboveAvgUtilized.clear(); this.belowAvgUtilized.clear(); @@ -467,16 +466,47 @@ public class Balancer { this.policy.reset(); dispatcher.reset(conf);; } - + + static class Result { +final ExitStatus exitStatus; +final long bytesLeftToMove; +final long bytesBeingMoved; +final long bytesAlreadyMoved; + +Result(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved, +long bytesAlreadyMoved) { + this.exitStatus = exitStatus; + this.bytesLeftToMove = bytesLeftToMove; + this.bytesBeingMoved = bytesBeingMoved; + this.bytesAlreadyMoved = bytesAlreadyMoved; +} + +void print(int iteration, PrintStream out) { + out.printf("%-24s %10d %19s %18s %17s%n", + DateFormat.getDateTimeInstance().format(new Date()), iteration, + StringUtils.byteDesc(bytesAlreadyMoved), + StringUtils.byteDesc(bytesLeftToMove), + StringUtils.byteDesc(bytesBeingMoved)); +} + } + + Result newResult(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved) { +return new Result(exitStatus, bytesLeftToMove, bytesBeingMoved, +dispatcher.getBytesMoved()); + } + + Result newResult(ExitStatus exitStatus) { +return new Result(exitStatus, -1, -1, dispatcher.getBytesMoved()); + } + /** Run an iteration for all datanodes. */ - private ExitStatus run(int iteration, Formatter formatter, -
hadoop git commit: HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz Wo Nicholas Sze.
Repository: hadoop Updated Branches: refs/heads/branch-2 52e57a95d -> 10fe37522 HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz Wo Nicholas Sze. Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10fe3752 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10fe3752 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10fe3752 Branch: refs/heads/branch-2 Commit: 10fe37522c943274d14263fc2f67c0720c26b2ad Parents: 52e57a9 Author: Jing Zhao Authored: Thu Nov 6 17:48:36 2014 -0800 Committer: Jing Zhao Committed: Thu Nov 6 17:55:49 2014 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/hdfs/server/balancer/Balancer.java | 81 --- .../hadoop/hdfs/server/balancer/Dispatcher.java | 13 ++- .../hdfs/server/balancer/NameNodeConnector.java | 7 ++ .../hdfs/server/balancer/TestBalancer.java | 84 ++-- 5 files changed, 143 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/10fe3752/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 29211b9..7676030 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -897,6 +897,9 @@ Release 2.6.0 - UNRELEASED file descriptors when SASL is enabled on DataTransferProtocol. (Chris Nauroth via wheat9) +HDFS-7364. Balancer always shows zero Bytes Already Moved. +(Tsz Wo Nicholas Sze via jing9) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an http://git-wip-us.apache.org/repos/asf/hadoop/blob/10fe3752/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 87348b3..eeac6ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -27,7 +27,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; -import java.util.Formatter; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -163,7 +162,7 @@ import com.google.common.base.Preconditions; public class Balancer { static final Log LOG = LogFactory.getLog(Balancer.class); - private static final Path BALANCER_ID_PATH = new Path("/system/balancer.id"); + static final Path BALANCER_ID_PATH = new Path("/system/balancer.id"); private static final long GB = 1L << 30; //1GB private static final long MAX_SIZE_TO_MOVE = 10*GB; @@ -460,7 +459,7 @@ public class Balancer { } /* reset all fields in a balancer preparing for the next iteration */ - private void resetData(Configuration conf) { + void resetData(Configuration conf) { this.overUtilized.clear(); this.aboveAvgUtilized.clear(); this.belowAvgUtilized.clear(); @@ -468,16 +467,47 @@ public class Balancer { this.policy.reset(); dispatcher.reset(conf);; } - + + static class Result { +final ExitStatus exitStatus; +final long bytesLeftToMove; +final long bytesBeingMoved; +final long bytesAlreadyMoved; + +Result(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved, +long bytesAlreadyMoved) { + this.exitStatus = exitStatus; + this.bytesLeftToMove = bytesLeftToMove; + this.bytesBeingMoved = bytesBeingMoved; + this.bytesAlreadyMoved = bytesAlreadyMoved; +} + +void print(int iteration, PrintStream out) { + out.printf("%-24s %10d %19s %18s %17s%n", + DateFormat.getDateTimeInstance().format(new Date()), iteration, + StringUtils.byteDesc(bytesAlreadyMoved), + StringUtils.byteDesc(bytesLeftToMove), + StringUtils.byteDesc(bytesBeingMoved)); +} + } + + Result newResult(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved) { +return new Result(exitStatus, bytesLeftToMove, bytesBeingMoved, +dispatcher.getBytesMoved()); + } + + Result newResult(ExitStatus exitStatus) { +return new Result(exitStatus, -1, -1, dispatcher.getB
hadoop git commit: HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz Wo Nicholas Sze.
Repository: hadoop Updated Branches: refs/heads/branch-2.6 21ef5afaf -> a9b043613 HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz Wo Nicholas Sze. Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9b04361 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9b04361 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9b04361 Branch: refs/heads/branch-2.6 Commit: a9b04361329c93f09eb7616fc92b4e8140716f15 Parents: 21ef5af Author: Jing Zhao Authored: Thu Nov 6 17:48:36 2014 -0800 Committer: Jing Zhao Committed: Thu Nov 6 17:56:15 2014 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/hdfs/server/balancer/Balancer.java | 81 --- .../hadoop/hdfs/server/balancer/Dispatcher.java | 13 ++- .../hdfs/server/balancer/NameNodeConnector.java | 7 ++ .../hdfs/server/balancer/TestBalancer.java | 84 ++-- 5 files changed, 143 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9b04361/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 44b99ca..87825b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -733,6 +733,9 @@ Release 2.6.0 - UNRELEASED file descriptors when SASL is enabled on DataTransferProtocol. (Chris Nauroth via wheat9) +HDFS-7364. Balancer always shows zero Bytes Already Moved. +(Tsz Wo Nicholas Sze via jing9) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9b04361/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 87348b3..eeac6ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -27,7 +27,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; -import java.util.Formatter; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -163,7 +162,7 @@ import com.google.common.base.Preconditions; public class Balancer { static final Log LOG = LogFactory.getLog(Balancer.class); - private static final Path BALANCER_ID_PATH = new Path("/system/balancer.id"); + static final Path BALANCER_ID_PATH = new Path("/system/balancer.id"); private static final long GB = 1L << 30; //1GB private static final long MAX_SIZE_TO_MOVE = 10*GB; @@ -460,7 +459,7 @@ public class Balancer { } /* reset all fields in a balancer preparing for the next iteration */ - private void resetData(Configuration conf) { + void resetData(Configuration conf) { this.overUtilized.clear(); this.aboveAvgUtilized.clear(); this.belowAvgUtilized.clear(); @@ -468,16 +467,47 @@ public class Balancer { this.policy.reset(); dispatcher.reset(conf);; } - + + static class Result { +final ExitStatus exitStatus; +final long bytesLeftToMove; +final long bytesBeingMoved; +final long bytesAlreadyMoved; + +Result(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved, +long bytesAlreadyMoved) { + this.exitStatus = exitStatus; + this.bytesLeftToMove = bytesLeftToMove; + this.bytesBeingMoved = bytesBeingMoved; + this.bytesAlreadyMoved = bytesAlreadyMoved; +} + +void print(int iteration, PrintStream out) { + out.printf("%-24s %10d %19s %18s %17s%n", + DateFormat.getDateTimeInstance().format(new Date()), iteration, + StringUtils.byteDesc(bytesAlreadyMoved), + StringUtils.byteDesc(bytesLeftToMove), + StringUtils.byteDesc(bytesBeingMoved)); +} + } + + Result newResult(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved) { +return new Result(exitStatus, bytesLeftToMove, bytesBeingMoved, +dispatcher.getBytesMoved()); + } + + Result newResult(ExitStatus exitStatus) { +return new Result(exitStatus, -1, -1, dispatcher.
[08/30] hadoop git commit: HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz Wo Nicholas Sze.
HDFS-7364. Balancer always shows zero Bytes Already Moved. Contributed by Tsz Wo Nicholas Sze. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae71a671 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae71a671 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae71a671 Branch: refs/heads/HDFS-EC Commit: ae71a671a3b4b454aa393c2974b6f1f16dd61405 Parents: a3839a9 Author: Jing Zhao Authored: Thu Nov 6 17:48:36 2014 -0800 Committer: Jing Zhao Committed: Thu Nov 6 17:48:36 2014 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/hdfs/server/balancer/Balancer.java | 81 --- .../hadoop/hdfs/server/balancer/Dispatcher.java | 13 ++- .../hdfs/server/balancer/NameNodeConnector.java | 6 ++ .../hdfs/server/balancer/TestBalancer.java | 84 ++-- 5 files changed, 142 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae71a671/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1a41251..5a7dac0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1042,6 +1042,9 @@ Release 2.6.0 - UNRELEASED file descriptors when SASL is enabled on DataTransferProtocol. (Chris Nauroth via wheat9) +HDFS-7364. Balancer always shows zero Bytes Already Moved. +(Tsz Wo Nicholas Sze via jing9) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae71a671/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 2a19537..dba1e2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -27,7 +27,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; -import java.util.Formatter; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -162,7 +161,7 @@ import com.google.common.base.Preconditions; public class Balancer { static final Log LOG = LogFactory.getLog(Balancer.class); - private static final Path BALANCER_ID_PATH = new Path("/system/balancer.id"); + static final Path BALANCER_ID_PATH = new Path("/system/balancer.id"); private static final long GB = 1L << 30; //1GB private static final long MAX_SIZE_TO_MOVE = 10*GB; @@ -459,7 +458,7 @@ public class Balancer { } /* reset all fields in a balancer preparing for the next iteration */ - private void resetData(Configuration conf) { + void resetData(Configuration conf) { this.overUtilized.clear(); this.aboveAvgUtilized.clear(); this.belowAvgUtilized.clear(); @@ -467,16 +466,47 @@ public class Balancer { this.policy.reset(); dispatcher.reset(conf);; } - + + static class Result { +final ExitStatus exitStatus; +final long bytesLeftToMove; +final long bytesBeingMoved; +final long bytesAlreadyMoved; + +Result(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved, +long bytesAlreadyMoved) { + this.exitStatus = exitStatus; + this.bytesLeftToMove = bytesLeftToMove; + this.bytesBeingMoved = bytesBeingMoved; + this.bytesAlreadyMoved = bytesAlreadyMoved; +} + +void print(int iteration, PrintStream out) { + out.printf("%-24s %10d %19s %18s %17s%n", + DateFormat.getDateTimeInstance().format(new Date()), iteration, + StringUtils.byteDesc(bytesAlreadyMoved), + StringUtils.byteDesc(bytesLeftToMove), + StringUtils.byteDesc(bytesBeingMoved)); +} + } + + Result newResult(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved) { +return new Result(exitStatus, bytesLeftToMove, bytesBeingMoved, +dispatcher.getBytesMoved()); + } + + Result newResult(ExitStatus exitStatus) { +return new Result(exitStatus, -1, -1, dispatcher.getBytesMoved()); + } + /** Run an iteration for all datanodes. */ - private ExitStatus run(int iteration, Formatter formatter, - Configuration conf) { + Result runOneIteration() { try { final