Repository: ambari Updated Branches: refs/heads/branch-2.5 f1cd0b76d -> d1d58ad8e
AMBARI-20175. Rebalance HDFS operation returns after the command is issued. (Laszlo Puskas via stoader) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d1d58ad8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d1d58ad8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d1d58ad8 Branch: refs/heads/branch-2.5 Commit: d1d58ad8e931684b6d4a592026aa9f3f38eafdc2 Parents: f1cd0b7 Author: Laszlo Puskas <lpus...@hortonworks.com> Authored: Fri Mar 3 21:00:57 2017 +0100 Committer: Toader, Sebastian <stoa...@hortonworks.com> Committed: Fri Mar 3 21:00:57 2017 +0100 ---------------------------------------------------------------------- .../2.1.0.2.0/package/scripts/hdfs_rebalance.py | 31 +++++++++++++++++++- .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 14 ++++++--- .../python/stacks/2.0.6/HDFS/test_namenode.py | 19 ++++++++---- 3 files changed, 53 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d1d58ad8/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py index 1dc545e..e0432a4 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py @@ -19,6 +19,10 @@ limitations under the License. ''' import re +import sys +from resource_management.core.exceptions import Fail +from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop + class HdfsParser(): def __init__(self): @@ -127,4 +131,29 @@ class HdfsLine(): 'bytesBeingMoved': self.bytesBeingMoved, } def __str__(self): - return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, bytesBeingMoved=%d]"%(self.date, self.iteration, self.bytesAlreadyMoved, self.bytesLeftToMove, self.bytesBeingMoved) \ No newline at end of file + return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, bytesBeingMoved=%d]"%(self.date, self.iteration, self.bytesAlreadyMoved, self.bytesLeftToMove, self.bytesBeingMoved) + + + +def is_balancer_running(): + import params + check_balancer_command = "fs -test -e /system/balancer.id" + does_hdfs_file_exist = False + try: + _print("Checking if the balancer is running ...") + ExecuteHadoop(check_balancer_command, + user=params.hdfs_user, + logoutput=True, + conf_dir=params.hadoop_conf_dir, + bin_dir=params.hadoop_bin_dir) + + does_hdfs_file_exist = True + _print("Balancer is running. ") + except Fail: + pass + + return does_hdfs_file_exist + +def _print(line): + sys.stdout.write(line) + sys.stdout.flush() http://git-wip-us.apache.org/repos/asf/ambari/blob/d1d58ad8/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index 123486e..350e704 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -341,10 +341,16 @@ class NameNodeDefault(NameNode): self.put_structured_out({'completePercent' : 1}) return - Execute(command, - on_new_line = handle_new_line, - logoutput = False, - ) + if (not hdfs_rebalance.is_balancer_running()): + # As the rebalance may take a long time (haours, days) the process is triggered only + # Tracking the progress based on the command output is no longer supported due to this + Execute(command, wait_for_finish=False) + + _print("The rebalance process has been triggered") + else: + _print("There is another balancer running. This means you or another Ambari user may have triggered the " + "operation earlier. The process may take a long time to finish (hours, even days). If the problem persists " + "please consult with the HDFS administrators if they have triggred or killed the operation.") if params.security_enabled: # Delete the kerberos credentials cache (ccache) file http://git-wip-us.apache.org/repos/asf/ambari/blob/d1d58ad8/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py index fae500f..da5d353 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py @@ -1205,8 +1205,10 @@ class TestNamenode(RMFTestCase): cd_access='a' ) + @patch("hdfs_rebalance.is_balancer_running") @patch("resource_management.libraries.script.Script.put_structured_out") - def test_rebalance_hdfs(self, pso): + def test_rebalance_hdfs(self, pso, hdfs_rebalance_mock): + hdfs_rebalance_mock.return_value = False self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py", classname = "NameNode", command = "rebalancehdfs", @@ -1214,17 +1216,20 @@ class TestNamenode(RMFTestCase): stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) + self.assertResourceCalled('Execute', "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf balancer -threshold -1'", - logoutput = False, - on_new_line = FunctionMock('handle_new_line'), + wait_for_finish=False ) + self.assertNoMoreResources() + @patch("hdfs_rebalance.is_balancer_running") @patch("resource_management.libraries.script.Script.put_structured_out") @patch("os.system") - def test_rebalance_secured_hdfs(self, pso, system_mock): + def test_rebalance_secured_hdfs(self, pso, system_mock, hdfs_rebalance_mock): system_mock.return_value = -1 + hdfs_rebalance_mock.return_value = False self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py", classname = "NameNode", command = "rebalancehdfs", @@ -1241,13 +1246,15 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('Execute', kinit_cmd, user = 'hdfs', ) + self.assertResourceCalled('Execute', rebalance_cmd, - logoutput = False, - on_new_line = FunctionMock('handle_new_line'), + wait_for_finish=False ) + self.assertResourceCalled('File', ccache_path, action = ['delete'], ) + self.assertNoMoreResources() @patch("os.path.isfile")
