[01/25] ambari git commit: AMBARI-20175. Rebalance HDFS operation returns after the command is issued. (Laszlo Puskas via stoader)

2017-03-06 Thread jonathanhurley 
Repository: ambari
Updated Branches:
  refs/heads/branch-feature-AMBARI-12556 4527aa15d -> 353cd9dfd


AMBARI-20175. Rebalance HDFS operation returns after the command is issued. 
(Laszlo Puskas via stoader)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f896b841
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f896b841
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f896b841

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: f896b84116573c5432b60bdf90315edad20e61de
Parents: a28e0ca
Author: Laszlo Puskas 
Authored: Fri Mar 3 20:50:46 2017 +0100
Committer: Toader, Sebastian 
Committed: Fri Mar 3 20:51:16 2017 +0100

--
 .../2.1.0.2.0/package/scripts/hdfs_rebalance.py | 32 +++-
 .../HDFS/2.1.0.2.0/package/scripts/namenode.py  | 14 ++---
 .../3.0.0.3.0/package/scripts/hdfs_rebalance.py | 28 -
 .../HDFS/3.0.0.3.0/package/scripts/namenode.py  | 14 ++---
 .../python/stacks/2.0.6/HDFS/test_namenode.py   | 19 
 5 files changed, 91 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/f896b841/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
--
diff --git 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
index 1dc545e..68c609f 100644
--- 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
+++ 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
@@ -19,6 +19,10 @@ limitations under the License.
 '''
 
 import re
+import sys
+from resource_management.core.exceptions import Fail
+from resource_management.libraries.resources.execute_hadoop import 
ExecuteHadoop
+
 
 class HdfsParser():
   def __init__(self):
@@ -127,4 +131,30 @@ class HdfsLine():
 'bytesBeingMoved': self.bytesBeingMoved,
   }
   def __str__(self):
-return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, 
bytesBeingMoved=%d]"%(self.date, self.iteration, self.bytesAlreadyMoved, 
self.bytesLeftToMove, self.bytesBeingMoved)
\ No newline at end of file
+return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, 
bytesBeingMoved=%d]"%(self.date, self.iteration, self.bytesAlreadyMoved, 
self.bytesLeftToMove, self.bytesBeingMoved)
+
+
+
+def is_balancer_running():
+  import params
+  check_balancer_command = "fs -test -e /system/balancer.id"
+  does_hdfs_file_exist = False
+  try:
+_print("Checking if the balancer is running ...")
+ExecuteHadoop(check_balancer_command,
+  user=params.hdfs_user,
+  logoutput=True,
+  conf_dir=params.hadoop_conf_dir,
+  bin_dir=params.hadoop_bin_dir)
+
+does_hdfs_file_exist = True
+_print("Balancer is running. ")
+  except Fail:
+pass
+
+  return does_hdfs_file_exist
+
+def _print(line):
+  sys.stdout.write(line)
+  sys.stdout.flush()
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/f896b841/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
--
diff --git 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
index bd05a95..1347f37 100644
--- 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
+++ 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
@@ -341,10 +341,16 @@ class NameNodeDefault(NameNode):
 self.put_structured_out({'completePercent' : 1})
 return
 
-Execute(command,
-on_new_line = handle_new_line,
-logoutput = False,
-)
+if (not hdfs_rebalance.is_balancer_running()):
+  # As the rebalance may take a long time (haours, days) the process is 
triggered only
+  # Tracking the progress based on the command output is no longer 
supported due to this
+  Execute(command, wait_for_finish=False)
+
+  _print("The rebalance process has been triggered")
+else:
+  _print("There is another balancer running. This means you or another 
Ambari user may have triggered the "
+ "operation earlier. The process may take a long time to finish 
(hours, even days). If the problem persists "
+ "please consult with the HDFS administrators if they have

ambari git commit: AMBARI-20175. Rebalance HDFS operation returns after the command is issued. (Laszlo Puskas via stoader)

2017-03-03 Thread stoader 
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 f1cd0b76d -> d1d58ad8e


AMBARI-20175. Rebalance HDFS operation returns after the command is issued. 
(Laszlo Puskas via stoader)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d1d58ad8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d1d58ad8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d1d58ad8

Branch: refs/heads/branch-2.5
Commit: d1d58ad8e931684b6d4a592026aa9f3f38eafdc2
Parents: f1cd0b7
Author: Laszlo Puskas 
Authored: Fri Mar 3 21:00:57 2017 +0100
Committer: Toader, Sebastian 
Committed: Fri Mar 3 21:00:57 2017 +0100

--
 .../2.1.0.2.0/package/scripts/hdfs_rebalance.py | 31 +++-
 .../HDFS/2.1.0.2.0/package/scripts/namenode.py  | 14 ++---
 .../python/stacks/2.0.6/HDFS/test_namenode.py   | 19 
 3 files changed, 53 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/d1d58ad8/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
--
diff --git 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
index 1dc545e..e0432a4 100644
--- 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
+++ 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
@@ -19,6 +19,10 @@ limitations under the License.
 '''
 
 import re
+import sys
+from resource_management.core.exceptions import Fail
+from resource_management.libraries.resources.execute_hadoop import 
ExecuteHadoop
+
 
 class HdfsParser():
   def __init__(self):
@@ -127,4 +131,29 @@ class HdfsLine():
 'bytesBeingMoved': self.bytesBeingMoved,
   }
   def __str__(self):
-return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, 
bytesBeingMoved=%d]"%(self.date, self.iteration, self.bytesAlreadyMoved, 
self.bytesLeftToMove, self.bytesBeingMoved)
\ No newline at end of file
+return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, 
bytesBeingMoved=%d]"%(self.date, self.iteration, self.bytesAlreadyMoved, 
self.bytesLeftToMove, self.bytesBeingMoved)
+
+
+
+def is_balancer_running():
+  import params
+  check_balancer_command = "fs -test -e /system/balancer.id"
+  does_hdfs_file_exist = False
+  try:
+_print("Checking if the balancer is running ...")
+ExecuteHadoop(check_balancer_command,
+  user=params.hdfs_user,
+  logoutput=True,
+  conf_dir=params.hadoop_conf_dir,
+  bin_dir=params.hadoop_bin_dir)
+
+does_hdfs_file_exist = True
+_print("Balancer is running. ")
+  except Fail:
+pass
+
+  return does_hdfs_file_exist
+
+def _print(line):
+  sys.stdout.write(line)
+  sys.stdout.flush()

http://git-wip-us.apache.org/repos/asf/ambari/blob/d1d58ad8/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
--
diff --git 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
index 123486e..350e704 100644
--- 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
+++ 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
@@ -341,10 +341,16 @@ class NameNodeDefault(NameNode):
 self.put_structured_out({'completePercent' : 1})
 return
 
-Execute(command,
-on_new_line = handle_new_line,
-logoutput = False,
-)
+if (not hdfs_rebalance.is_balancer_running()):
+  # As the rebalance may take a long time (haours, days) the process is 
triggered only
+  # Tracking the progress based on the command output is no longer 
supported due to this
+  Execute(command, wait_for_finish=False)
+
+  _print("The rebalance process has been triggered")
+else:
+  _print("There is another balancer running. This means you or another 
Ambari user may have triggered the "
+ "operation earlier. The process may take a long time to finish 
(hours, even days). If the problem persists "
+ "please consult with the HDFS administrators if they have 
triggred or killed the operation.")
 
 if params.security_enabled:
   # Delete the kerberos credentials cache (ccache) file

ambari git commit: AMBARI-20175. Rebalance HDFS operation returns after the command is issued. (Laszlo Puskas via stoader)

2017-03-03 Thread stoader 
Repository: ambari
Updated Branches:
  refs/heads/trunk a28e0ca13 -> f896b8411


AMBARI-20175. Rebalance HDFS operation returns after the command is issued. 
(Laszlo Puskas via stoader)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f896b841
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f896b841
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f896b841

Branch: refs/heads/trunk
Commit: f896b84116573c5432b60bdf90315edad20e61de
Parents: a28e0ca
Author: Laszlo Puskas 
Authored: Fri Mar 3 20:50:46 2017 +0100
Committer: Toader, Sebastian 
Committed: Fri Mar 3 20:51:16 2017 +0100

--
 .../2.1.0.2.0/package/scripts/hdfs_rebalance.py | 32 +++-
 .../HDFS/2.1.0.2.0/package/scripts/namenode.py  | 14 ++---
 .../3.0.0.3.0/package/scripts/hdfs_rebalance.py | 28 -
 .../HDFS/3.0.0.3.0/package/scripts/namenode.py  | 14 ++---
 .../python/stacks/2.0.6/HDFS/test_namenode.py   | 19 
 5 files changed, 91 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/f896b841/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
--
diff --git 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
index 1dc545e..68c609f 100644
--- 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
+++ 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_rebalance.py
@@ -19,6 +19,10 @@ limitations under the License.
 '''
 
 import re
+import sys
+from resource_management.core.exceptions import Fail
+from resource_management.libraries.resources.execute_hadoop import 
ExecuteHadoop
+
 
 class HdfsParser():
   def __init__(self):
@@ -127,4 +131,30 @@ class HdfsLine():
 'bytesBeingMoved': self.bytesBeingMoved,
   }
   def __str__(self):
-return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, 
bytesBeingMoved=%d]"%(self.date, self.iteration, self.bytesAlreadyMoved, 
self.bytesLeftToMove, self.bytesBeingMoved)
\ No newline at end of file
+return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, 
bytesBeingMoved=%d]"%(self.date, self.iteration, self.bytesAlreadyMoved, 
self.bytesLeftToMove, self.bytesBeingMoved)
+
+
+
+def is_balancer_running():
+  import params
+  check_balancer_command = "fs -test -e /system/balancer.id"
+  does_hdfs_file_exist = False
+  try:
+_print("Checking if the balancer is running ...")
+ExecuteHadoop(check_balancer_command,
+  user=params.hdfs_user,
+  logoutput=True,
+  conf_dir=params.hadoop_conf_dir,
+  bin_dir=params.hadoop_bin_dir)
+
+does_hdfs_file_exist = True
+_print("Balancer is running. ")
+  except Fail:
+pass
+
+  return does_hdfs_file_exist
+
+def _print(line):
+  sys.stdout.write(line)
+  sys.stdout.flush()
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/f896b841/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
--
diff --git 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
index bd05a95..1347f37 100644
--- 
a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
+++ 
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
@@ -341,10 +341,16 @@ class NameNodeDefault(NameNode):
 self.put_structured_out({'completePercent' : 1})
 return
 
-Execute(command,
-on_new_line = handle_new_line,
-logoutput = False,
-)
+if (not hdfs_rebalance.is_balancer_running()):
+  # As the rebalance may take a long time (haours, days) the process is 
triggered only
+  # Tracking the progress based on the command output is no longer 
supported due to this
+  Execute(command, wait_for_finish=False)
+
+  _print("The rebalance process has been triggered")
+else:
+  _print("There is another balancer running. This means you or another 
Ambari user may have triggered the "
+ "operation earlier. The process may take a long time to finish 
(hours, even days). If the problem persists "
+ "please consult with the HDFS administrators if they have 
triggred or killed the operation.")