Repository: ambari Updated Branches: refs/heads/trunk 2df3a954e -> c13c5828b
AMBARI-11743. NameNode is forced to leave safemode, which causes HBMaster master to crash if done too quickly (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c13c5828 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c13c5828 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c13c5828 Branch: refs/heads/trunk Commit: c13c5828b2f9dc169aca41df5bc2c4363a6919be Parents: 2df3a95 Author: Alejandro Fernandez <[email protected]> Authored: Fri Jun 5 17:24:25 2015 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Mon Jun 8 12:56:46 2015 -0700 ---------------------------------------------------------------------- .../libraries/functions/copy_tarball.py | 9 ++- .../2.1.0.2.0/package/scripts/hdfs_namenode.py | 50 +++++------- .../python/stacks/2.0.6/HDFS/test_namenode.py | 83 +++++++++++--------- .../stacks/2.0.6/HIVE/test_hive_server.py | 29 ++++--- .../stacks/2.2/SPARK/test_job_history_server.py | 39 ++++++++- 5 files changed, 125 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c13c5828/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py index 1226439..8eab473 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py @@ -73,14 +73,15 @@ def _get_single_version_from_hdp_select(): code, stdoutdata = shell.call(get_hdp_versions_cmd, logoutput=True) with open(tmp_file, 'r+') as file: out = file.read() - except: - Logger.error("Could not parse output of {0}".format(str(tmp_file))) + except Exception, e: + Logger.logger.exception("Could not parse output of {0}. Error: {1}".format(str(tmp_file), str(e))) finally: try: if os.path.exists(tmp_file): os.remove(tmp_file) - except: - pass + except Exception, e: + Logger.logger.exception("Could not remove file {0}. Error: {1}".format(str(tmp_file), str(e))) + if code != 0 or out is None or out == "": Logger.error("Could not verify HDP version by calling '{0}'. Return Code: {1}, Output: {2}.".format(get_hdp_versions_cmd, str(code), str(out))) return None http://git-wip-us.apache.org/repos/asf/ambari/blob/c13c5828/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py index d26d145..827c03d 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py @@ -89,61 +89,53 @@ def namenode(action=None, do_format=True, rolling_restart=False, env=None): Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"), user = params.hdfs_user) - is_namenode_safe_mode_off = format("hadoop dfsadmin -fs {namenode_address} -safemode get | grep 'Safe mode is OFF'") + is_namenode_safe_mode_off = format("hdfs dfsadmin -fs {namenode_address} -safemode get | grep 'Safe mode is OFF'") if params.dfs_ha_enabled: is_active_namenode_cmd = as_user(format("hdfs --config {hadoop_conf_dir} haadmin -getServiceState {namenode_id} | grep active"), params.hdfs_user, env={'PATH':params.hadoop_bin_dir}) else: is_active_namenode_cmd = None - # During normal operations, if HA is enabled and it is in standby, then stay in current state, otherwise, leave safemode. - # During Rolling Upgrade, both namenodes must leave safemode. + # During normal operations, if HA is enabled and it is in standby, then no need to check safemode staus. + # During Rolling Upgrade, both namenodes must eventually leave safemode, and Ambari can wait for this. # ___Scenario_________|_Expected safemode state___|_Wait for safemode OFF____| # 1 (HA and active) | ON -> OFF | Yes | # 2 (HA and standby) | no change (yes during RU) | no check (yes during RU) | # 3 (no-HA) | ON -> OFF | Yes | - leave_safe_mode = False + check_for_safemode_off = False msg = "" if params.dfs_ha_enabled: code, out = shell.call(is_active_namenode_cmd, logoutput=True) # If active NN, code will be 0 if code == 0: # active - leave_safe_mode = True - msg = "Must leave safemode since High Availability is enabled and this is the Active NameNode." + check_for_safemode_off = True + msg = "Must wait to leave safemode since High Availability is enabled and this is the Active NameNode." elif rolling_restart: - leave_safe_mode = True - msg = "Must leave safemode since High Availability is enabled during a Rolling Upgrade" + check_for_safemode_off = True + msg = "Must wait to leave safemode since High Availability is enabled during a Rolling Upgrade" else: - msg = "Must leave safemode since High Availability is not enabled." - leave_safe_mode = True + msg = "Must wait to leave safemode since High Availability is not enabled." + check_for_safemode_off = True if not msg: msg = "Will remain in the current safemode state." Logger.info(msg) - if leave_safe_mode: - # First check if Namenode is not in 'safemode OFF' (equivalent to safemode ON), if so, then leave it + if check_for_safemode_off: + # First check if Namenode is not in 'safemode OFF' (equivalent to safemode ON). If safemode is OFF, no change. + # If safemode is ON, first wait for NameNode to leave safemode on its own (if that doesn't happen within 30 seconds, then + # force NameNode to leave safemode). Logger.info("Checking the NameNode safemode status since may need to transition from ON to OFF.") - code, out = shell.call(is_namenode_safe_mode_off, user=params.hdfs_user) - if code != 0: - Logger.info("Will need to leave safemode, state should be OFF.") - leave_safe_mode_cmd = format("hdfs --config {hadoop_conf_dir} dfsadmin -fs {namenode_address} -safemode leave") - Execute(leave_safe_mode_cmd, - tries=10, - try_sleep=10, - user=params.hdfs_user, - path=[params.hadoop_bin_dir], - ) - Logger.info("Checking if safemode state is now OFF.") - # Verify if Namenode should be in safemode OFF + try: + # Wait up to 30 mins Execute(is_namenode_safe_mode_off, - tries=40, + tries=180, try_sleep=10, - path=[params.hadoop_bin_dir], - user=params.hdfs_user + user=params.hdfs_user, + logoutput=True ) - pass - pass + except Fail: + Logger.error("NameNode is still in safemode, please be careful with commands that need safemode OFF.") # Always run this on non-HA, or active NameNode during HA. create_hdfs_directories(is_active_namenode_cmd) http://git-wip-us.apache.org/repos/asf/ambari/blob/c13c5828/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py index b920c17..f6c3b1a 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py @@ -50,7 +50,7 @@ class TestNamenode(RMFTestCase): config_file = "altfs_plus_hdfs.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, - call_mocks = [(5,"")], + call_mocks = [(0,"")], ) self.assert_configure_default() self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l | grep -q ^0$',) @@ -87,18 +87,12 @@ class TestNamenode(RMFTestCase): environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'}, not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1', ) - self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf dfsadmin -fs hdfs://c6405.ambari.apache.org:8020 -safemode leave', - path = ['/usr/bin'], - tries = 10, - try_sleep = 10, - user = 'hdfs', - ) - self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://c6405.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'", - path = ['/usr/bin'], - tries = 40, - user = 'hdfs', - try_sleep = 10, - ) + self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://c6405.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'", + tries=180, + try_sleep=10, + user="hdfs", + logoutput=True + ) self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = False, only_if=None, @@ -166,7 +160,7 @@ class TestNamenode(RMFTestCase): config_file = "default.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, - call_mocks = [(5,"")], + call_mocks = [(0,"")], ) self.assert_configure_default() self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l | grep -q ^0$',) @@ -203,17 +197,11 @@ class TestNamenode(RMFTestCase): environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'}, not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1', ) - self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode leave', - path = ['/usr/bin'], - tries = 10, - try_sleep = 10, - user = 'hdfs', - ) - self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'", - path = ['/usr/bin'], - tries = 40, - user = 'hdfs', - try_sleep = 10, + self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'", + tries=180, + try_sleep=10, + user="hdfs", + logoutput=True ) self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = False, @@ -302,7 +290,7 @@ class TestNamenode(RMFTestCase): config_file = "secured.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, - call_mocks = [(5,"")], + call_mocks = [(0,"")], ) self.assert_configure_secured() self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l | grep -q ^0$',) @@ -342,17 +330,11 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs', user='hdfs', ) - self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode leave', - path = ['/usr/bin'], - tries = 10, - try_sleep = 10, - user = 'hdfs', - ) - self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'", - path = ['/usr/bin'], - tries = 40, - user = 'hdfs', - try_sleep = 10, + self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'", + tries=180, + try_sleep=10, + user="hdfs", + logoutput=True ) self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = True, @@ -448,6 +430,12 @@ class TestNamenode(RMFTestCase): environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'}, not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1', ) + self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://ns1 -safemode get | grep 'Safe mode is OFF'", + tries=180, + try_sleep=10, + user="hdfs", + logoutput=True + ) self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = False, only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active'", @@ -533,6 +521,12 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs', user = 'hdfs', ) + self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://ns1 -safemode get | grep 'Safe mode is OFF'", + tries=180, + try_sleep=10, + user="hdfs", + logoutput=True + ) self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = True, only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active'", @@ -627,6 +621,12 @@ class TestNamenode(RMFTestCase): environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'}, not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1', ) + self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://ns1 -safemode get | grep 'Safe mode is OFF'", + tries=180, + try_sleep=10, + user="hdfs", + logoutput=True + ) self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = False, only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active'", @@ -677,7 +677,7 @@ class TestNamenode(RMFTestCase): # tests namenode start command when NameNode HA is enabled, and # the HA cluster is started initially, rather than using the UI Wizard # this test verifies the startup of a "standby" namenode - @patch.object(shell, "call", new=MagicMock(return_value=(5,""))) + @patch.object(shell, "call", new=MagicMock(return_value=(0,""))) def test_start_ha_bootstrap_standby_from_blueprint(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py", classname = "NameNode", @@ -718,7 +718,12 @@ class TestNamenode(RMFTestCase): environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'}, not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1', ) - + self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://ns1 -safemode get | grep 'Safe mode is OFF'", + tries=180, + try_sleep=10, + user="hdfs", + logoutput=True + ) self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = False, only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn2 | grep active'", http://git-wip-us.apache.org/repos/asf/ambari/blob/c13c5828/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py index 6fcd0f1..5800bf1 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py @@ -29,16 +29,17 @@ from resource_management.core import shell from resource_management.libraries.script.script import Script from resource_management.libraries.functions import copy_tarball from resource_management.libraries import functions - +from resource_management.core.logger import Logger @patch.object(functions, "get_hdp_version", new = MagicMock(return_value="2.0.0.0-1234")) @patch("resource_management.libraries.functions.check_thrift_port_sasl", new=MagicMock()) -#@patch("atlas_plugin_utils.configure_for_plugin", new=MagicMock()) class TestHiveServer(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "HIVE/0.12.0.2.0/package" STACK_VERSION = "2.0.6" UPGRADE_STACK_VERSION = "2.2" + def setUp(self): + Logger.logger = MagicMock() @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=False)) @@ -54,9 +55,11 @@ class TestHiveServer(RMFTestCase): self.assert_configure_default() self.assertNoMoreResources() + @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") @patch("socket.socket") @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=False)) - def test_start_default(self, socket_mock): + def test_start_default(self, socket_mock, copy_to_hfds_mock): + copy_to_hfds_mock.return_value = None s = socket_mock.return_value self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py", classname="HiveServer", @@ -116,8 +119,10 @@ class TestHiveServer(RMFTestCase): ) self.assertNoMoreResources() + @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=False)) - def test_start_default_alt_tmp(self): + def test_start_default_alt_tmp(self, copy_to_hfds_mock): + copy_to_hfds_mock.return_value = None self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py", classname = "HiveServer", command = "start", @@ -144,9 +149,10 @@ class TestHiveServer(RMFTestCase): ) self.assertNoMoreResources() - + @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=False)) - def test_start_default_alt_nn_ha_tmp(self): + def test_start_default_alt_nn_ha_tmp(self, copy_to_hfds_mock): + copy_to_hfds_mock.return_value = None self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py", classname = "HiveServer", command = "start", @@ -211,11 +217,13 @@ class TestHiveServer(RMFTestCase): self.assert_configure_secured() self.assertNoMoreResources() + @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") @patch("hive_service.check_fs_root") @patch("socket.socket") @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=False)) - def test_start_secured(self, socket_mock, check_fs_root_mock): + def test_start_secured(self, socket_mock, check_fs_root_mock, copy_to_hfds_mock): s = socket_mock.return_value + copy_to_hfds_mock.return_value = None self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py", classname = "HiveServer", @@ -302,6 +310,10 @@ class TestHiveServer(RMFTestCase): mode = 0755, ) + if self._testMethodName == "test_socket_timeout": + # This test will not call any more resources. + return + self.assertResourceCalled('HdfsResource', '/apps/hive/warehouse', security_enabled = False, hadoop_bin_dir = '/usr/bin', @@ -630,10 +642,9 @@ class TestHiveServer(RMFTestCase): cd_access='a', ) - @patch("hive_service.check_fs_root") @patch("time.time") @patch("socket.socket") - def test_socket_timeout(self, socket_mock, time_mock, check_fs_root_mock): + def test_socket_timeout(self, socket_mock, time_mock): s = socket_mock.return_value s.connect = MagicMock() s.connect.side_effect = socket.error("") http://git-wip-us.apache.org/repos/asf/ambari/blob/c13c5828/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py index 8e91481..db83824 100644 --- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py +++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py @@ -26,7 +26,9 @@ class TestJobHistoryServer(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.0.2.2/package" STACK_VERSION = "2.2" - def test_configure_default(self): + @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") + def test_configure_default(self, copy_to_hdfs_mock): + copy_to_hdfs_mock = True self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py", classname = "JobHistoryServer", command = "configure", @@ -36,8 +38,10 @@ class TestJobHistoryServer(RMFTestCase): ) self.assert_configure_default() self.assertNoMoreResources() - - def test_start_default(self): + + @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") + def test_start_default(self, copy_to_hdfs_mock): + copy_to_hdfs_mock.return_value = True self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py", classname = "JobHistoryServer", command = "start", @@ -46,6 +50,18 @@ class TestJobHistoryServer(RMFTestCase): target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assert_configure_default() + self.assertResourceCalled('HdfsResource', None, + security_enabled = False, + hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = {u'a': u'b'}, + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + action = ['execute'], + hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', + ) self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/start-history-server.sh', environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, not_if = 'ls /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid >/dev/null 2>&1 && ps -p `cat /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid` >/dev/null 2>&1', @@ -81,7 +97,9 @@ class TestJobHistoryServer(RMFTestCase): self.assert_configure_secured() self.assertNoMoreResources() - def test_start_secured(self): + @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") + def test_start_secured(self, copy_to_hdfs_mock): + copy_to_hdfs_mock.return_value = True self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py", classname = "JobHistoryServer", command = "start", @@ -94,6 +112,19 @@ class TestJobHistoryServer(RMFTestCase): user = 'spark', ) + self.assertResourceCalled('HdfsResource', None, + action=['execute'], + default_fs= UnknownConfigurationMock(), + hadoop_bin_dir='/usr/hdp/current/hadoop-client/bin', + hadoop_conf_dir='/usr/hdp/current/hadoop-client/conf', + hdfs_site=UnknownConfigurationMock(), + keytab=UnknownConfigurationMock(), + kinit_path_local='/usr/bin/kinit', + principal_name=UnknownConfigurationMock(), + security_enabled=True, + user=UnknownConfigurationMock() + ) + self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/start-history-server.sh', environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'}, not_if = 'ls /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid >/dev/null 2>&1 && ps -p `cat /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid` >/dev/null 2>&1',
