Repository: ambari Updated Branches: refs/heads/trunk 4ed15d018 -> b79fd597c
AMBARI-11558. Ooozie start takes too long (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b79fd597 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b79fd597 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b79fd597 Branch: refs/heads/trunk Commit: b79fd597cd54f8029b21d247b2647695aa95a7b9 Parents: 4ed15d0 Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Sat May 30 19:23:58 2015 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Sat May 30 19:23:58 2015 +0300 ---------------------------------------------------------------------- .../python/resource_management/core/logger.py | 2 +- .../libraries/providers/hdfs_resource.py | 37 +++-- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 39 +++-- .../4.0.0.2.0/package/scripts/oozie_service.py | 43 +++-- .../4.0.0.2.0/package/scripts/params_linux.py | 4 + .../stacks/2.0.6/OOZIE/test_oozie_server.py | 161 ++++++++++++++----- .../src/test/python/stacks/utils/RMFTestCase.py | 2 +- 7 files changed, 205 insertions(+), 83 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b79fd597/ambari-common/src/main/python/resource_management/core/logger.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/logger.py b/ambari-common/src/main/python/resource_management/core/logger.py index c060753..1d6ace2 100644 --- a/ambari-common/src/main/python/resource_management/core/logger.py +++ b/ambari-common/src/main/python/resource_management/core/logger.py @@ -25,7 +25,7 @@ import sys import logging from resource_management.libraries.script.config_dictionary import UnknownConfiguration -MESSAGE_MAX_LEN = 256 +MESSAGE_MAX_LEN = 512 DICTIONARY_MAX_LEN = 5 class Logger: http://git-wip-us.apache.org/repos/asf/ambari/blob/b79fd597/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py index 25f0cd5..ac9a0e2 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py @@ -119,11 +119,23 @@ class HdfsResourceJar: env.config['hdfs_files'] = [] class WebHDFSUtil: - def __init__(self, address, run_user, logoutput, security_enabled): - self.address = address + def __init__(self, hdfs_site, run_user, security_enabled, logoutput=None): + https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.https-address') + http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address') + is_https_enabled = hdfs_site['dfs.https.enabled'] if not is_empty(hdfs_site['dfs.https.enabled']) else False + + address = https_nn_address if is_https_enabled else http_nn_address + protocol = "https" if is_https_enabled else "http" + + self.address = format("{protocol}://{address}") self.run_user = run_user - self.logoutput = logoutput self.security_enabled = security_enabled + self.logoutput = logoutput + + @staticmethod + def is_webhdfs_available(is_webhdfs_enabled, default_fs): + # only hdfs seems to support webHDFS + return (is_webhdfs_enabled and default_fs.startswith("hdfs")) def parse_path(self, path): """ @@ -144,7 +156,7 @@ class WebHDFSUtil: return re.sub("[/]+", "/", path) - valid_status_codes = ["200", "201", "500"] + valid_status_codes = ["200", "201", ""] def run_command(self, target, operation, method='POST', assertable_result=True, file_to_put=None, ignore_status_codes=[], **kwargs): """ assertable_result - some POST requests return '{"boolean":false}' or '{"boolean":true}' @@ -176,7 +188,7 @@ class WebHDFSUtil: except ValueError: result_dict = out - if status_code not in WebHDFSUtil.valid_status_codes+ignore_status_codes or assertable_result and not result_dict['boolean']: + if status_code not in WebHDFSUtil.valid_status_codes+ignore_status_codes or assertable_result and result_dict and not result_dict['boolean']: formatted_output = json.dumps(result_dict, indent=2) if isinstance(result_dict, dict) else result_dict err_msg = "Execution of '%s' returned status_code=%s. %s" % (shell.string_cmd_from_args_list(cmd), status_code, formatted_output) raise Fail(err_msg) @@ -215,12 +227,9 @@ class HdfsResourceWebHDFS: if main_resource.resource.security_enabled: main_resource.kinit() - - address = main_resource.https_nn_address if main_resource.is_https_enabled else main_resource.http_nn_address - protocol = "https" if main_resource.is_https_enabled else "http" - - self.util = WebHDFSUtil(format("{protocol}://{address}"), main_resource.resource.user, - main_resource.resource.logoutput, main_resource.resource.security_enabled) + + self.util = WebHDFSUtil(main_resource.resource.hdfs_site, main_resource.resource.user, + main_resource.resource.security_enabled, main_resource.resource.logoutput) self.mode = oct(main_resource.resource.mode)[1:] if main_resource.resource.mode else main_resource.resource.mode self.mode_set = False self.main_resource = main_resource @@ -364,9 +373,6 @@ class HdfsResourceProvider(Provider): self.assert_parameter_is_set('hdfs_site') self.webhdfs_enabled = self.resource.hdfs_site['dfs.webhdfs.enabled'] - self.is_https_enabled = self.resource.hdfs_site['dfs.https.enabled'] if not is_empty(self.resource.hdfs_site['dfs.https.enabled']) else False - self.https_nn_address = namenode_ha_utils.get_property_for_active_namenode(self.resource.hdfs_site, 'dfs.namenode.https-address') - self.http_nn_address = namenode_ha_utils.get_property_for_active_namenode(self.resource.hdfs_site, 'dfs.namenode.http-address') def action_delayed(self, action_name): self.assert_parameter_is_set('type') @@ -383,8 +389,7 @@ class HdfsResourceProvider(Provider): self.get_hdfs_resource_executor().action_execute(self) def get_hdfs_resource_executor(self): - # only hdfs seems to support webHDFS - if self.webhdfs_enabled and self.resource.default_fs.startswith("hdfs"): + if WebHDFSUtil.is_webhdfs_available(self.webhdfs_enabled, self.resource.default_fs): return HdfsResourceWebHDFS() else: return HdfsResourceJar() http://git-wip-us.apache.org/repos/asf/ambari/blob/b79fd597/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index c773fca..83b5715 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -17,6 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. """ +import hashlib import os from resource_management.core.resources.service import ServiceConfig @@ -196,17 +197,27 @@ def oozie_server_specific(): ) no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1") - if not params.host_sys_prepped: - configure_cmds = [] - configure_cmds.append(('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home)) - configure_cmds.append(('cp', params.ext_js_path, params.oozie_libext_dir)) - configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) - configure_cmds.append(('chown', '-RL', format('{oozie_user}:{user_group}'), params.oozie_webapps_conf_dir)) + + hashcode_file = format("{oozie_home}/.hashcode") + hashcode = hashlib.md5(format('{oozie_home}/oozie-sharelib.tar.gz')).hexdigest() + skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share && [[ `cat {hashcode_file}` == '{hashcode}' ]]") - Execute( configure_cmds, - not_if = no_op_test, - sudo = True, - ) + untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home) + + Execute( untar_sharelib, # time-expensive + not_if = format("{no_op_test} || {skip_recreate_sharelib}"), + sudo = True, + ) + + configure_cmds = [] + configure_cmds.append(('cp', params.ext_js_path, params.oozie_libext_dir)) + configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) + configure_cmds.append(('chown', '-RL', format('{oozie_user}:{user_group}'), params.oozie_webapps_conf_dir)) + + Execute( configure_cmds, + not_if = no_op_test, + sudo = True, + ) if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \ params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \ @@ -240,9 +251,13 @@ def oozie_server_specific(): not_if = no_op_test, ) - Execute(format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}"), + Execute(format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}"), # time-expensive user = params.oozie_user, - not_if = no_op_test + not_if = format("{no_op_test} || {skip_recreate_sharelib}") + ) + File(hashcode_file, + content = hashcode, + mode = 0644, ) if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0: http://git-wip-us.apache.org/repos/asf/ambari/blob/b79fd597/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py index ef721ba..6afd250 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py @@ -19,6 +19,7 @@ limitations under the License. """ import os from resource_management import * +from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst @@ -69,9 +70,6 @@ def oozie_service(action = 'start', rolling_restart=False): db_connection_check_command = None if not rolling_restart: - cmd1 = format("cd {oozie_tmp_dir} && {oozie_home}/bin/ooziedb.sh create -sqlfile oozie.sql -run") - cmd2 = format("{kinit_if_needed} {put_shared_lib_to_hdfs_cmd} ; hadoop --config {hadoop_conf_dir} dfs -chmod -R 755 {oozie_hdfs_user_dir}/share") - if not os.path.isfile(params.target) and params.jdbc_driver_name == "org.postgresql.Driver": print format("ERROR: jdbc file {target} is unavailable. Please, follow next steps:\n" \ "1) Download postgresql-9.0-801.jdbc4.jar.\n2) Create needed directory: mkdir -p {oozie_home}/libserver/\n" \ @@ -83,13 +81,38 @@ def oozie_service(action = 'start', rolling_restart=False): if db_connection_check_command: Execute( db_connection_check_command, tries=5, try_sleep=10) - Execute( cmd1, user = params.oozie_user, not_if = no_op_test, - ignore_failures = True ) - - not_if_command = as_user(format("{kinit_if_needed} hadoop --config {hadoop_conf_dir} dfs -ls /user/oozie/share | awk 'BEGIN {{count=0;}} /share/ {{count++}} END {{if (count > 0) {{exit 0}} else {{exit 1}}}}'"), - params.oozie_user) - Execute( cmd2, user = params.oozie_user, not_if = not_if_command, - path = params.execute_path ) + Execute( format("cd {oozie_tmp_dir} && {oozie_home}/bin/ooziedb.sh create -sqlfile oozie.sql -run"), + user = params.oozie_user, not_if = no_op_test, + ignore_failures = True + ) + + if params.security_enabled: + Execute(kinit_if_needed, + user = params.oozie_user, + ) + + if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs): + # check with webhdfs is much faster than executing hadoop fs -ls. + util = WebHDFSUtil(params.hdfs_site, params.oozie_user, params.security_enabled) + list_status = util.run_command(params.hdfs_share_dir, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False) + hdfs_share_dir_exists = ('FileStatus' in list_status) + else: + # have to do time expensive hadoop fs -ls check. + hdfs_share_dir_exists = shell.call(format("{kinit_if_needed} hadoop --config {hadoop_conf_dir} dfs -ls {hdfs_share_dir} | awk 'BEGIN {{count=0;}} /share/ {{count++}} END {{if (count > 0) {{exit 0}} else {{exit 1}}}}'"), + user=params.oozie_user)[0] + + if not hdfs_share_dir_exists: + Execute( params.put_shared_lib_to_hdfs_cmd, + user = params.oozie_user, + path = params.execute_path + ) + params.HdfsResource(format("{oozie_hdfs_user_dir}/share"), + type="directory", + action="create_on_execute", + mode=0755, + recursive_chmod=True, + ) + params.HdfsResource(None, action="execute") # start oozie Execute( start_cmd, environment=environment, user = params.oozie_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/b79fd597/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py index 6963410..942b171 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py @@ -142,6 +142,7 @@ if 'oozie.https.port' in config['configurations']['oozie-site'] or 'oozie.https. else: oozie_secure = '' +hdfs_site = config['configurations']['hdfs-site'] fs_root = config['configurations']['core-site']['fs.defaultFS'] if Script.is_hdp_stack_greater_or_equal("2.0") and Script.is_hdp_stack_less_than("2.2"): @@ -177,6 +178,7 @@ else: target = format("{oozie_libext_dir}/{jdbc_driver_jar}") +hdfs_share_dir = "/user/oozie/share" ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] falcon_host = default("/clusterHostInfo/falcon_server_hosts", []) has_falcon_host = not len(falcon_host) == 0 @@ -210,6 +212,8 @@ HdfsResource = functools.partial( default_fs = default_fs ) +is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'] + # The logic for LZO also exists in HDFS' params.py io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() http://git-wip-us.apache.org/repos/asf/ambari/blob/b79fd597/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index 284f392..a75b7d0 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -24,8 +24,15 @@ from stacks.utils.RMFTestCase import * from resource_management.core import shell from resource_management.core.exceptions import Fail from resource_management.libraries import functions +from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil +import hashlib + +md5_mock = MagicMock() +md5_mock.hexdigest.return_value = "abc123hash" @patch("platform.linux_distribution", new = MagicMock(return_value="Linux")) +@patch.object(hashlib, "md5", new=MagicMock(return_value=md5_mock)) +@patch.object(WebHDFSUtil, "run_command", new=MagicMock(return_value={})) class TestOozieServer(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "OOZIE/4.0.0.2.0/package" STACK_VERSION = "2.0.6" @@ -190,45 +197,49 @@ class TestOozieServer(RMFTestCase): recursive = True, ) self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'), - not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', - sudo = True, - ) + not_if = "ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1 || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]", + sudo = True, + ) self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'), - not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', - sudo = True, - ) + not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', + sudo = True, + ) self.assertResourceCalled('Execute', ('chown', u'oozie:hadoop', '/usr/lib/oozie/libext/ext-2.2.zip'), - not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', - sudo = True, - ) + not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', + sudo = True, + ) self.assertResourceCalled('Execute', ('chown', '-RL', u'oozie:hadoop', '/var/lib/oozie/oozie-server/conf'), - not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', - sudo = True, - ) + not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', + sudo = True, + ) self.assertResourceCalled('File', '/tmp/mysql-connector-java.jar', - content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar'), - ) + content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar'), + ) self.assertResourceCalled('Execute', ('cp', - '--remove-destination', - '/tmp/mysql-connector-java.jar', - '/usr/lib/oozie/libext/mysql-connector-java.jar'), - path = ['/bin', '/usr/bin/'], - sudo = True, - ) + '--remove-destination', + '/tmp/mysql-connector-java.jar', + '/usr/lib/oozie/libext/mysql-connector-java.jar'), + path = ['/bin', '/usr/bin/'], + sudo = True, + ) self.assertResourceCalled('File', '/usr/lib/oozie/libext/mysql-connector-java.jar', - owner = 'oozie', - group = 'hadoop', - ) + owner = 'oozie', + group = 'hadoop', + ) self.assertResourceCalled('Execute', 'ambari-sudo.sh cp /usr/lib/falcon/oozie/ext/falcon-oozie-el-extension-*.jar /usr/lib/oozie/libext', - not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', - ) + not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', + ) self.assertResourceCalled('Execute', 'ambari-sudo.sh chown oozie:hadoop /usr/lib/oozie/libext/falcon-oozie-el-extension-*.jar', - not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', - ) + not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', + ) self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war ', - not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', - user = 'oozie', - ) + not_if = "ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1 || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]", + user = 'oozie', + ) + self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode', + content = 'abc123hash', + mode = 0644, + ) self.assertResourceCalled('Execute', ('chown', '-R', u'oozie:hadoop', '/var/lib/oozie/oozie-server'), sudo = True, ) @@ -249,11 +260,37 @@ class TestOozieServer(RMFTestCase): ignore_failures = True, user = 'oozie', ) - self.assertResourceCalled('Execute', ' hadoop --config /etc/hadoop/conf dfs -put /usr/lib/oozie/share /user/oozie ; hadoop --config /etc/hadoop/conf dfs -chmod -R 755 /user/oozie/share', - not_if = shell.as_user(" hadoop --config /etc/hadoop/conf dfs -ls /user/oozie/share | awk 'BEGIN {count=0;} /share/ {count++} END {if (count > 0) {exit 0} else {exit 1}}'", "oozie"), - user = u'oozie', + self.assertResourceCalled('Execute', 'hadoop --config /etc/hadoop/conf dfs -put /usr/lib/oozie/share /user/oozie', path = ['/usr/bin:/usr/bin'], - ) + user = 'oozie', + ) + self.assertResourceCalled('HdfsResource', '/user/oozie/share', + security_enabled = False, + hadoop_bin_dir = '/usr/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + user = 'hdfs', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + recursive_chmod = True, + action = ['create_on_execute'], + hadoop_conf_dir = '/etc/hadoop/conf', + type = 'directory', + mode = 0755, + ) + self.assertResourceCalled('HdfsResource', None, + security_enabled = False, + hadoop_bin_dir = '/usr/bin', + keytab = UnknownConfigurationMock(), + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = UnknownConfigurationMock(), + user = 'hdfs', + action = ['execute'], + hadoop_conf_dir = '/etc/hadoop/conf', + ) self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-start.sh', environment = {'OOZIE_CONFIG': '/etc/oozie/conf'}, not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', @@ -308,11 +345,41 @@ class TestOozieServer(RMFTestCase): ignore_failures = True, user = 'oozie', ) - self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/oozie.service.keytab oozie/c6402.ambari.apache....@example.com; hadoop --config /etc/hadoop/conf dfs -put /usr/lib/oozie/share /user/oozie ; hadoop --config /etc/hadoop/conf dfs -chmod -R 755 /user/oozie/share', - not_if = shell.as_user("/usr/bin/kinit -kt /etc/security/keytabs/oozie.service.keytab oozie/c6402.ambari.apache....@example.com; hadoop --config /etc/hadoop/conf dfs -ls /user/oozie/share | awk 'BEGIN {count=0;} /share/ {count++} END {if (count > 0) {exit 0} else {exit 1}}'", "oozie"), - user = 'oozie', - path = ['/usr/bin:/usr/bin'], - ) + + self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/oozie.service.keytab oozie/c6402.ambari.apache....@example.com;', + user = 'oozie', + ) + self.assertResourceCalled('Execute', 'hadoop --config /etc/hadoop/conf dfs -put /usr/lib/oozie/share /user/oozie', + path = ['/usr/bin:/usr/bin'], + user = 'oozie', + ) + self.assertResourceCalled('HdfsResource', '/user/oozie/share', + security_enabled = True, + hadoop_bin_dir = '/usr/bin', + keytab = '/etc/security/keytabs/hdfs.headless.keytab', + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + user = 'hdfs', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = 'hdfs', + recursive_chmod = True, + action = ['create_on_execute'], + hadoop_conf_dir = '/etc/hadoop/conf', + type = 'directory', + mode = 0755, + ) + self.assertResourceCalled('HdfsResource', None, + security_enabled = True, + hadoop_bin_dir = '/usr/bin', + keytab = '/etc/security/keytabs/hdfs.headless.keytab', + default_fs = 'hdfs://c6401.ambari.apache.org:8020', + hdfs_site = self.getConfig()['configurations']['hdfs-site'], + kinit_path_local = '/usr/bin/kinit', + principal_name = 'hdfs', + user = 'hdfs', + action = ['execute'], + hadoop_conf_dir = '/etc/hadoop/conf', + ) self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-start.sh', environment = {'OOZIE_CONFIG': '/etc/oozie/conf'}, not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', @@ -475,7 +542,7 @@ class TestOozieServer(RMFTestCase): recursive = True, ) self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'), - not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', + not_if = "ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1 || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]", sudo = True, ) self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'), @@ -497,9 +564,13 @@ class TestOozieServer(RMFTestCase): not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', ) self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war ', - not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', + not_if = "ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1 || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]", user = 'oozie', ) + self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode', + content = 'abc123hash', + mode = 0644, + ) self.assertResourceCalled('Execute', ('chown', '-R', u'oozie:hadoop', '/var/lib/oozie/oozie-server'), sudo = True, ) @@ -643,7 +714,7 @@ class TestOozieServer(RMFTestCase): recursive = True, ) self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'), - not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', + not_if = "ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1 || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]", sudo = True, ) self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'), @@ -665,9 +736,13 @@ class TestOozieServer(RMFTestCase): not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', ) self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war -secure', - not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1', + not_if = "ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1 || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]", user = 'oozie', ) + self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode', + content = 'abc123hash', + mode = 0644, + ) self.assertResourceCalled('Execute', ('chown', '-R', u'oozie:hadoop', '/var/lib/oozie/oozie-server'), sudo = True, ) http://git-wip-us.apache.org/repos/asf/ambari/blob/b79fd597/ambari-server/src/test/python/stacks/utils/RMFTestCase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py index 2f608c4..a186527 100644 --- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py +++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py @@ -174,7 +174,7 @@ class RMFTestCase(TestCase): def _ppformat(self, val): if isinstance(val, dict) and len(val) > MAX_SHOWN_DICT_LEN: - return "self.getConfig()['configurations']['hdfs-site']" + return "self.getConfig()['configurations']['?']" val = pprint.pformat(val)
