Repository: ambari Updated Branches: refs/heads/trunk 6c21b0942 -> e3c9aa7a4
AMBARI-8478. Falcon service components should indicate security state. (robert levas via jaimin) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e3c9aa7a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e3c9aa7a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e3c9aa7a Branch: refs/heads/trunk Commit: e3c9aa7a4336a43be61342c80395b45288979d5f Parents: 6c21b09 Author: Jaimin Jetly <jai...@hortonworks.com> Authored: Tue Dec 30 11:01:50 2014 -0800 Committer: Jaimin Jetly <jai...@hortonworks.com> Committed: Tue Dec 30 11:01:50 2014 -0800 ---------------------------------------------------------------------- .../libraries/functions/security_commons.py | 52 +++++---- .../0.5.0.2.1/package/scripts/falcon_client.py | 9 ++ .../0.5.0.2.1/package/scripts/falcon_server.py | 62 +++++++++++ .../0.5.0.2.1/package/scripts/status_params.py | 11 ++ .../HDFS/2.1.0.2.0/package/scripts/datanode.py | 15 ++- .../2.1.0.2.0/package/scripts/hdfs_client.py | 15 +-- .../2.1.0.2.0/package/scripts/journalnode.py | 16 +-- .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 16 +-- .../HDFS/2.1.0.2.0/package/scripts/snamenode.py | 16 +-- .../2.1.0.2.0/package/scripts/zkfc_slave.py | 16 +-- .../stacks/2.1/FALCON/test_falcon_client.py | 25 +++++ .../stacks/2.1/FALCON/test_falcon_server.py | 106 +++++++++++++++++++ .../src/test/python/stacks/utils/RMFTestCase.py | 8 +- 13 files changed, 300 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-common/src/main/python/resource_management/libraries/functions/security_commons.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/security_commons.py b/ambari-common/src/main/python/resource_management/libraries/functions/security_commons.py index d3cd1a2..535a53b 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/security_commons.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/security_commons.py @@ -23,6 +23,8 @@ from tempfile import mkstemp import os import json +FILE_TYPE_XML = 'XML' +FILE_TYPE_PROPERTIES = 'PROPERTIES' def validate_security_config_properties(params, configuration_rules): """ @@ -103,29 +105,47 @@ def build_expectations(config_file, value_checks, empty_checks, read_checks): def get_params_from_filesystem(conf_dir, config_files): """ Used to retrieve properties from xml config files and build a dict + + The dictionary of configuration files to file types should contain one of the following values" + 'XML' + 'PROPERTIES' + :param conf_dir: directory where the configuration files sit - :param config_files: list of configuration file names - :return: + :param config_files: dictionary of configuration file names to (supported) file types + :return: a dictionary of config-type to a dictionary of key/value pairs for """ result = {} from xml.etree import ElementTree as ET - - for config_file in config_files: - configuration = ET.parse(conf_dir + os.sep + config_file) - props = configuration.getroot().getchildren() - config_file_id = config_file[:-4] if len(config_file) > 4 else config_file - result[config_file_id] = {} - for prop in props: - result[config_file_id].update({prop[0].text: prop[1].text}) + import ConfigParser, StringIO + for config_file, file_type in config_files.iteritems(): + file_name, file_ext = os.path.splitext(config_file) + + if file_type == FILE_TYPE_XML: + configuration = ET.parse(conf_dir + os.sep + config_file) + props = configuration.getroot().getchildren() + config_file_id = file_name if file_name else config_file + result[config_file_id] = {} + for prop in props: + result[config_file_id].update({prop[0].text: prop[1].text}) + + elif file_type == FILE_TYPE_PROPERTIES: + with open(conf_dir + os.sep + config_file, 'r') as f: + config_string = '[root]\n' + f.read() + ini_fp = StringIO.StringIO(config_string) + config = ConfigParser.RawConfigParser() + config.readfp(ini_fp) + props = config.items('root') + result[file_name] = {} + for key, value in props: + result[file_name].update({key : value}) return result def cached_kinit_executor(kinit_path, exec_user, keytab_file, principal, hostname, temp_dir, - expiration_time): + expiration_time=5): """ Main cached kinit executor - Uses a temporary file on the FS to cache executions. Each command will have its own file and only one entry (last successful execution) will be stored - :return: """ key = str(hash("%s|%s" % (principal, keytab_file))) filename = key + "_tmp.txt" @@ -151,15 +171,13 @@ def cached_kinit_executor(kinit_path, exec_user, keytab_file, principal, hostnam cache_file.write("{}") if (not output) or (key not in output) or ("last_successful_execution" not in output[key]): - return new_cached_exec(key, file_path, kinit_path, exec_user, keytab_file, principal, hostname) + new_cached_exec(key, file_path, kinit_path, exec_user, keytab_file, principal, hostname) else: last_run_time = output[key]["last_successful_execution"] now = datetime.now() if (now - datetime.strptime(last_run_time, "%Y-%m-%d %H:%M:%S.%f") > timedelta( minutes=expiration_time)): - return new_cached_exec(key, file_path, kinit_path, exec_user, keytab_file, principal, hostname) - else: - return True + new_cached_exec(key, file_path, kinit_path, exec_user, keytab_file, principal, hostname) def new_cached_exec(key, file_path, kinit_path, exec_user, keytab_file, principal, hostname): @@ -180,5 +198,3 @@ def new_cached_exec(key, file_path, kinit_path, exec_user, keytab_file, principa json.dump(result, cache_file) finally: os.remove(temp_kinit_cache_file) - - return True http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py index 6f1cd56..23260df 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py @@ -50,5 +50,14 @@ class FalconClient(Script): Execute(format("hdp-select set hadoop-client {version}")) + def security_status(self, env): + import status_params + env.set_params(status_params) + + if status_params.security_enabled: + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + else: + self.put_structured_out({"securityState": "UNSECURED"}) + if __name__ == "__main__": FalconClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py index 536bd49..52a6f2b 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py @@ -21,6 +21,9 @@ import falcon_server_upgrade from resource_management import * from resource_management.libraries.functions.version import * +from resource_management.libraries.functions.security_commons import build_expectations, \ + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_PROPERTIES from falcon import falcon class FalconServer(Script): @@ -80,6 +83,65 @@ class FalconServer(Script): Execute(format("hdp-select set falcon-server {version}")) falcon_server_upgrade.pre_start_restore() + def security_status(self, env): + import status_params + env.set_params(status_params) + if status_params.security_enabled: + props_value_check = {"*.falcon.authentication.type": "kerberos", + "*.falcon.http.authentication.type": "kerberos"} + props_empty_check = ["*.falcon.service.authentication.kerberos.principal", + "*.falcon.service.authentication.kerberos.keytab", + "*.falcon.http.authentication.kerberos.principal", + "*.falcon.http.authentication.kerberos.keytab"] + props_read_check = ["*.falcon.service.authentication.kerberos.keytab", + "*.falcon.http.authentication.kerberos.keytab"] + falcon_startup_props = build_expectations('startup', props_value_check, props_empty_check, + props_read_check) + + falcon_expectations ={} + falcon_expectations.update(falcon_startup_props) + + security_params = get_params_from_filesystem('/etc/falcon/conf', + {'startup.properties': FILE_TYPE_PROPERTIES}) + result_issues = validate_security_config_properties(security_params, falcon_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ( 'startup' not in security_params + or '*.falcon.service.authentication.kerberos.keytab' not in security_params['startup'] + or '*.falcon.service.authentication.kerberos.principal' not in security_params['startup']) \ + or '*.falcon.http.authentication.kerberos.keytab' not in security_params['startup'] \ + or '*.falcon.http.authentication.kerberos.principal' not in security_params['startup']: + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.falcon_user, + security_params['startup']['*.falcon.service.authentication.kerberos.keytab'], + security_params['startup']['*.falcon.service.authentication.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + cached_kinit_executor(status_params.kinit_path_local, + status_params.falcon_user, + security_params['startup']['*.falcon.http.authentication.kerberos.keytab'], + security_params['startup']['*.falcon.http.authentication.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + else: + self.put_structured_out({"securityState": "UNSECURED"}) + if __name__ == "__main__": FalconServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py index 6ebb35f..facc983 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py @@ -22,3 +22,14 @@ from resource_management import * config = Script.get_config() falcon_pid_dir = config['configurations']['falcon-env']['falcon_pid_dir'] server_pid_file = format('{falcon_pid_dir}/falcon.pid') + +# Security related/required params +hostname = config['hostname'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +hadoop_conf_dir = "/etc/hadoop/conf" +kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +tmp_dir = Script.get_tmp_dir() +falcon_conf_dir_prefix = "/etc/falcon" +falcon_conf_dir = format("{falcon_conf_dir_prefix}/conf") +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +falcon_user = config['configurations']['falcon-env']['falcon_user'] http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py index 8bce423..237ef07 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py @@ -22,7 +22,7 @@ from resource_management import * from resource_management.libraries.functions.version import compare_versions, \ format_hdp_stack_version from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, FILE_TYPE_XML from hdfs import hdfs @@ -108,7 +108,8 @@ class DataNode(Script): hdfs_expectations.update(hdfs_site_expectations) security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, - ['core-site.xml', 'hdfs-site.xml']) + {'core-site.xml': FILE_TYPE_XML, + 'hdfs-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, hdfs_expectations) if not result_issues: # If all validations passed successfully try: @@ -126,18 +127,16 @@ class DataNode(Script): security_params['hdfs-site']['dfs.datanode.keytab.file'], security_params['hdfs-site']['dfs.datanode.kerberos.principal'], status_params.hostname, - status_params.tmp_dir, - 30) + status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: - issues = "" + issues = [] for cf in result_issues: - issues += "Configuration file " + cf + " did not pass the validation. Reason: " + \ - result_issues[cf] - self.put_structured_out({"securityIssuesFound": issues}) + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py index cf93c2f..59be175 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py @@ -19,7 +19,8 @@ limitations under the License. from resource_management import * from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML from hdfs import hdfs from utils import service @@ -70,7 +71,8 @@ class HdfsClient(Script): hdfs_expectations ={} hdfs_expectations.update(core_site_expectations) - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, ['core-site.xml']) + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + {'core-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, hdfs_expectations) if not result_issues: # If all validations passed successfully if status_params.hdfs_user_principal or status_params.hdfs_user_keytab: @@ -80,8 +82,7 @@ class HdfsClient(Script): status_params.hdfs_user_keytab, status_params.hdfs_user_principal, status_params.hostname, - status_params.tmp_dir, - 30) + status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) @@ -90,10 +91,10 @@ class HdfsClient(Script): self.put_structured_out({"securityIssuesFound": "hdfs principal and/or keytab file is not specified"}) self.put_structured_out({"securityState": "UNSECURED"}) else: - issues="" + issues = [] for cf in result_issues: - issues+="Configuration file " + cf + " did not pass the validation. Reason: " + result_issues[cf] - self.put_structured_out({"securityIssuesFound": issues}) + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py index 60e91ce..addf0e1 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py @@ -22,7 +22,8 @@ from resource_management.libraries.functions.version import compare_versions, \ format_hdp_stack_version from resource_management.libraries.functions.format import format from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML from utils import service from hdfs import hdfs @@ -110,7 +111,8 @@ class JournalNode(Script): hdfs_expectations.update(hdfs_site_expectations) hdfs_expectations.update(core_site_expectations) - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, ['core-site.xml']) + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + {'core-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, hdfs_expectations) if not result_issues: # If all validations passed successfully try: @@ -128,18 +130,16 @@ class JournalNode(Script): security_params['hdfs-site']['dfs.journalnode.kerberos.keytab.file'], security_params['hdfs-site']['dfs.journalnode.kerberos.principal'], status_params.hostname, - status_params.tmp_dir, - 30) + status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: - issues = "" + issues = [] for cf in result_issues: - issues += "Configuration file " + cf + " did not pass the validation. Reason: " + \ - result_issues[cf] - self.put_structured_out({"securityIssuesFound": issues}) + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index e8dfe16..a2b7c53 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -25,7 +25,8 @@ from datetime import datetime from resource_management import * from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML from resource_management.libraries.functions.version import compare_versions, \ format_hdp_stack_version from resource_management.libraries.functions.format import format @@ -120,7 +121,8 @@ class NameNode(Script): hdfs_expectations.update(hdfs_site_expectations) security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, - ['core-site.xml', 'hdfs-site.xml']) + {'core-site.xml': FILE_TYPE_XML, + 'hdfs-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, hdfs_expectations) if not result_issues: # If all validations passed successfully try: @@ -138,18 +140,16 @@ class NameNode(Script): security_params['hdfs-site']['dfs.namenode.keytab.file'], security_params['hdfs-site']['dfs.namenode.kerberos.principal'], status_params.hostname, - status_params.tmp_dir, - 30) + status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: - issues = "" + issues = [] for cf in result_issues: - issues += "Configuration file " + cf + " did not pass the validation. Reason: " + \ - result_issues[cf] - self.put_structured_out({"securityIssuesFound": issues}) + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py index 9900a7e..01d2dfb 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py @@ -19,7 +19,8 @@ limitations under the License. from resource_management import * from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML from hdfs_snamenode import snamenode from hdfs import hdfs @@ -88,7 +89,8 @@ class SNameNode(Script): hdfs_expectations.update(hdfs_site_expectations) security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, - ['core-site.xml', 'hdfs-site.xml']) + {'core-site.xml': FILE_TYPE_XML, + 'hdfs-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, hdfs_expectations) if not result_issues: # If all validations passed successfully try: @@ -107,18 +109,16 @@ class SNameNode(Script): security_params['hdfs-site'][ 'dfs.secondary.namenode.kerberos.principal'], status_params.hostname, - status_params.tmp_dir, - 30) + status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) self.put_structured_out({"securityStateErrorInfo": str(e)}) else: - issues = "" + issues = [] for cf in result_issues: - issues += "Configuration file " + cf + " did not pass the validation. Reason: " + \ - result_issues[cf] - self.put_structured_out({"securityIssuesFound": issues}) + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py index 0e5d666..c5b0b1b 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py @@ -20,7 +20,8 @@ limitations under the License. from resource_management import * from resource_management.libraries.functions.check_process_status import check_process_status from resource_management.libraries.functions.security_commons import build_expectations, \ - cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML import utils # this is needed to avoid a circular dependency since utils.py calls this class from hdfs import hdfs @@ -90,7 +91,8 @@ class ZkfcSlave(Script): hdfs_expectations = {} hdfs_expectations.update(core_site_expectations) - security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, ['core-site.xml']) + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + {'core-site.xml': FILE_TYPE_XML}) result_issues = validate_security_config_properties(security_params, hdfs_expectations) if not result_issues: # If all validations passed successfully if status_params.hdfs_user_principal or status_params.hdfs_user_keytab: @@ -100,8 +102,7 @@ class ZkfcSlave(Script): status_params.hdfs_user_keytab, status_params.hdfs_user_principal, status_params.hostname, - status_params.tmp_dir, - 30) + status_params.tmp_dir) self.put_structured_out({"securityState": "SECURED_KERBEROS"}) except Exception as e: self.put_structured_out({"securityState": "ERROR"}) @@ -111,11 +112,10 @@ class ZkfcSlave(Script): {"securityIssuesFound": "hdfs principal and/or keytab file is not specified"}) self.put_structured_out({"securityState": "UNSECURED"}) else: - issues = "" + issues = [] for cf in result_issues: - issues += "Configuration file " + cf + " did not pass the validation. Reason: " + \ - result_issues[cf] - self.put_structured_out({"securityIssuesFound": issues}) + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"}) http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_client.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_client.py index 267e1ab..85ffba8 100644 --- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_client.py +++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_client.py @@ -18,6 +18,7 @@ See the License for the specific language governing permissions and limitations under the License. ''' +from mock.mock import patch from stacks.utils.RMFTestCase import * @@ -73,3 +74,27 @@ class TestFalconClient(RMFTestCase): owner = 'falcon' ) self.assertNoMoreResources() + + @patch("resource_management.libraries.script.Script.put_structured_out") + def test_security_status(self, put_structured_out_mock): + # Test that function works when is called with correct parameters + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_client.py", + classname="FalconClient", + command="security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"}) + + # Testing with security_enable = false + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_client.py", + classname="FalconClient", + command="security_status", + config_file="default.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py index 9b8579a..2e37e65 100644 --- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py +++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py @@ -172,4 +172,110 @@ class TestFalconServer(RMFTestCase): # 4 calls to tarfile.open (2 directories * read + write) self.assertTrue(tarfile_open_mock.called) self.assertEqual(tarfile_open_mock.call_count,4) + + @patch("resource_management.libraries.functions.security_commons.build_expectations") + @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem") + @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties") + @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor") + @patch("resource_management.libraries.script.Script.put_structured_out") + def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock): + # Test that function works when is called with correct parameters + import status_params + security_params = {} + security_params['startup'] = {} + security_params['startup']['*.falcon.service.authentication.kerberos.keytab'] = 'path/to/falcon/service/keytab' + security_params['startup']['*.falcon.service.authentication.kerberos.principal'] = 'falcon_service_keytab' + security_params['startup']['*.falcon.http.authentication.kerberos.keytab'] = 'path/to/falcon/http/keytab' + security_params['startup']['*.falcon.http.authentication.kerberos.principal'] = 'falcon_http_principal' + result_issues = [] + props_value_check = {"*.falcon.authentication.type": "kerberos", + "*.falcon.http.authentication.type": "kerberos"} + props_empty_check = ["*.falcon.service.authentication.kerberos.principal", + "*.falcon.service.authentication.kerberos.keytab", + "*.falcon.http.authentication.kerberos.principal", + "*.falcon.http.authentication.kerberos.keytab"] + + props_read_check = ["*.falcon.service.authentication.kerberos.keytab", + "*.falcon.http.authentication.kerberos.keytab"] + + get_params_mock.return_value = security_params + validate_security_config_mock.return_value = result_issues + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", + classname="FalconServer", + command="security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + + get_params_mock.assert_called_with('/etc/falcon/conf', {'startup.properties': 'PROPERTIES'}) + build_exp_mock.assert_called_with('startup', props_value_check, props_empty_check, props_read_check) + put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"}) + self.assertTrue(cached_kinit_executor_mock.call_count, 2) + cached_kinit_executor_mock.assert_called_with(status_params.kinit_path_local, + status_params.falcon_user, + security_params['startup']['*.falcon.http.authentication.kerberos.keytab'], + security_params['startup']['*.falcon.http.authentication.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + + # Testing that the exception throw by cached_executor is caught + cached_kinit_executor_mock.reset_mock() + cached_kinit_executor_mock.side_effect = Exception("Invalid command") + + try: + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", + classname="FalconServer", + command="security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + except: + self.assertTrue(True) + + # Testing with a security_params which doesn't contains startup + empty_security_params = {} + cached_kinit_executor_mock.reset_mock() + get_params_mock.reset_mock() + put_structured_out_mock.reset_mock() + get_params_mock.return_value = empty_security_params + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", + classname="FalconServer", + command="security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."}) + + # Testing with not empty result_issues + result_issues_with_params = {} + result_issues_with_params['startup']="Something bad happened" + + validate_security_config_mock.reset_mock() + get_params_mock.reset_mock() + validate_security_config_mock.return_value = result_issues_with_params + get_params_mock.return_value = security_params + + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", + classname="FalconServer", + command="security_status", + config_file="secured.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) + + # Testing with security_enable = false + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", + classname="FalconServer", + command="security_status", + config_file="default.json", + hdp_stack_version = self.STACK_VERSION, + target = RMFTestCase.TARGET_COMMON_SERVICES + ) + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e3c9aa7a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py index b8e819d..4e48f11 100644 --- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py +++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py @@ -116,8 +116,12 @@ class RMFTestCase(TestCase): # Reload params import, otherwise it won't change properties during next import if 'params' in sys.modules: - del(sys.modules["params"]) - + del(sys.modules["params"]) + + # Reload status_params import, otherwise it won't change properties during next import + if 'status_params' in sys.modules: + del(sys.modules["status_params"]) + # run with Environment(basedir, test_mode=True) as RMFTestCase.env: with patch('resource_management.core.shell.checked_call', return_value=shell_mock_value): # we must always mock any shell calls
