AMBARI-19296. When adding service, custom core-site properties get reverted back (echekanskiy via dlisnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d84aa3bc Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d84aa3bc Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d84aa3bc Branch: refs/heads/trunk Commit: d84aa3bcb184db60e8d1a68429e1c7abb3fa3646 Parents: 1bfffc0 Author: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Authored: Wed Dec 28 16:58:27 2016 +0200 Committer: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Committed: Wed Dec 28 16:58:27 2016 +0200 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 51 +++++++++++++++----- .../stacks/HDP/2.1/services/stack_advisor.py | 20 +------- .../stacks/HDP/2.3/services/stack_advisor.py | 2 +- .../stacks/HDP/2.5/services/stack_advisor.py | 20 +------- 4 files changed, 43 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d84aa3bc/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index a352cdb..77ed527 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -205,11 +205,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor): if "HDFS" in servicesList: ambari_user = self.getAmbariUser(services) ambariHostName = socket.getfqdn() - is_wildcard_value, hosts = self.get_hosts_for_proxyuser(ambari_user, services) - if not is_wildcard_value: - hosts.add(ambariHostName) - putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(ambari_user), ",".join(hosts)) - putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(ambari_user), "*") + self.put_proxyuser_value(ambari_user, ambariHostName, services=services, put_function=putCoreSiteProperty) + self.put_proxyuser_value(ambari_user, "*", is_groups=True, services=services, put_function=putCoreSiteProperty) old_ambari_user = self.getOldAmbariUser(services) if old_ambari_user is not None: putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.hosts".format(old_ambari_user), 'delete', 'true') @@ -254,7 +251,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): for proxyPropertyName, hostSelector in hostSelectorMap.iteritems(): componentHostNamesString = hostSelector if isinstance(hostSelector, basestring) else '*' if isinstance(hostSelector, (list, tuple)): - _, componentHostNames = self.get_hosts_for_proxyuser(user, services) # preserve old values + _, componentHostNames = self.get_data_for_proxyuser(user, services) # preserve old values for component in hostSelector: componentHosts = self.getHostsWithComponent(serviceName, component, services, hosts) if componentHosts is not None: @@ -288,13 +285,17 @@ class HDP206StackAdvisor(DefaultStackAdvisor): return users - def get_hosts_for_proxyuser(self, user_name, services): + def get_data_for_proxyuser(self, user_name, services, groups=False): if "core-site" in services["configurations"]: coreSite = services["configurations"]["core-site"]['properties'] else: coreSite = {} - property_name = "hadoop.proxyuser.{0}.hosts".format(user_name) + if groups: + property_name = "hadoop.proxyuser.{0}.groups".format(user_name) + else: + property_name = "hadoop.proxyuser.{0}.hosts".format(user_name) + if property_name in coreSite: property_value = coreSite[property_name] if property_value == "*": @@ -303,6 +304,32 @@ class HDP206StackAdvisor(DefaultStackAdvisor): return False, set(property_value.split(",")) return False, set() + def put_proxyuser_value(self, user_name, value, is_groups=False, services=None, put_function=None): + is_wildcard_value, current_value = self.get_data_for_proxyuser(user_name, services, is_groups) + result_value = "*" + result_values_set = self.merge_proxyusers_values(current_value, value) + if len(result_values_set) > 0: + result_value = ",".join(sorted([val for val in result_values_set if val])) + + if is_groups: + property_name = "hadoop.proxyuser.{0}.groups".format(user_name) + else: + property_name = "hadoop.proxyuser.{0}.hosts".format(user_name) + + put_function(property_name, result_value) + + def merge_proxyusers_values(self, first, second): + result = set() + def append(data): + if isinstance(data, str) or isinstance(data, unicode): + if data != "*": + result.update(data.split(",")) + else: + result.update(data) + append(first) + append(second) + return result + def getServiceHadoopProxyUsersConfigurationDict(self): """ Returns a map that is used by 'getHadoopProxyUsers' to determine service @@ -365,14 +392,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor): services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.hosts".format(hive_user)}) for user_name, user_properties in users.iteritems(): - is_wildcard_value, _ = self.get_hosts_for_proxyuser(user_name, services) # Add properties "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" to core-site for all users - if not is_wildcard_value: - putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(user_name) , user_properties["propertyHosts"]) - Logger.info("Updated hadoop.proxyuser.{0}.hosts as : {1}".format(user_name, user_properties["propertyHosts"])) + self.put_proxyuser_value(user_name, user_properties["propertyHosts"], services=services, put_function=putCoreSiteProperty) + Logger.info("Updated hadoop.proxyuser.{0}.hosts as : {1}".format(user_name, user_properties["propertyHosts"])) if "propertyGroups" in user_properties: - putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(user_name) , user_properties["propertyGroups"]) + self.put_proxyuser_value(user_name, user_properties["propertyGroups"], is_groups=True, services=services, put_function=putCoreSiteProperty) # Remove old properties if user was renamed userOldValue = getOldValue(self, services, user_properties["config"], user_properties["propertyName"]) http://git-wip-us.apache.org/repos/asf/ambari/blob/d84aa3bc/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py index 17225d0..f34f5a1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py @@ -139,27 +139,11 @@ class HDP21StackAdvisor(HDP206StackAdvisor): webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.groups".format(old_ambari_user), 'delete', 'true') if self.is_secured_cluster(services): - appendCoreSiteProperty = self.updateProperty(configurations, "core-site", services) - - def updateCallback(originalValue, newValue): - """ - :type originalValue str - :type newValue list - """ - if originalValue and not originalValue.isspace(): - hosts = originalValue.split(',') - - if newValue: - hosts.extend(newValue) - - result = ','.join(set(hosts)) - return result - else: - return ','.join(set(newValue)) + putCoreSiteProperty = self.putProperty(configurations, "core-site", services) meta = self.get_service_component_meta("HIVE", "WEBHCAT_SERVER", services) if "hostnames" in meta: - appendCoreSiteProperty('hadoop.proxyuser.HTTP.hosts', meta["hostnames"], updateCallback) + self.put_proxyuser_value("HTTP", meta["hostnames"], services=services, put_function=putCoreSiteProperty) def recommendTezConfigurations(self, configurations, clusterData, services, hosts): putTezProperty = self.putProperty(configurations, "tez-site") http://git-wip-us.apache.org/repos/asf/ambari/blob/d84aa3bc/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index d97efe2..eb81655 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -485,7 +485,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor): if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS' in servicesList: kmsUser = kmsEnvProperties['kms_user'] kmsUserOld = getOldValue(self, services, 'kms-env', 'kms_user') - putCoreSiteProperty('hadoop.proxyuser.{0}.groups'.format(kmsUser), '*') + self.put_proxyuser_value(kmsUser, '*', is_groups=True, services=services, put_function=putCoreSiteProperty) if kmsUserOld is not None and kmsUser != kmsUserOld: putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(kmsUserOld), 'delete', 'true') services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUserOld)}) http://git-wip-us.apache.org/repos/asf/ambari/blob/d84aa3bc/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index a130566..b52d753 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -618,7 +618,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts): super(HDP25StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData, services, hosts) putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services) - appendCoreSiteProperty = self.updateProperty(configurations, "core-site", services) + putCoreSiteProperty = self.putProperty(configurations, "core-site", services) if "cluster-env" in services["configurations"] \ and "security_enabled" in services["configurations"]["cluster-env"]["properties"] \ @@ -632,24 +632,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor): Logger.debug("Attempting to update hadoop.proxyuser.HTTP.hosts with %s" % str(phoenix_query_server_hosts)) # The PQS hosts we want to ensure are set new_value = ','.join(phoenix_query_server_hosts) - # Compute the unique set of hosts for the property - def updateCallback(originalValue, newValue): - Logger.debug("Original hadoop.proxyuser.HTTP.hosts value %s, appending %s" % (originalValue, newValue)) - # Only update the original value if it's not whitespace only - if originalValue and not originalValue.isspace(): - hosts = originalValue.split(',') - # Add in the new hosts if we have some - if newValue and not newValue.isspace(): - hosts.extend(newValue.split(',')) - # Return the combined (uniqued) list of hosts - result = ','.join(set(hosts)) - Logger.debug("Setting final to %s" % result) - return result - else: - Logger.debug("Setting final value to %s" % newValue) - return newValue # Update the proxyuser setting, deferring to out callback to merge results together - appendCoreSiteProperty('hadoop.proxyuser.HTTP.hosts', new_value, updateCallback) + self.put_proxyuser_value("HTTP", new_value, services=services, put_function=putCoreSiteProperty) else: Logger.debug("No phoenix query server hosts to update") else: