Repository: ambari Updated Branches: refs/heads/trunk c8c6c9ce1 -> 74dc9474b
AMBARI-13045. Handle dependencies for 'hadoop.proxyuser.**' properties in stack advisor (dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/74dc9474 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/74dc9474 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/74dc9474 Branch: refs/heads/trunk Commit: 74dc9474b600458a52e7bf038d239283e067708d Parents: c8c6c9c Author: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Authored: Wed Sep 9 15:23:09 2015 +0300 Committer: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Committed: Wed Sep 9 15:24:42 2015 +0300 ---------------------------------------------------------------------- .../stackadvisor/StackAdvisorRequest.java | 9 +- .../internal/StackAdvisorResourceProvider.java | 13 +- .../ambari/server/state/ChangedConfigInfo.java | 101 +++++++++++ .../stacks/HDP/2.0.6/services/stack_advisor.py | 79 ++++++++- .../stacks/HDP/2.1/services/stack_advisor.py | 18 ++ .../stacks/HDP/2.2/services/stack_advisor.py | 1 + .../src/main/resources/stacks/stack_advisor.py | 5 +- .../stacks/2.0.6/common/test_stack_advisor.py | 176 ++++++++++++++++++- .../stacks/2.1/common/test_stack_advisor.py | 60 ++++++- .../stacks/2.2/common/test_stack_advisor.py | 12 +- 10 files changed, 451 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/74dc9474/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java index 4db2133..b804c7e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Set; import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse; +import org.apache.ambari.server.state.ChangedConfigInfo; import org.apache.ambari.server.state.PropertyDependencyInfo; import org.apache.commons.lang.StringUtils; @@ -44,7 +45,7 @@ public class StackAdvisorRequest { private Map<String, Set<String>> hostComponents = new HashMap<String, Set<String>>(); private Map<String, Set<String>> hostGroupBindings = new HashMap<String, Set<String>>(); private Map<String, Map<String, Map<String, String>>> configurations = new HashMap<String, Map<String, Map<String, String>>>(); - private List<PropertyDependencyInfo> changedConfigurations = new LinkedList<PropertyDependencyInfo>(); + private List<ChangedConfigInfo> changedConfigurations = new LinkedList<ChangedConfigInfo>(); private Set<RecommendationResponse.ConfigGroup> configGroups; public String getStackName() { @@ -91,11 +92,11 @@ public class StackAdvisorRequest { return configurations; } - public List<PropertyDependencyInfo> getChangedConfigurations() { + public List<ChangedConfigInfo> getChangedConfigurations() { return changedConfigurations; } - public void setChangedConfigurations(List<PropertyDependencyInfo> changedConfigurations) { + public void setChangedConfigurations(List<ChangedConfigInfo> changedConfigurations) { this.changedConfigurations = changedConfigurations; } @@ -162,7 +163,7 @@ public class StackAdvisorRequest { } public StackAdvisorRequestBuilder withChangedConfigurations( - List<PropertyDependencyInfo> changedConfigurations) { + List<ChangedConfigInfo> changedConfigurations) { this.instance.changedConfigurations = changedConfigurations; return this; } http://git-wip-us.apache.org/repos/asf/ambari/blob/74dc9474/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java index 9412a2a..fe3d006 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java @@ -41,6 +41,7 @@ import org.apache.ambari.server.controller.spi.Resource.Type; import org.apache.ambari.server.controller.utilities.PropertyHelper; import com.google.inject.Inject; +import org.apache.ambari.server.state.ChangedConfigInfo; import org.apache.ambari.server.state.PropertyDependencyInfo; /** @@ -109,9 +110,9 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi hgHostsMap); Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request); - List<PropertyDependencyInfo> changedConfigurations = + List<ChangedConfigInfo> changedConfigurations = requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES ? - calculateChangedConfigurations(request) : Collections.<PropertyDependencyInfo>emptyList(); + calculateChangedConfigurations(request) : Collections.<ChangedConfigInfo>emptyList(); Set<RecommendationResponse.ConfigGroup> configGroups = calculateConfigGroups(request); return StackAdvisorRequestBuilder. @@ -193,13 +194,13 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi return map; } - protected List<PropertyDependencyInfo> calculateChangedConfigurations(Request request) { - List<PropertyDependencyInfo> configs = - new LinkedList<PropertyDependencyInfo>(); + protected List<ChangedConfigInfo> calculateChangedConfigurations(Request request) { + List<ChangedConfigInfo> configs = + new LinkedList<ChangedConfigInfo>(); HashSet<HashMap<String, String>> changedConfigs = (HashSet<HashMap<String, String>>) getRequestProperty(request, CHANGED_CONFIGURATIONS_PROPERTY); for (HashMap<String, String> props: changedConfigs) { - configs.add(new PropertyDependencyInfo(props.get("type"), props.get("name"))); + configs.add(new ChangedConfigInfo(props.get("type"), props.get("name"), props.get("old_value"))); } return configs; http://git-wip-us.apache.org/repos/asf/ambari/blob/74dc9474/ambari-server/src/main/java/org/apache/ambari/server/state/ChangedConfigInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ChangedConfigInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ChangedConfigInfo.java new file mode 100644 index 0000000..b69061f --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ChangedConfigInfo.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.state; + +import org.codehaus.jackson.annotate.JsonProperty; + +public class ChangedConfigInfo { + + private String type; + + private String name; + + @JsonProperty("old_value") + private String oldValue; + + public ChangedConfigInfo() { + + } + + public ChangedConfigInfo(String type, String name) { + this(type, name, null); + } + + public ChangedConfigInfo(String type, String name, String oldValue) { + this.type = type; + this.name = name; + this.oldValue = oldValue; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getName() { + return name; + } + + public String getOldValue() { + return oldValue; + } + + public void setOldValue(String oldValue) { + this.oldValue = oldValue; + } + + public void setName(String name) { + this.name = name; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ChangedConfigInfo that = (ChangedConfigInfo) o; + + if (type != null ? !type.equals(that.type) : that.type != null) return false; + if (name != null ? !name.equals(that.name) : that.name != null) return false; + return !(oldValue != null ? !oldValue.equals(that.oldValue) : that.oldValue != null); + + } + + @Override + public int hashCode() { + int result = type != null ? type.hashCode() : 0; + result = 31 * result + (name != null ? name.hashCode() : 0); + result = 31 * result + (oldValue != null ? oldValue.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "ChangedConfigInfo{" + + "type='" + type + '\'' + + ", name='" + name + '\'' + + ", oldValue='" + oldValue + '\'' + + '}'; + } + +} + http://git-wip-us.apache.org/repos/asf/ambari/blob/74dc9474/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 3d535de..cce0fc5 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -144,7 +144,72 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m") putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m") putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024)) - + + def recommendHadoopProxyUsers (self, configurations, services, hosts): + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + users = {} + + if 'forced-configurations' not in services: + services["forced-configurations"] = [] + + if "HDFS" in servicesList: + if "hadoop-env" in services["configurations"] and "hdfs_user" in services["configurations"]["hadoop-env"]["properties"]: + hdfs_user = services["configurations"]["hadoop-env"]["properties"]["hdfs_user"] + if not hdfs_user in users: + users[hdfs_user] = {"propertyHosts" : "*","propertyGroups" : "*", "config" : "hadoop-env", "propertyName" : "hdfs_user"} + + if "OOZIE" in servicesList: + if "oozie-env" in services["configurations"] and "oozie_user" in services["configurations"]["oozie-env"]["properties"]: + oozie_user = services["configurations"]["oozie-env"]["properties"]["oozie_user"] + oozieServerrHost = self.getHostWithComponent("OOZIE", "OOZIE_SERVER", services, hosts) + if oozieServerrHost is not None: + oozieServerHostName = oozieServerrHost["Hosts"]["public_host_name"] + if not oozie_user in users: + users[oozie_user] = {"propertyHosts" : oozieServerHostName,"propertyGroups" : "*", "config" : "oozie-env", "propertyName" : "oozie_user"} + + if "HIVE" in servicesList: + print "HIVE INSIDE" + if "hive-env" in services["configurations"] and "hive_user" in services["configurations"]["hive-env"]["properties"] \ + and "webhcat_user" in services["configurations"]["hive-env"]["properties"]: + print "HIVE INSIDE1" + hive_user = services["configurations"]["hive-env"]["properties"]["hive_user"] + webhcat_user = services["configurations"]["hive-env"]["properties"]["webhcat_user"] + if "hadoop-env" in services["configurations"] and "proxyuser_group" in services["configurations"]["hadoop-env"]["properties"]: + proxy_user_group = services["configurations"]["hadoop-env"]["properties"]["proxyuser_group"] + hiveServerrHost = self.getHostWithComponent("HIVE", "HIVE_SERVER", services, hosts) + if hiveServerrHost is not None: + hiveServerHostName = hiveServerrHost["Hosts"]["public_host_name"] + if not hive_user in users: + users[hive_user] = {"propertyHosts" : hiveServerHostName,"propertyGroups" : "*", "config" : "hive-env", "propertyName" : "hive_user"} + if not webhcat_user in users: + users[webhcat_user] = {"propertyHosts" : hiveServerHostName,"propertyGroups" : proxy_user_group, "config" : "hive-env", "propertyName" : "webhcat_user"} + + if "FALCON" in servicesList: + if "falcon-env" in services["configurations"] and "falcon_user" in services["configurations"]["falcon-env"]["properties"]: + falconUser = services["configurations"]["falcon-env"]["properties"]["falcon_user"] + if "hadoop-env" in services["configurations"] and "proxyuser_group" in services["configurations"]["hadoop-env"]["properties"]: + proxy_user_group = services["configurations"]["hadoop-env"]["properties"]["proxyuser_group"] + if not falconUser in users: + users[falconUser] = {"propertyHosts" : "*","propertyGroups" : proxy_user_group, "config" : "falcon-env", "propertyName" : "falcon_user"} + + putCoreSiteProperty = self.putProperty(configurations, "core-site", services) + putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site") + + for user_name, user_properties in users.iteritems(): + # Add properties "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" to core-site for all users + putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(user_name) , user_properties["propertyHosts"]) + putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(user_name) , user_properties["propertyGroups"]) + + # Remove old properties if user was renamed + userOldValue = getOldValue(self, services, user_properties["config"], user_properties["propertyName"]) + if userOldValue is not None: + putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.hosts".format(userOldValue), 'delete', 'true') + putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(userOldValue), 'delete', 'true') + services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.hosts".format(userOldValue)}) + services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(userOldValue)}) + services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.hosts".format(user_name)}) + services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(user_name)}) + def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts): putHDFSProperty = self.putProperty(configurations, "hadoop-env", services) putHDFSProperty('namenode_heapsize', max(int(clusterData['totalAvailableRam'] / 2), 1024)) @@ -153,6 +218,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putHDFSProperty = self.putProperty(configurations, "hadoop-env", services) putHDFSProperty('namenode_opt_maxnewsize', max(int(clusterData['totalAvailableRam'] / 8), 256)) + # recommendations for "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" properties in core-site + self.recommendHadoopProxyUsers(configurations, services, hosts) + def recommendHbaseConfigurations(self, configurations, clusterData, services, hosts): # recommendations for HBase env config putHbaseProperty = self.putProperty(configurations, "hbase-env", services) @@ -806,6 +874,15 @@ class HDP206StackAdvisor(DefaultStackAdvisor): parentValidators[service] = {} parentValidators[service].update(configsDict) +def getOldValue(self, services, configType, propertyName): + if services: + if 'changed-configurations' in services.keys(): + changedConfigs = services["changed-configurations"] + for changedConfig in changedConfigs: + if changedConfig["type"] == configType and changedConfig["name"]== propertyName and "old_value" in changedConfig: + return changedConfig["old_value"] + return None + # Validation helper methods def getSiteProperties(configurations, siteName): siteConfig = configurations.get(siteName) http://git-wip-us.apache.org/repos/asf/ambari/blob/74dc9474/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py index 1607287..2bb5aad 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py @@ -31,6 +31,24 @@ class HDP21StackAdvisor(HDP206StackAdvisor): def recommendOozieConfigurations(self, configurations, clusterData, services, hosts): if "FALCON_SERVER" in clusterData["components"]: + putOozieSiteProperty = self.putProperty(configurations, "oozie-site", services) + + if "falcon-env" in services["configurations"] and "falcon_user" in services["configurations"]["falcon-env"]["properties"]: + falconUser = services["configurations"]["falcon-env"]["properties"]["falcon_user"] + putOozieSiteProperty("oozie.service.ProxyUserService.proxyuser.{0}.groups".format(falconUser) , "*") + putOozieSiteProperty("oozie.service.ProxyUserService.proxyuser.{0}.hosts".format(falconUser) , "*") + falconUserOldValue = getOldValue(self, services, "falcon-env", "falcon_user") + if falconUserOldValue is not None: + if 'forced-configurations' not in services: + services["forced-configurations"] = [] + putOozieSitePropertyAttribute = self.putPropertyAttribute(configurations, "oozie-site") + putOozieSitePropertyAttribute("oozie.service.ProxyUserService.proxyuser.{0}.groups".format(falconUserOldValue), 'delete', 'true') + putOozieSitePropertyAttribute("oozie.service.ProxyUserService.proxyuser.{0}.hosts".format(falconUserOldValue), 'delete', 'true') + services["forced-configurations"].append({"type" : "oozie-site", "name" : "oozie.service.ProxyUserService.proxyuser.{0}.hosts".format(falconUserOldValue)}) + services["forced-configurations"].append({"type" : "oozie-site", "name" : "oozie.service.ProxyUserService.proxyuser.{0}.groups".format(falconUserOldValue)}) + services["forced-configurations"].append({"type" : "oozie-site", "name" : "oozie.service.ProxyUserService.proxyuser.{0}.hosts".format(falconUser)}) + services["forced-configurations"].append({"type" : "oozie-site", "name" : "oozie.service.ProxyUserService.proxyuser.{0}.groups".format(falconUser)}) + putMapredProperty = self.putProperty(configurations, "oozie-site") putMapredProperty("oozie.services.ext", "org.apache.oozie.service.JMSAccessorService," + http://git-wip-us.apache.org/repos/asf/ambari/blob/74dc9474/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 9a7d7a0..f09488f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -90,6 +90,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putYarnPropertyAttribute('yarn.nodemanager.linux-container-executor.cgroups.mount-path', 'delete', 'true') def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts): + super(HDP22StackAdvisor, self).recommendHDFSConfigurations(configurations, clusterData, services, hosts) putHdfsSiteProperty = self.putProperty(configurations, "hdfs-site", services) putHdfsSitePropertyAttribute = self.putPropertyAttribute(configurations, "hdfs-site") putHdfsSiteProperty("dfs.datanode.max.transfer.threads", 16384 if clusterData["hBaseInstalled"] else 4096) http://git-wip-us.apache.org/repos/asf/ambari/blob/74dc9474/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 0ef953d..d993feb 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -730,6 +730,7 @@ class DefaultStackAdvisor(StackAdvisor): def getAffectedConfigs(self, services): """returns properties dict including changed-configurations and depended-by configs""" changedConfigs = services['changed-configurations'] + changedConfigs = [{"type": entry["type"], "name": entry["name"]} for entry in changedConfigs] allDependencies = [] for item in services['services']: @@ -754,10 +755,12 @@ class DefaultStackAdvisor(StackAdvisor): if dependency not in dependencies: dependencies.append(dependency) + if "forced-configurations" in services and services["forced-configurations"] is not None: + dependencies.extend(services["forced-configurations"]) return dependencies def versionCompare(self, version1, version2): def normalize(v): return [int(x) for x in re.sub(r'(\.0+)*$','', v).split(".")] return cmp(normalize(version1), normalize(version2)) - pass \ No newline at end of file + pass http://git-wip-us.apache.org/repos/asf/ambari/blob/74dc9474/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 887e172..60ca33d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -677,7 +677,102 @@ class TestHDP206StackAdvisor(TestCase): self.assertEquals(configurations, expected) def test_recommendHDFSConfigurations(self): - configurations = {} + configurations = { + "hadoop-env": { + "properties": { + "hdfs_user": "hdfs", + "proxyuser_group": "users" + } + }, + "hive-env": { + "properties": { + "webhcat_user": "webhcat", + "hive_user": "hive" + } + }, + "oozie-env": { + "properties": { + "oozie_user": "oozie" + } + }, + "falcon-env": { + "properties": { + "falcon_user": "falcon" + } + } + } + + hosts = { + "items": [ + { + "href": "/api/v1/hosts/host1", + "Hosts": { + "cpu_count": 1, + "host_name": "c6401.ambari.apache.org", + "os_arch": "x86_64", + "os_type": "centos6", + "ph_cpu_count": 1, + "public_host_name": "c6401.ambari.apache.org", + "rack_info": "/default-rack", + "total_mem": 2097152 + } + }]} + + services = { + "services": [ + { + "StackServices": { + "service_name": "HDFS" + }, "components": [] + }, + { + "StackServices": { + "service_name": "FALCON" + }, "components": [] + }, + { + "StackServices": { + "service_name": "HIVE" + }, "components": [{ + "href": "/api/v1/stacks/HDP/versions/2.0.6/services/HIVE/components/HIVE_SERVER", + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "HIVE_SERVER", + "custom_commands": [], + "display_name": "Hive Server", + "is_client": "false", + "is_master": "true", + "service_name": "HIVE", + "stack_name": "HDP", + "stack_version": "2.0.6", + "hostnames": ["c6401.ambari.apache.org"] + }, }] + }, + { + "StackServices": { + "service_name": "OOZIE" + }, "components": [{ + "href": "/api/v1/stacks/HDP/versions/2.0.6/services/HIVE/components/OOZIE_SERVER", + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "OOZIE_SERVER", + "custom_commands": [], + "display_name": "Oozie Server", + "is_client": "false", + "is_master": "true", + "service_name": "HIVE", + "stack_name": "HDP", + "stack_version": "2.0.6", + "hostnames": ["c6401.ambari.apache.org"] + }, }] + }], + "configurations": configurations + } + clusterData = { "totalAvailableRam": 2048 } @@ -685,13 +780,86 @@ class TestHDP206StackAdvisor(TestCase): 'hadoop-env': { 'properties': { 'namenode_heapsize': '1024', - 'namenode_opt_newsize' : '256', - 'namenode_opt_maxnewsize' : '256' + 'namenode_opt_newsize': '256', + 'namenode_opt_maxnewsize': '256', + 'hdfs_user': 'hdfs', + "proxyuser_group": "users" + } + }, + "core-site": { + "properties": { + "hadoop.proxyuser.hdfs.hosts": "*", + "hadoop.proxyuser.hdfs.groups": "*", + "hadoop.proxyuser.hive.hosts": "c6401.ambari.apache.org", + "hadoop.proxyuser.hive.groups": "*", + "hadoop.proxyuser.webhcat.hosts": "c6401.ambari.apache.org", + "hadoop.proxyuser.webhcat.groups": "users", + "hadoop.proxyuser.oozie.groups": "*", + "hadoop.proxyuser.oozie.hosts": "c6401.ambari.apache.org", + "hadoop.proxyuser.falcon.groups": "users", + "hadoop.proxyuser.falcon.hosts": "*" + } + }, + "hive-env": { + "properties": { + "hive_user": "hive", + "webhcat_user": "webhcat" + } + }, + "oozie-env": { + "properties": { + "oozie_user": "oozie" + } + }, + "falcon-env": { + "properties": { + "falcon_user": "falcon" } } } - self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, '', '') + self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + + configurations["hadoop-env"]["properties"]['hdfs_user'] = "hdfs1" + + changedConfigurations = [{"type":"hadoop-env", + "name":"hdfs_user", + "old_value":"hdfs"}] + + services["changed-configurations"] = changedConfigurations + services['configurations'] = configurations + + core_site = { + "properties": { + "hadoop.proxyuser.hdfs1.hosts": "*", + "hadoop.proxyuser.hdfs1.groups": "*", + "hadoop.proxyuser.hdfs.hosts": "*", + "hadoop.proxyuser.hdfs.groups": "*", + "hadoop.proxyuser.hive.hosts": "c6401.ambari.apache.org", + "hadoop.proxyuser.hive.groups": "*", + "hadoop.proxyuser.webhcat.hosts": "c6401.ambari.apache.org", + "hadoop.proxyuser.webhcat.groups": "users", + "hadoop.proxyuser.oozie.groups": "*", + "hadoop.proxyuser.oozie.hosts": "c6401.ambari.apache.org", + "hadoop.proxyuser.falcon.groups": "users", + "hadoop.proxyuser.falcon.hosts": "*" + }, + "property_attributes": { + "hadoop.proxyuser.hdfs.hosts": { + "delete": "true" + }, + "hadoop.proxyuser.hdfs.groups": { + "delete": "true" + } + } + + } + + expected["core-site"] = core_site + expected["hadoop-env"]["properties"]["hdfs_user"] = "hdfs1" + + self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) def test_validateHDFSConfigurationsEnv(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/74dc9474/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py index 0d5caf6..1fa4e44 100644 --- a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py @@ -51,7 +51,24 @@ class TestHDP21StackAdvisor(TestCase): self.assertEquals(configurations, expected) def test_recommendOozieConfigurations_withFalconServer(self): - configurations = {} + configurations = { + "falcon-env" : { + "properties" : { + "falcon_user" : "falcon" + } + } + } + + services = { + "services": [ + { + "StackServices": { + "service_name": "FALCON" + }, "components": [] + },], + "configurations": configurations + } + clusterData = { "components" : ["FALCON_SERVER"] } @@ -60,12 +77,19 @@ class TestHDP21StackAdvisor(TestCase): "properties": { "oozie.services.ext": "org.apache.oozie.service.JMSAccessorService," + "org.apache.oozie.service.PartitionDependencyManagerService," + - "org.apache.oozie.service.HCatAccessorService" + "org.apache.oozie.service.HCatAccessorService", + "oozie.service.ProxyUserService.proxyuser.falcon.groups" : "*", + "oozie.service.ProxyUserService.proxyuser.falcon.hosts" : "*" + } + }, + "falcon-env" : { + "properties" : { + "falcon_user" : "falcon" } } } - self.stackAdvisor.recommendOozieConfigurations(configurations, clusterData, None, None) + self.stackAdvisor.recommendOozieConfigurations(configurations, clusterData, services, None) self.assertEquals(configurations, expected) def test_recommendHiveConfigurations_mapMemoryLessThan2048(self): @@ -220,7 +244,24 @@ class TestHDP21StackAdvisor(TestCase): self.assertEquals(configurations, expected) def test_recommendHDFSConfigurations(self): - configurations = {} + configurations = { + "hadoop-env": { + "properties": { + "hdfs_user": "hdfs" + } + } + } + + services = { + "services": [ + { + "StackServices": { + "service_name": "HDFS" + }, "components": [] + }], + "configurations": configurations + } + clusterData = { "totalAvailableRam": 2048 } @@ -229,12 +270,19 @@ class TestHDP21StackAdvisor(TestCase): 'properties': { 'namenode_heapsize': '1024', 'namenode_opt_newsize' : '256', - 'namenode_opt_maxnewsize' : '256' + 'namenode_opt_maxnewsize' : '256', + 'hdfs_user' : "hdfs" + } + }, + "core-site": { + "properties": { + "hadoop.proxyuser.hdfs.hosts": "*", + "hadoop.proxyuser.hdfs.groups": "*", } } } - self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, '', '') + self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, '') self.assertEquals(configurations, expected) def test_validateHDFSConfigurationsEnv(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/74dc9474/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index d0260f9..7443b51 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -2223,6 +2223,9 @@ class TestHDP22StackAdvisor(TestCase): }, 'hdfs-site': { "properties": {"dfs.datanode.data.dir": "/path/1,/path/2,/path/3,/path/4"} + }, + "hadoop-env": { + "properties": {"hdfs_user": "hdfs"} } } clusterData = { @@ -2236,7 +2239,8 @@ class TestHDP22StackAdvisor(TestCase): 'properties': { 'namenode_heapsize': '1024', 'namenode_opt_newsize' : '128', - 'namenode_opt_maxnewsize' : '128' + 'namenode_opt_maxnewsize' : '128', + "hdfs_user": "hdfs" }, 'property_attributes': { 'dtnode_heapsize': {'maximum': '2048'}, @@ -2259,6 +2263,12 @@ class TestHDP22StackAdvisor(TestCase): 'properties': { 'ranger-hdfs-plugin-enabled': 'Yes' } + }, + "core-site": { + "properties": { + "hadoop.proxyuser.hdfs.hosts": "*", + "hadoop.proxyuser.hdfs.groups": "*", + } } } services = {"services":