Repository: ambari Updated Branches: refs/heads/trunk 6e73363bb -> 1ea56f9f8
AMBARI-7464. Make stack_advisor code more human readable (srimanth) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1ea56f9f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1ea56f9f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1ea56f9f Branch: refs/heads/trunk Commit: 1ea56f9f817cc1f588093004ab033695a2b25c2b Parents: 6e73363 Author: Srimanth Gunturi <sgunt...@hortonworks.com> Authored: Tue Sep 23 19:05:17 2014 -0700 Committer: Srimanth Gunturi <sgunt...@hortonworks.com> Committed: Tue Sep 23 19:05:24 2014 -0700 ---------------------------------------------------------------------- .../src/main/resources/stacks/stack_advisor.py | 270 ++++++++++++++++++- 1 file changed, 266 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1ea56f9f/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index dca2834..f2b5446 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -20,23 +20,285 @@ limitations under the License. import socket class StackAdvisor(object): + """ + Abstract class implemented by all stack advisors. Stack advisors advise on stack specific questions. + + Currently stack advisors provide following abilities: + - Recommend where services should be installed in cluster + - Recommend configurations based on host hardware + - Validate user selection of where services are installed on cluster + - Validate user configuration values + + Each of the above methods is passed in parameters about services and hosts involved as described below. + + @type services: dictionary + @param services: Dictionary containing all information about services selected by the user. + Example: { + "services": [ + { + "StackServices": { + "service_name" : "HDFS", + "service_version" : "2.6.0.2.2", + }, + "components" : [ + { + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "SLAVE", + "component_name" : "DATANODE", + "display_name" : "DataNode", + "service_name" : "HDFS", + "hostnames" : [] + }, + "dependencies" : [] + }, { + "StackServiceComponents" : { + "cardinality" : "1-2", + "component_category" : "MASTER", + "component_name" : "NAMENODE", + "display_name" : "NameNode", + "service_name" : "HDFS", + "hostnames" : [] + }, + "dependencies" : [] + }, + ... + ] + }, + ... + ] + } + @type hosts: dictionary + @param hosts: Dictionary containing all information about hosts in this cluster + Example: { + "items": [ + { + Hosts: { + "host_name": "c6401.ambari.apache.org", + "public_host_name" : "c6401.ambari.apache.org", + "ip": "192.168.1.101", + "cpu_count" : 1, + "disk_info" : [ + { + "available" : "4564632", + "used" : "5230344", + "percent" : "54%", + "size" : "10319160", + "type" : "ext4", + "mountpoint" : "/" + }, + { + "available" : "1832436", + "used" : "0", + "percent" : "0%", + "size" : "1832436", + "type" : "tmpfs", + "mountpoint" : "/dev/shm" + } + ], + "host_state" : "HEALTHY", + "os_arch" : "x86_64", + "os_type" : "centos6", + "total_mem" : 3664872 + } + }, + ... + ] + } + + Each of the methods can either return recommendations or validations. + + Recommendations are made in a Ambari Blueprints friendly format. + Validations are an array of validation objects. + """ def recommendComponentLayout(self, services, hosts): - """Returns Services object with hostnames array populated for components""" + """ + Returns recommendation of which hosts various service components should be installed on. + + This function takes as input all details about services being installed, and hosts + they are being installed into, to generate hostname assignments to various components + of each service. + + @type services: dictionary + @param services: Dictionary containing all information about services selected by the user. + @type hosts: dictionary + @param hosts: Dictionary containing all information about hosts in this cluster + @rtype: dictionary + @return: Layout recommendation of service components on cluster hosts in Ambari Blueprints friendly format. + Example: { + "resources" : [ + { + "hosts" : [ + "c6402.ambari.apache.org", + "c6401.ambari.apache.org" + ], + "services" : [ + "HDFS" + ], + "recommendations" : { + "blueprint" : { + "host_groups" : [ + { + "name" : "host-group-2", + "components" : [ + { "name" : "JOURNALNODE" }, + { "name" : "ZKFC" }, + { "name" : "DATANODE" }, + { "name" : "SECONDARY_NAMENODE" } + ] + }, + { + "name" : "host-group-1", + "components" : [ + { "name" : "HDFS_CLIENT" }, + { "name" : "NAMENODE" }, + { "name" : "JOURNALNODE" }, + { "name" : "ZKFC" }, + { "name" : "DATANODE" } + ] + } + ] + }, + "blueprint_cluster_binding" : { + "host_groups" : [ + { + "name" : "host-group-1", + "hosts" : [ { "fqdn" : "c6401.ambari.apache.org" } ] + }, + { + "name" : "host-group-2", + "hosts" : [ { "fqdn" : "c6402.ambari.apache.org" } ] + } + ] + } + } + } + ] + } + """ pass def validateComponentLayout(self, services, hosts): - """Returns array of Validation objects about issues with hostnames components assigned to""" + """ + Returns array of Validation issues with service component layout on hosts + + This function takes as input all details about services being installed along with + hosts the components are being installed on (hostnames property is populated for + each component). + + @type services: dictionary + @param services: Dictionary containing information about services and host layout selected by the user. + @type hosts: dictionary + @param hosts: Dictionary containing all information about hosts in this cluster + @rtype: dictionary + @return: Dictionary containing array of validation items + Example: { + "items": [ + { + "type" : "host-group", + "level" : "ERROR", + "message" : "NameNode and Secondary NameNode should not be hosted on the same machine", + "component-name" : "NAMENODE", + "host" : "c6401.ambari.apache.org" + }, + ... + ] + } + """ pass def recommendConfigurations(self, services, hosts): - """Returns Services object with configurations object populated""" + """ + Returns recommendation of service configurations based on host-specific layout of components. + + This function takes as input all details about services being installed, and hosts + they are being installed into, to recommend host-specific configurations. + + @type services: dictionary + @param services: Dictionary containing all information about services and component layout selected by the user. + @type hosts: dictionary + @param hosts: Dictionary containing all information about hosts in this cluster + @rtype: dictionary + @return: Layout recommendation of service components on cluster hosts in Ambari Blueprints friendly format. + Example: { + "services": [ + "HIVE", + "TEZ", + "YARN" + ], + "recommendations": { + "blueprint": { + "host_groups": [], + "configurations": { + "yarn-site": { + "properties": { + "yarn.scheduler.minimum-allocation-mb": "682", + "yarn.scheduler.maximum-allocation-mb": "2048", + "yarn.nodemanager.resource.memory-mb": "2048" + } + }, + "tez-site": { + "properties": { + "tez.am.java.opts": "-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC", + "tez.am.resource.memory.mb": "682" + } + }, + "hive-site": { + "properties": { + "hive.tez.container.size": "682", + "hive.tez.java.opts": "-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC", + "hive.auto.convert.join.noconditionaltask.size": "238026752" + } + } + } + }, + "blueprint_cluster_binding": { + "host_groups": [] + } + }, + "hosts": [ + "c6401.ambari.apache.org", + "c6402.ambari.apache.org", + "c6403.ambari.apache.org" + ] + } + """ pass def validateConfigurations(self, services, hosts): - """Returns array of Validation objects about issues with configuration values provided in services""" + """" + Returns array of Validation issues with configurations provided by user + + This function takes as input all details about services being installed along with + configuration values entered by the user. These configurations can be validated against + service requirements, or host hardware to generate validation issues. + + @type services: dictionary + @param services: Dictionary containing information about services and user configurations. + @type hosts: dictionary + @param hosts: Dictionary containing all information about hosts in this cluster + @rtype: dictionary + @return: Dictionary containing array of validation items + Example: { + "items": [ + { + "config-type": "yarn-site", + "message": "Value is less than the recommended default of 682", + "type": "configuration", + "config-name": "yarn.scheduler.minimum-allocation-mb", + "level": "WARN" + } + ] + } + """ pass + + + + + class DefaultStackAdvisor(StackAdvisor): def recommendComponentLayout(self, services, hosts):