Repository: ambari Updated Branches: refs/heads/branch-1.7.0 cc8c8b9a7 -> c9ce6072b
http://git-wip-us.apache.org/repos/asf/ambari/blob/c9ce6072/ambari-server/src/test/python/stacks/2.2/common/2/services.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/2/services.json b/ambari-server/src/test/python/stacks/2.2/common/2/services.json new file mode 100644 index 0000000..c47f139 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.2/common/2/services.json @@ -0,0 +1,1122 @@ +{ + "href" : "/api/v1/stacks/HDP/versions/2.2?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy&services/StackServices/service_name.in(HDFS,MAPREDUCE2,YARN,TEZ,NAGIOS,GANGLIA,HIVE,HBASE,PIG,SQOOP,OOZIE,ZOOKEEPER,FALCON,STORM,FLUME,SLIDER,KNOX,KAFKA)", + "Versions" : { + "parent_stack_version" : "2.1", + "stack_name" : "HDP", + "stack_version" : "2.2", + "stack_hierarchy" : { + "stack_name" : "HDP", + "stack_versions" : [ "2.1", "2.0.6" ] + } + }, + "services" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/FALCON", + "StackServices" : { + "service_name" : "FALCON", + "service_version" : "0.6.0.2.2.9.9", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/FALCON/components/FALCON_CLIENT", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "CLIENT", + "component_name" : "FALCON_CLIENT", + "custom_commands" : [ ], + "display_name" : "Falcon Client", + "is_client" : true, + "is_master" : false, + "service_name" : "FALCON", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/FALCON/components/FALCON_SERVER", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "FALCON_SERVER", + "custom_commands" : [ ], + "display_name" : "Falcon Server", + "is_client" : false, + "is_master" : true, + "service_name" : "FALCON", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/FALCON/components/FALCON_SERVER/dependencies/OOZIE_CLIENT", + "Dependencies" : { + "component_name" : "OOZIE_CLIENT", + "dependent_component_name" : "FALCON_SERVER", + "dependent_service_name" : "FALCON", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/FALCON/components/FALCON_SERVER/dependencies/OOZIE_SERVER", + "Dependencies" : { + "component_name" : "OOZIE_SERVER", + "dependent_component_name" : "FALCON_SERVER", + "dependent_service_name" : "FALCON", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/FLUME", + "StackServices" : { + "service_name" : "FLUME", + "service_version" : "1.5.0.1.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/FLUME/components/FLUME_HANDLER", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "SLAVE", + "component_name" : "FLUME_HANDLER", + "custom_commands" : [ ], + "display_name" : "Flume", + "is_client" : false, + "is_master" : false, + "service_name" : "FLUME", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/GANGLIA", + "StackServices" : { + "service_name" : "GANGLIA", + "service_version" : "3.5.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/GANGLIA/components/GANGLIA_MONITOR", + "StackServiceComponents" : { + "cardinality" : "ALL", + "component_category" : "SLAVE", + "component_name" : "GANGLIA_MONITOR", + "custom_commands" : [ ], + "display_name" : "Ganglia Monitor", + "is_client" : false, + "is_master" : false, + "service_name" : "GANGLIA", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "auto_deploy" : { + "enabled" : true + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/GANGLIA/components/GANGLIA_SERVER", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "GANGLIA_SERVER", + "custom_commands" : [ ], + "display_name" : "Ganglia Server", + "is_client" : false, + "is_master" : true, + "service_name" : "GANGLIA", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HBASE", + "StackServices" : { + "service_name" : "HBASE", + "service_version" : "0.98.4.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HBASE/components/HBASE_CLIENT", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "CLIENT", + "component_name" : "HBASE_CLIENT", + "custom_commands" : [ ], + "display_name" : "HBase Client", + "is_client" : true, + "is_master" : false, + "service_name" : "HBASE", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HBASE/components/HBASE_MASTER", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "MASTER", + "component_name" : "HBASE_MASTER", + "custom_commands" : [ "DECOMMISSION" ], + "display_name" : "HBase Master", + "is_client" : false, + "is_master" : true, + "service_name" : "HBASE", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HBASE/components/HBASE_MASTER/dependencies/HDFS_CLIENT", + "Dependencies" : { + "component_name" : "HDFS_CLIENT", + "dependent_component_name" : "HBASE_MASTER", + "dependent_service_name" : "HBASE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HBASE/components/HBASE_MASTER/dependencies/ZOOKEEPER_SERVER", + "Dependencies" : { + "component_name" : "ZOOKEEPER_SERVER", + "dependent_component_name" : "HBASE_MASTER", + "dependent_service_name" : "HBASE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HBASE/components/HBASE_REGIONSERVER", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "SLAVE", + "component_name" : "HBASE_REGIONSERVER", + "custom_commands" : [ ], + "display_name" : "RegionServer", + "is_client" : false, + "is_master" : false, + "service_name" : "HBASE", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HDFS", + "StackServices" : { + "service_name" : "HDFS", + "service_version" : "2.6.0.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/DATANODE", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "SLAVE", + "component_name" : "DATANODE", + "custom_commands" : [ ], + "display_name" : "DataNode", + "is_client" : false, + "is_master" : false, + "service_name" : "HDFS", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/HDFS_CLIENT", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "CLIENT", + "component_name" : "HDFS_CLIENT", + "custom_commands" : [ ], + "display_name" : "HDFS Client", + "is_client" : true, + "is_master" : false, + "service_name" : "HDFS", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/JOURNALNODE", + "StackServiceComponents" : { + "cardinality" : "0+", + "component_category" : "SLAVE", + "component_name" : "JOURNALNODE", + "custom_commands" : [ ], + "display_name" : "JournalNode", + "is_client" : false, + "is_master" : false, + "service_name" : "HDFS", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/NAMENODE", + "StackServiceComponents" : { + "cardinality" : "1-2", + "component_category" : "MASTER", + "component_name" : "NAMENODE", + "custom_commands" : [ "DECOMMISSION", "REBALANCEHDFS" ], + "display_name" : "NameNode", + "is_client" : false, + "is_master" : true, + "service_name" : "HDFS", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/SECONDARY_NAMENODE", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "SECONDARY_NAMENODE", + "custom_commands" : [ ], + "display_name" : "SNameNode", + "is_client" : false, + "is_master" : true, + "service_name" : "HDFS", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/ZKFC", + "StackServiceComponents" : { + "cardinality" : "0+", + "component_category" : "SLAVE", + "component_name" : "ZKFC", + "custom_commands" : [ ], + "display_name" : "ZKFailoverController", + "is_client" : false, + "is_master" : false, + "service_name" : "HDFS", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE", + "StackServices" : { + "service_name" : "HIVE", + "service_version" : "0.14.0.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/HCAT", + "StackServiceComponents" : { + "cardinality" : null, + "component_category" : "CLIENT", + "component_name" : "HCAT", + "custom_commands" : [ ], + "display_name" : "HCat Client", + "is_client" : true, + "is_master" : false, + "service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/HIVE_CLIENT", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "CLIENT", + "component_name" : "HIVE_CLIENT", + "custom_commands" : [ ], + "display_name" : "Hive Client", + "is_client" : true, + "is_master" : false, + "service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/HIVE_METASTORE", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "HIVE_METASTORE", + "custom_commands" : [ ], + "display_name" : "Hive Metastore", + "is_client" : false, + "is_master" : true, + "service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "auto_deploy" : { + "enabled" : true, + "location" : "HIVE/HIVE_SERVER" + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/HIVE_SERVER", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "HIVE_SERVER", + "custom_commands" : [ ], + "display_name" : "HiveServer2", + "is_client" : false, + "is_master" : true, + "service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/HIVE_SERVER/dependencies/MAPREDUCE2_CLIENT", + "Dependencies" : { + "component_name" : "MAPREDUCE2_CLIENT", + "dependent_component_name" : "HIVE_SERVER", + "dependent_service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/HIVE_SERVER/dependencies/TEZ_CLIENT", + "Dependencies" : { + "component_name" : "TEZ_CLIENT", + "dependent_component_name" : "HIVE_SERVER", + "dependent_service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/HIVE_SERVER/dependencies/YARN_CLIENT", + "Dependencies" : { + "component_name" : "YARN_CLIENT", + "dependent_component_name" : "HIVE_SERVER", + "dependent_service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/HIVE_SERVER/dependencies/ZOOKEEPER_SERVER", + "Dependencies" : { + "component_name" : "ZOOKEEPER_SERVER", + "dependent_component_name" : "HIVE_SERVER", + "dependent_service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/MYSQL_SERVER", + "StackServiceComponents" : { + "cardinality" : "0-1", + "component_category" : "MASTER", + "component_name" : "MYSQL_SERVER", + "custom_commands" : [ ], + "display_name" : "MySQL Server", + "is_client" : false, + "is_master" : true, + "service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/WEBHCAT_SERVER", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "WEBHCAT_SERVER", + "custom_commands" : [ ], + "display_name" : "WebHCat Server", + "is_client" : false, + "is_master" : true, + "service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/WEBHCAT_SERVER/dependencies/HDFS_CLIENT", + "Dependencies" : { + "component_name" : "HDFS_CLIENT", + "dependent_component_name" : "WEBHCAT_SERVER", + "dependent_service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/WEBHCAT_SERVER/dependencies/MAPREDUCE2_CLIENT", + "Dependencies" : { + "component_name" : "MAPREDUCE2_CLIENT", + "dependent_component_name" : "WEBHCAT_SERVER", + "dependent_service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/WEBHCAT_SERVER/dependencies/YARN_CLIENT", + "Dependencies" : { + "component_name" : "YARN_CLIENT", + "dependent_component_name" : "WEBHCAT_SERVER", + "dependent_service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/WEBHCAT_SERVER/dependencies/ZOOKEEPER_CLIENT", + "Dependencies" : { + "component_name" : "ZOOKEEPER_CLIENT", + "dependent_component_name" : "WEBHCAT_SERVER", + "dependent_service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/HIVE/components/WEBHCAT_SERVER/dependencies/ZOOKEEPER_SERVER", + "Dependencies" : { + "component_name" : "ZOOKEEPER_SERVER", + "dependent_component_name" : "WEBHCAT_SERVER", + "dependent_service_name" : "HIVE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/KAFKA", + "StackServices" : { + "service_name" : "KAFKA", + "service_version" : "0.8.1.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/KAFKA/components/KAFKA_BROKER", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "MASTER", + "component_name" : "KAFKA_BROKER", + "custom_commands" : [ ], + "display_name" : "Kafka Broker", + "is_client" : false, + "is_master" : true, + "service_name" : "KAFKA", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/KAFKA/components/KAFKA_BROKER/dependencies/ZOOKEEPER_SERVER", + "Dependencies" : { + "component_name" : "ZOOKEEPER_SERVER", + "dependent_component_name" : "KAFKA_BROKER", + "dependent_service_name" : "KAFKA", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/KNOX", + "StackServices" : { + "service_name" : "KNOX", + "service_version" : "0.5.0.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/KNOX/components/KNOX_GATEWAY", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "MASTER", + "component_name" : "KNOX_GATEWAY", + "custom_commands" : [ "STARTDEMOLDAP", "STOPDEMOLDAP" ], + "display_name" : null, + "is_client" : false, + "is_master" : true, + "service_name" : "KNOX", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2", + "StackServices" : { + "service_name" : "MAPREDUCE2", + "service_version" : "2.6.0.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/HISTORYSERVER", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "HISTORYSERVER", + "custom_commands" : [ ], + "display_name" : "History Server", + "is_client" : false, + "is_master" : true, + "service_name" : "MAPREDUCE2", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "auto_deploy" : { + "enabled" : true, + "location" : "YARN/RESOURCEMANAGER" + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/HISTORYSERVER/dependencies/HDFS_CLIENT", + "Dependencies" : { + "component_name" : "HDFS_CLIENT", + "dependent_component_name" : "HISTORYSERVER", + "dependent_service_name" : "MAPREDUCE2", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT", + "StackServiceComponents" : { + "cardinality" : "0+", + "component_category" : "CLIENT", + "component_name" : "MAPREDUCE2_CLIENT", + "custom_commands" : [ ], + "display_name" : "MapReduce2 Client", + "is_client" : true, + "is_master" : false, + "service_name" : "MAPREDUCE2", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/NAGIOS", + "StackServices" : { + "service_name" : "NAGIOS", + "service_version" : "3.5.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/NAGIOS/components/NAGIOS_SERVER", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "NAGIOS_SERVER", + "custom_commands" : [ ], + "display_name" : "Nagios Server", + "is_client" : false, + "is_master" : true, + "service_name" : "NAGIOS", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/NAGIOS/components/NAGIOS_SERVER/dependencies/HCAT", + "Dependencies" : { + "component_name" : "HCAT", + "dependent_component_name" : "NAGIOS_SERVER", + "dependent_service_name" : "NAGIOS", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/NAGIOS/components/NAGIOS_SERVER/dependencies/HDFS_CLIENT", + "Dependencies" : { + "component_name" : "HDFS_CLIENT", + "dependent_component_name" : "NAGIOS_SERVER", + "dependent_service_name" : "NAGIOS", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/NAGIOS/components/NAGIOS_SERVER/dependencies/MAPREDUCE2_CLIENT", + "Dependencies" : { + "component_name" : "MAPREDUCE2_CLIENT", + "dependent_component_name" : "NAGIOS_SERVER", + "dependent_service_name" : "NAGIOS", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/NAGIOS/components/NAGIOS_SERVER/dependencies/OOZIE_CLIENT", + "Dependencies" : { + "component_name" : "OOZIE_CLIENT", + "dependent_component_name" : "NAGIOS_SERVER", + "dependent_service_name" : "NAGIOS", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/NAGIOS/components/NAGIOS_SERVER/dependencies/TEZ_CLIENT", + "Dependencies" : { + "component_name" : "TEZ_CLIENT", + "dependent_component_name" : "NAGIOS_SERVER", + "dependent_service_name" : "NAGIOS", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/NAGIOS/components/NAGIOS_SERVER/dependencies/YARN_CLIENT", + "Dependencies" : { + "component_name" : "YARN_CLIENT", + "dependent_component_name" : "NAGIOS_SERVER", + "dependent_service_name" : "NAGIOS", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/OOZIE", + "StackServices" : { + "service_name" : "OOZIE", + "service_version" : "4.1.0.2.2.9.9", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/OOZIE/components/OOZIE_CLIENT", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "CLIENT", + "component_name" : "OOZIE_CLIENT", + "custom_commands" : [ ], + "display_name" : "Oozie Client", + "is_client" : true, + "is_master" : false, + "service_name" : "OOZIE", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/OOZIE/components/OOZIE_CLIENT/dependencies/HDFS_CLIENT", + "Dependencies" : { + "component_name" : "HDFS_CLIENT", + "dependent_component_name" : "OOZIE_CLIENT", + "dependent_service_name" : "OOZIE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/OOZIE/components/OOZIE_CLIENT/dependencies/MAPREDUCE2_CLIENT", + "Dependencies" : { + "component_name" : "MAPREDUCE2_CLIENT", + "dependent_component_name" : "OOZIE_CLIENT", + "dependent_service_name" : "OOZIE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/OOZIE/components/OOZIE_SERVER", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "OOZIE_SERVER", + "custom_commands" : [ ], + "display_name" : "Oozie Server", + "is_client" : false, + "is_master" : true, + "service_name" : "OOZIE", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/OOZIE/components/OOZIE_SERVER/dependencies/HDFS_CLIENT", + "Dependencies" : { + "component_name" : "HDFS_CLIENT", + "dependent_component_name" : "OOZIE_SERVER", + "dependent_service_name" : "OOZIE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/OOZIE/components/OOZIE_SERVER/dependencies/MAPREDUCE2_CLIENT", + "Dependencies" : { + "component_name" : "MAPREDUCE2_CLIENT", + "dependent_component_name" : "OOZIE_SERVER", + "dependent_service_name" : "OOZIE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/OOZIE/components/OOZIE_SERVER/dependencies/YARN_CLIENT", + "Dependencies" : { + "component_name" : "YARN_CLIENT", + "dependent_component_name" : "OOZIE_SERVER", + "dependent_service_name" : "OOZIE", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/PIG", + "StackServices" : { + "service_name" : "PIG", + "service_version" : "0.14.0.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/PIG/components/PIG", + "StackServiceComponents" : { + "cardinality" : "0+", + "component_category" : "CLIENT", + "component_name" : "PIG", + "custom_commands" : [ ], + "display_name" : "Pig", + "is_client" : true, + "is_master" : false, + "service_name" : "PIG", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/SLIDER", + "StackServices" : { + "service_name" : "SLIDER", + "service_version" : "0.60.0.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/SLIDER/components/SLIDER", + "StackServiceComponents" : { + "cardinality" : "0+", + "component_category" : "CLIENT", + "component_name" : "SLIDER", + "custom_commands" : [ ], + "display_name" : "Slider", + "is_client" : true, + "is_master" : false, + "service_name" : "SLIDER", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/SLIDER/components/SLIDER/dependencies/HDFS_CLIENT", + "Dependencies" : { + "component_name" : "HDFS_CLIENT", + "dependent_component_name" : "SLIDER", + "dependent_service_name" : "SLIDER", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/SLIDER/components/SLIDER/dependencies/YARN_CLIENT", + "Dependencies" : { + "component_name" : "YARN_CLIENT", + "dependent_component_name" : "SLIDER", + "dependent_service_name" : "SLIDER", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/SQOOP", + "StackServices" : { + "service_name" : "SQOOP", + "service_version" : "1.4.5.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/SQOOP/components/SQOOP", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "CLIENT", + "component_name" : "SQOOP", + "custom_commands" : [ ], + "display_name" : "Sqoop", + "is_client" : true, + "is_master" : false, + "service_name" : "SQOOP", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/SQOOP/components/SQOOP/dependencies/HDFS_CLIENT", + "Dependencies" : { + "component_name" : "HDFS_CLIENT", + "dependent_component_name" : "SQOOP", + "dependent_service_name" : "SQOOP", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/SQOOP/components/SQOOP/dependencies/MAPREDUCE2_CLIENT", + "Dependencies" : { + "component_name" : "MAPREDUCE2_CLIENT", + "dependent_component_name" : "SQOOP", + "dependent_service_name" : "SQOOP", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/STORM", + "StackServices" : { + "service_name" : "STORM", + "service_version" : "0.9.3.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/STORM/components/DRPC_SERVER", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "DRPC_SERVER", + "custom_commands" : [ ], + "display_name" : "DRPC Server", + "is_client" : false, + "is_master" : true, + "service_name" : "STORM", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/STORM/components/NIMBUS", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "NIMBUS", + "custom_commands" : [ ], + "display_name" : "Nimbus", + "is_client" : false, + "is_master" : true, + "service_name" : "STORM", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/STORM/components/NIMBUS/dependencies/ZOOKEEPER_SERVER", + "Dependencies" : { + "component_name" : "ZOOKEEPER_SERVER", + "dependent_component_name" : "NIMBUS", + "dependent_service_name" : "STORM", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/STORM/components/STORM_UI_SERVER", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "STORM_UI_SERVER", + "custom_commands" : [ ], + "display_name" : "Storm UI Server", + "is_client" : false, + "is_master" : true, + "service_name" : "STORM", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/STORM/components/SUPERVISOR", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "SLAVE", + "component_name" : "SUPERVISOR", + "custom_commands" : [ ], + "display_name" : "Supervisor", + "is_client" : false, + "is_master" : false, + "service_name" : "STORM", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/TEZ", + "StackServices" : { + "service_name" : "TEZ", + "service_version" : "0.6.0.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/TEZ/components/TEZ_CLIENT", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "CLIENT", + "component_name" : "TEZ_CLIENT", + "custom_commands" : [ ], + "display_name" : "Tez Client", + "is_client" : true, + "is_master" : false, + "service_name" : "TEZ", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/YARN", + "StackServices" : { + "service_name" : "YARN", + "service_version" : "2.6.0.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/YARN/components/APP_TIMELINE_SERVER", + "StackServiceComponents" : { + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "APP_TIMELINE_SERVER", + "custom_commands" : [ ], + "display_name" : "App Timeline Server", + "is_client" : false, + "is_master" : true, + "service_name" : "YARN", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/YARN/components/NODEMANAGER", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "SLAVE", + "component_name" : "NODEMANAGER", + "custom_commands" : [ ], + "display_name" : "NodeManager", + "is_client" : false, + "is_master" : false, + "service_name" : "YARN", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/YARN/components/RESOURCEMANAGER", + "StackServiceComponents" : { + "cardinality" : "1-2", + "component_category" : "MASTER", + "component_name" : "RESOURCEMANAGER", + "custom_commands" : [ "DECOMMISSION", "REFRESHQUEUES" ], + "display_name" : "ResourceManager", + "is_client" : false, + "is_master" : true, + "service_name" : "YARN", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/YARN/components/YARN_CLIENT", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "CLIENT", + "component_name" : "YARN_CLIENT", + "custom_commands" : [ ], + "display_name" : "YARN Client", + "is_client" : true, + "is_master" : false, + "service_name" : "YARN", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + } ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/ZOOKEEPER", + "StackServices" : { + "service_name" : "ZOOKEEPER", + "service_version" : "3.4.5.2.2.0.0", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/ZOOKEEPER/components/ZOOKEEPER_CLIENT", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "CLIENT", + "component_name" : "ZOOKEEPER_CLIENT", + "custom_commands" : [ ], + "display_name" : "ZooKeeper Client", + "is_client" : true, + "is_master" : false, + "service_name" : "ZOOKEEPER", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/ZOOKEEPER/components/ZOOKEEPER_SERVER", + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "MASTER", + "component_name" : "ZOOKEEPER_SERVER", + "custom_commands" : [ ], + "display_name" : "ZooKeeper Server", + "is_client" : false, + "is_master" : true, + "service_name" : "ZOOKEEPER", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ ] + }, + "dependencies" : [ ] + } ] + } ], + "configurations" : { } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c9ce6072/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor_perf.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor_perf.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor_perf.py new file mode 100644 index 0000000..5b8c30c --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor_perf.py @@ -0,0 +1,69 @@ +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import json +import os +import time +import imp +from unittest import TestCase +from mock.mock import patch + +class TestHDP22StackAdvisor(TestCase): + + def instantiate_stack_advisor(self, testDirectory, base_stack_advisor_path): + hdp_206_stack_advisor_path = os.path.join(testDirectory, '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py') + hdp_21_stack_advisor_path = os.path.join(testDirectory, '../../../../../main/resources/stacks/HDP/2.1/services/stack_advisor.py') + hdp_22_stack_advisor_path = os.path.join(testDirectory, '../../../../../main/resources/stacks/HDP/2.2/services/stack_advisor.py') + hdp_206_stack_advisor_classname = 'HDP22StackAdvisor' + with open(base_stack_advisor_path, 'rb') as fp: + imp.load_module('stack_advisor', fp, base_stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE)) + with open(hdp_206_stack_advisor_path, 'rb') as fp: + imp.load_module('stack_advisor_impl', fp, hdp_206_stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE)) + with open(hdp_21_stack_advisor_path, 'rb') as fp: + imp.load_module('stack_advisor_impl', fp, hdp_21_stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE)) + with open(hdp_22_stack_advisor_path, 'rb') as fp: + stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp_22_stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE)) + clazz = getattr(stack_advisor_impl, hdp_206_stack_advisor_classname) + return clazz() + + @patch('socket.getfqdn') + def test_performance(self, getfqdn_method): + getfqdn_method.side_effect = lambda host='perf400-a-1.c.pramod-thangali.internal': host + testDirectory = os.path.dirname(os.path.abspath(__file__)) + old_stack_advisor_path = os.path.join(testDirectory, '../../../../../test/resources/stacks/old_stack_advisor.py') + current_stack_advisor_path = os.path.join(testDirectory, '../../../../../main/resources/stacks/stack_advisor.py') + + for folder_name in ['1', '2']: + stack_advisor_old = self.instantiate_stack_advisor(testDirectory, old_stack_advisor_path) + services = json.load(open(os.path.join(testDirectory, folder_name + '/services.json'))) + hosts = json.load(open(os.path.join(testDirectory, folder_name + '/hosts.json'))) + start = time.time() + recommendation_old = stack_advisor_old.recommendComponentLayout(services, hosts) + time_taken = time.time() - start + print "time taken by old stack_advisor.py = " + str(time_taken) + + stack_advisor = self.instantiate_stack_advisor(testDirectory, current_stack_advisor_path) + start = time.time() + recommendation = stack_advisor.recommendComponentLayout(services, hosts) + time_taken = time.time() - start + print "time taken by current stack_advisor.py = " + str(time_taken) + + self.assertEquals(recommendation, recommendation_old, + "current stack_advisor gives different results running on folder '" + folder_name + "'") + + http://git-wip-us.apache.org/repos/asf/ambari/blob/c9ce6072/ambari-server/src/test/resources/stacks/old_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/old_stack_advisor.py b/ambari-server/src/test/resources/stacks/old_stack_advisor.py new file mode 100644 index 0000000..8a880e0 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/old_stack_advisor.py @@ -0,0 +1,581 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import socket + +class StackAdvisor(object): + """ + Abstract class implemented by all stack advisors. Stack advisors advise on stack specific questions. + + Currently stack advisors provide following abilities: + - Recommend where services should be installed in cluster + - Recommend configurations based on host hardware + - Validate user selection of where services are installed on cluster + - Validate user configuration values + + Each of the above methods is passed in parameters about services and hosts involved as described below. + + @type services: dictionary + @param services: Dictionary containing all information about services selected by the user. + Example: { + "services": [ + { + "StackServices": { + "service_name" : "HDFS", + "service_version" : "2.6.0.2.2", + }, + "components" : [ + { + "StackServiceComponents" : { + "cardinality" : "1+", + "component_category" : "SLAVE", + "component_name" : "DATANODE", + "display_name" : "DataNode", + "service_name" : "HDFS", + "hostnames" : [] + }, + "dependencies" : [] + }, { + "StackServiceComponents" : { + "cardinality" : "1-2", + "component_category" : "MASTER", + "component_name" : "NAMENODE", + "display_name" : "NameNode", + "service_name" : "HDFS", + "hostnames" : [] + }, + "dependencies" : [] + }, + ... + ] + }, + ... + ] + } + @type hosts: dictionary + @param hosts: Dictionary containing all information about hosts in this cluster + Example: { + "items": [ + { + Hosts: { + "host_name": "c6401.ambari.apache.org", + "public_host_name" : "c6401.ambari.apache.org", + "ip": "192.168.1.101", + "cpu_count" : 1, + "disk_info" : [ + { + "available" : "4564632", + "used" : "5230344", + "percent" : "54%", + "size" : "10319160", + "type" : "ext4", + "mountpoint" : "/" + }, + { + "available" : "1832436", + "used" : "0", + "percent" : "0%", + "size" : "1832436", + "type" : "tmpfs", + "mountpoint" : "/dev/shm" + } + ], + "host_state" : "HEALTHY", + "os_arch" : "x86_64", + "os_type" : "centos6", + "total_mem" : 3664872 + } + }, + ... + ] + } + + Each of the methods can either return recommendations or validations. + + Recommendations are made in a Ambari Blueprints friendly format. + Validations are an array of validation objects. + """ + + def recommendComponentLayout(self, services, hosts): + """ + Returns recommendation of which hosts various service components should be installed on. + + This function takes as input all details about services being installed, and hosts + they are being installed into, to generate hostname assignments to various components + of each service. + + @type services: dictionary + @param services: Dictionary containing all information about services selected by the user. + @type hosts: dictionary + @param hosts: Dictionary containing all information about hosts in this cluster + @rtype: dictionary + @return: Layout recommendation of service components on cluster hosts in Ambari Blueprints friendly format. + Example: { + "resources" : [ + { + "hosts" : [ + "c6402.ambari.apache.org", + "c6401.ambari.apache.org" + ], + "services" : [ + "HDFS" + ], + "recommendations" : { + "blueprint" : { + "host_groups" : [ + { + "name" : "host-group-2", + "components" : [ + { "name" : "JOURNALNODE" }, + { "name" : "ZKFC" }, + { "name" : "DATANODE" }, + { "name" : "SECONDARY_NAMENODE" } + ] + }, + { + "name" : "host-group-1", + "components" : [ + { "name" : "HDFS_CLIENT" }, + { "name" : "NAMENODE" }, + { "name" : "JOURNALNODE" }, + { "name" : "ZKFC" }, + { "name" : "DATANODE" } + ] + } + ] + }, + "blueprint_cluster_binding" : { + "host_groups" : [ + { + "name" : "host-group-1", + "hosts" : [ { "fqdn" : "c6401.ambari.apache.org" } ] + }, + { + "name" : "host-group-2", + "hosts" : [ { "fqdn" : "c6402.ambari.apache.org" } ] + } + ] + } + } + } + ] + } + """ + pass + + def validateComponentLayout(self, services, hosts): + """ + Returns array of Validation issues with service component layout on hosts + + This function takes as input all details about services being installed along with + hosts the components are being installed on (hostnames property is populated for + each component). + + @type services: dictionary + @param services: Dictionary containing information about services and host layout selected by the user. + @type hosts: dictionary + @param hosts: Dictionary containing all information about hosts in this cluster + @rtype: dictionary + @return: Dictionary containing array of validation items + Example: { + "items": [ + { + "type" : "host-group", + "level" : "ERROR", + "message" : "NameNode and Secondary NameNode should not be hosted on the same machine", + "component-name" : "NAMENODE", + "host" : "c6401.ambari.apache.org" + }, + ... + ] + } + """ + pass + + def recommendConfigurations(self, services, hosts): + """ + Returns recommendation of service configurations based on host-specific layout of components. + + This function takes as input all details about services being installed, and hosts + they are being installed into, to recommend host-specific configurations. + + @type services: dictionary + @param services: Dictionary containing all information about services and component layout selected by the user. + @type hosts: dictionary + @param hosts: Dictionary containing all information about hosts in this cluster + @rtype: dictionary + @return: Layout recommendation of service components on cluster hosts in Ambari Blueprints friendly format. + Example: { + "services": [ + "HIVE", + "TEZ", + "YARN" + ], + "recommendations": { + "blueprint": { + "host_groups": [], + "configurations": { + "yarn-site": { + "properties": { + "yarn.scheduler.minimum-allocation-mb": "682", + "yarn.scheduler.maximum-allocation-mb": "2048", + "yarn.nodemanager.resource.memory-mb": "2048" + } + }, + "tez-site": { + "properties": { + "tez.am.java.opts": "-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC", + "tez.am.resource.memory.mb": "682" + } + }, + "hive-site": { + "properties": { + "hive.tez.container.size": "682", + "hive.tez.java.opts": "-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC", + "hive.auto.convert.join.noconditionaltask.size": "238026752" + } + } + } + }, + "blueprint_cluster_binding": { + "host_groups": [] + } + }, + "hosts": [ + "c6401.ambari.apache.org", + "c6402.ambari.apache.org", + "c6403.ambari.apache.org" + ] + } + """ + pass + + def validateConfigurations(self, services, hosts): + """" + Returns array of Validation issues with configurations provided by user + + This function takes as input all details about services being installed along with + configuration values entered by the user. These configurations can be validated against + service requirements, or host hardware to generate validation issues. + + @type services: dictionary + @param services: Dictionary containing information about services and user configurations. + @type hosts: dictionary + @param hosts: Dictionary containing all information about hosts in this cluster + @rtype: dictionary + @return: Dictionary containing array of validation items + Example: { + "items": [ + { + "config-type": "yarn-site", + "message": "Value is less than the recommended default of 682", + "type": "configuration", + "config-name": "yarn.scheduler.minimum-allocation-mb", + "level": "WARN" + } + ] + } + """ + pass + + + + + + +class DefaultStackAdvisor(StackAdvisor): + """ + Default stack advisor implementation. + + This implementation is used when a stack-version, or its hierarchy does not + have an advisor. Stack-versions can extend this class to provide their own + implement + """ + + def recommendComponentLayout(self, services, hosts): + """Returns Services object with hostnames array populated for components""" + + stackName = services["Versions"]["stack_name"] + stackVersion = services["Versions"]["stack_version"] + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + + layoutRecommendations = self.createComponentLayoutRecommendations(services, hosts) + + recommendations = { + "Versions": {"stack_name": stackName, "stack_version": stackVersion}, + "hosts": hostsList, + "services": servicesList, + "recommendations": layoutRecommendations + } + + return recommendations + + def createComponentLayoutRecommendations(self, services, hosts): + + recommendations = { + "blueprint": { + "host_groups": [ ] + }, + "blueprint_cluster_binding": { + "host_groups": [ ] + } + } + + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + + hostsComponentsMap = {} + for hostName in hostsList: + if hostName not in hostsComponentsMap: + hostsComponentsMap[hostName] = [] + + #extend 'hostsComponentsMap' with MASTER components + for service in services["services"]: + masterComponents = [component for component in service["components"] if self.isMasterComponent(component)] + for component in masterComponents: + componentName = component["StackServiceComponents"]["component_name"] + + if self.isComponentHostsPopulated(component): + hostsForComponent = component["StackServiceComponents"]["hostnames"] + else: + availableHosts = hostsList + if len(hostsList) > 1 and self.isComponentNotPreferableOnAmbariServerHost(component): + availableHosts = [hostName for hostName in hostsList if not self.isLocalHost(hostName)] + + if self.isMasterComponentWithMultipleInstances(component): + hostsCount = self.getMinComponentCount(component) + if hostsCount > 1: # get first 'hostsCount' available hosts + if len(availableHosts) < hostsCount: + hostsCount = len(availableHosts) + hostsForComponent = availableHosts[:hostsCount] + else: + hostsForComponent = [self.getHostForComponent(component, availableHosts)] + else: + hostsForComponent = [self.getHostForComponent(component, availableHosts)] + + #extend 'hostsComponentsMap' with 'hostsForComponent' + for hostName in hostsForComponent: + hostsComponentsMap[hostName].append( { "name":componentName } ) + + #extend 'hostsComponentsMap' with Slave and Client Components + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)] + utilizedHosts = [item for sublist in usedHostsListList for item in sublist] + freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts] + + for service in services["services"]: + slaveClientComponents = [component for component in service["components"] + if self.isSlaveComponent(component) or self.isClientComponent(component)] + for component in slaveClientComponents: + componentName = component["StackServiceComponents"]["component_name"] + + if self.isComponentHostsPopulated(component): + hostsForComponent = component["StackServiceComponents"]["hostnames"] + elif component["StackServiceComponents"]["cardinality"] == "ALL": + hostsForComponent = hostsList + else: + if len(freeHosts) == 0: + hostsForComponent = hostsList[-1:] + else: # len(freeHosts) >= 1 + hostsForComponent = freeHosts + if self.isClientComponent(component): + hostsForComponent = freeHosts[0:1] + + #extend 'hostsComponentsMap' with 'hostsForComponent' + for hostName in hostsForComponent: + if hostName not in hostsComponentsMap: + hostsComponentsMap[hostName] = [] + hostsComponentsMap[hostName].append( { "name": componentName } ) + + #prepare 'host-group's from 'hostsComponentsMap' + host_groups = recommendations["blueprint"]["host_groups"] + bindings = recommendations["blueprint_cluster_binding"]["host_groups"] + index = 0 + for key in hostsComponentsMap.keys(): + index += 1 + host_group_name = "host-group-{0}".format(index) + host_groups.append( { "name": host_group_name, "components": hostsComponentsMap[key] } ) + bindings.append( { "name": host_group_name, "hosts": [{ "fqdn": socket.getfqdn(key) }] } ) + + return recommendations + pass + + def createValidationResponse(self, services, validationItems): + """Returns array of Validation objects about issues with hostnames components assigned to""" + stackName = services["Versions"]["stack_name"] + stackVersion = services["Versions"]["stack_version"] + + validations = { + "Versions": {"stack_name": stackName, "stack_version": stackVersion}, + "items": validationItems + } + + return validations + + def validateComponentLayout(self, services, hosts): + """Returns array of Validation objects about issues with hostnames components assigned to""" + validationItems = self.getComponentLayoutValidations(services, hosts) + return self.createValidationResponse(services, validationItems) + + def validateConfigurations(self, services, hosts): + """Returns array of Validation objects about issues with hostnames components assigned to""" + validationItems = self.getConfigurationsValidationItems(services, hosts) + return self.createValidationResponse(services, validationItems) + + def getComponentLayoutValidations(self, services, hosts): + return [] + + def getConfigurationClusterSummary(self, servicesList, hosts, components): + pass + + def getConfigurationsValidationItems(self, services, hosts): + return [] + + def recommendConfigurations(self, services, hosts): + stackName = services["Versions"]["stack_name"] + stackVersion = services["Versions"]["stack_version"] + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + components = [component["StackServiceComponents"]["component_name"] + for service in services["services"] + for component in service["components"]] + + clusterSummary = self.getConfigurationClusterSummary(servicesList, hosts, components) + + recommendations = { + "Versions": {"stack_name": stackName, "stack_version": stackVersion}, + "hosts": hostsList, + "services": servicesList, + "recommendations": { + "blueprint": { + "configurations": {}, + "host_groups": [] + }, + "blueprint_cluster_binding": { + "host_groups": [] + } + } + } + + configurations = recommendations["recommendations"]["blueprint"]["configurations"] + + for service in servicesList: + calculation = self.getServiceConfigurationRecommender(service) + if calculation is not None: + calculation(configurations, clusterSummary) + + return recommendations + + def getServiceConfigurationRecommender(self, service): + return self.getServiceConfigurationRecommenderDict().get(service, None) + + def getServiceConfigurationRecommenderDict(self): + return {} + + # Recommendation helper methods + def isComponentHostsPopulated(self, component): + hostnames = self.getComponentAttribute(component, "hostnames") + if hostnames is not None: + return len(hostnames) > 0 + return False + + def isClientComponent(self, component): + return self.getComponentAttribute(component, "component_category") == 'CLIENT' + + def isSlaveComponent(self, component): + return self.getComponentAttribute(component, "component_category") == 'SLAVE' + + def isMasterComponent(self, component): + return self.getComponentAttribute(component, "is_master") + + def getComponentAttribute(self, component, attribute): + serviceComponent = component.get("StackServiceComponents", None) + if serviceComponent is None: + return None + return serviceComponent.get(attribute, None) + + def isLocalHost(self, hostName): + return socket.getfqdn(hostName) == socket.getfqdn() + + def isMasterComponentWithMultipleInstances(self, component): + componentName = self.getComponentName(component) + masters = self.getMastersWithMultipleInstances() + return componentName in masters + + def isComponentNotValuable(self, component): + componentName = self.getComponentName(component) + service = self.getNotValuableComponents() + return componentName in service + + def getMinComponentCount(self, component): + componentName = self.getComponentName(component) + return self.getComponentCardinality(componentName)["min"] + + # Helper dictionaries + def getComponentCardinality(self, componentName): + return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1}) + + def getHostForComponent(self, component, hostsList): + componentName = self.getComponentName(component) + + if len(hostsList) != 1: + scheme = self.getComponentLayoutScheme(componentName) + if scheme is not None: + for key in scheme.keys(): + if isinstance(key, ( int, long )): + if len(hostsList) < key: + return hostsList[scheme[key]] + return hostsList[scheme['else']] + return hostsList[0] + + def getComponentLayoutScheme(self, componentName): + """ + Provides a scheme for laying out given component on different number of hosts. + """ + return self.getComponentLayoutSchemes().get(componentName, None) + + def getComponentName(self, component): + return self.getComponentAttribute(component, "component_name") + + def isComponentNotPreferableOnAmbariServerHost(self, component): + componentName = self.getComponentName(component) + service = self.getNotPreferableOnServerComponents() + return componentName in service + + def getMastersWithMultipleInstances(self): + return [] + + def getNotValuableComponents(self): + return [] + + def getNotPreferableOnServerComponents(self): + return [] + + def getCardinalitiesDict(self): + return {} + + def getComponentLayoutSchemes(self): + """ + Provides layout scheme dictionaries for components. + + The scheme dictionary basically maps the number of hosts to + host index where component should exist. + """ + return {}
