AMBARI-19685. Stack advisor needs to enforce component dependency for slaves and masters. (jaimin)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c00399c8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c00399c8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c00399c8 Branch: refs/heads/branch-dev-patch-upgrade Commit: c00399c8505212086c8d7d2deffcc5df74d4e770 Parents: f571ceb Author: Jaimin Jetly <jai...@hortonworks.com> Authored: Wed Jan 25 12:23:36 2017 -0800 Committer: Jaimin Jetly <jai...@hortonworks.com> Committed: Wed Jan 25 12:23:36 2017 -0800 ---------------------------------------------------------------------- .../commands/StackAdvisorCommand.java | 6 +- .../StackDependencyResourceProvider.java | 8 ++ .../server/state/DependencyConditionInfo.java | 13 +++ .../src/main/resources/properties.json | 1 + .../stacks/HDP/2.3/services/stack_advisor.py | 25 ---- .../src/main/resources/stacks/stack_advisor.py | 63 ++++++++++ .../services-master_ambari_colo-3-hosts.json | 33 +++--- .../services-master_standby_colo-3-hosts.json | 33 +++--- .../configs/services-normal-hawq-3-hosts.json | 33 +++--- .../configs/services-normal-nohawq-3-hosts.json | 10 +- .../services-standby_ambari_colo-3-hosts.json | 33 +++--- .../stacks/2.0.6/common/test_stack_advisor.py | 107 ++++++++++++++++- .../2.3/common/services-sparkts-hive.json | 102 ++++++++++++----- .../stacks/2.3/common/services-sparkts.json | 77 ++++++------- .../stacks/2.3/common/test_stack_advisor.py | 2 +- .../2.5/common/services-normal-his-2-hosts.json | 114 ++++++++++++++----- 16 files changed, 466 insertions(+), 194 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java index adcb5b5..2cfa8e7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java @@ -78,11 +78,11 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend private static final String GET_SERVICES_INFO_URI = "/api/v1/stacks/%s/versions/%s/" + "?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version" + ",services/StackServices/service_name,services/StackServices/service_version" - + ",services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy" + + ",services/components/StackServiceComponents,services/components/dependencies/Dependencies/scope" + + ",services/components/dependencies/Dependencies/conditions,services/components/auto_deploy" + ",services/configurations/StackConfigurations/property_depends_on" + ",services/configurations/dependencies/StackConfigurationDependency/dependency_name" - + ",services/configurations/dependencies/StackConfigurationDependency/dependency_type" - + ",services/configurations/StackConfigurations/type" + + ",services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type" + "&services/StackServices/service_name.in(%s)"; private static final String SERVICES_PROPERTY = "services"; private static final String SERVICES_COMPONENTS_PROPERTY = "components"; http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDependencyResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDependencyResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDependencyResourceProvider.java index 5fc5f52..d9cd557 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDependencyResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDependencyResourceProvider.java @@ -39,6 +39,7 @@ import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; import org.apache.ambari.server.controller.utilities.PropertyHelper; import org.apache.ambari.server.state.AutoDeployInfo; +import org.apache.ambari.server.state.DependencyConditionInfo; import org.apache.ambari.server.state.DependencyInfo; /** @@ -62,6 +63,8 @@ public class StackDependencyResourceProvider extends AbstractResourceProvider { PropertyHelper.getPropertyId("Dependencies", "component_name"); protected static final String SCOPE_ID = PropertyHelper.getPropertyId("Dependencies", "scope"); + protected static final String CONDITIONS_ID = PropertyHelper + .getPropertyId("Dependencies","conditions"); protected static final String AUTO_DEPLOY_ENABLED_ID = PropertyHelper .getPropertyId("auto_deploy", "enabled"); protected static final String AUTO_DEPLOY_LOCATION_ID = PropertyHelper @@ -249,6 +252,11 @@ public class StackDependencyResourceProvider extends AbstractResourceProvider { autoDeployInfo.getCoLocate(), requestedIds); } } + List<DependencyConditionInfo> dependencyConditionsInfo = dependency.getDependencyConditions(); + if(dependencyConditionsInfo != null){ + setResourceProperty(resource, CONDITIONS_ID, + dependencyConditionsInfo, requestedIds); + } return resource; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java index 5d9358c..f5d642d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java @@ -55,6 +55,19 @@ class PropertyExists implements DependencyConditionInfo { */ protected String property; + /** + * type of conditional dependency + */ + protected String type = this.getClass().getSimpleName(); + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + @XmlElement public String getProperty() { return property; http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/main/resources/properties.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json index 698b6c5..72104a9 100644 --- a/ambari-server/src/main/resources/properties.json +++ b/ambari-server/src/main/resources/properties.json @@ -272,6 +272,7 @@ "Dependencies/service_name", "Dependencies/component_name", "Dependencies/scope", + "Dependencies/conditions", "auto_deploy/enabled", "auto_deploy/location" ], http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 1fc8598..1425abc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -36,31 +36,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor): super(HDP23StackAdvisor, self).__init__() Logger.initialize_logger() - def getComponentLayoutValidations(self, services, hosts): - parentItems = super(HDP23StackAdvisor, self).getComponentLayoutValidations(services, hosts) - - servicesList = [service["StackServices"]["service_name"] for service in services["services"]] - componentsListList = [service["components"] for service in services["services"]] - componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] - childItems = [] - - if "SPARK" in servicesList: - if "SPARK_THRIFTSERVER" in servicesList: - if not "HIVE_SERVER" in servicesList: - message = "SPARK_THRIFTSERVER requires HIVE services to be selected." - childItems.append( {"type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER'} ) - - hmsHosts = self.__getHosts(componentsList, "HIVE_METASTORE") if "HIVE" in servicesList else [] - sparkTsHosts = self.__getHosts(componentsList, "SPARK_THRIFTSERVER") if "SPARK" in servicesList else [] - - # if Spark Thrift Server is deployed but no Hive Server is deployed - if len(sparkTsHosts) > 0 and len(hmsHosts) == 0: - message = "SPARK_THRIFTSERVER requires HIVE_METASTORE to be selected/deployed." - childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER' } ) - - parentItems.extend(childItems) - return parentItems - def __getHosts(self, componentsList, componentName): host_lists = [component["hostnames"] for component in componentsList if component["component_name"] == componentName] http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 6fb014e..215e807 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -884,6 +884,8 @@ class DefaultStackAdvisor(StackAdvisor): if services is None: return items + items.extend(self.validateRequiredComponentsPresent(services)) + for service in services["services"]: serviceName = service["StackServices"]["service_name"] serviceAdvisor = self.getServiceAdvisor(serviceName) @@ -892,6 +894,52 @@ class DefaultStackAdvisor(StackAdvisor): return items + def validateRequiredComponentsPresent(self, services): + """ + Returns validation items derived from component dependencies as specified in service metainfo.xml for all services + :type services dict + :rtype list + """ + items = [] + for service in services["services"]: + for component in service["components"]: + + # Client components are not validated for the dependencies + # Rather dependent client components are auto-deployed in both UI deployments and blueprint deployments + if (self.isSlaveComponent(component) or self.isMasterComponent(component)) and \ + component["StackServiceComponents"]["hostnames"]: + for dependency in component['dependencies']: + # account for only dependencies that are not conditional + conditionsPresent = "conditions" in dependency["Dependencies"] and dependency["Dependencies"]["conditions"] + if not conditionsPresent: + requiredComponent = self.getRequiredComponent(services, dependency["Dependencies"]["component_name"]) + componentDisplayName = component["StackServiceComponents"]["display_name"] + requiredComponentDisplayName = requiredComponent["display_name"] \ + if requiredComponent is not None else dependency["Dependencies"]["component_name"] + requiredComponentHosts = requiredComponent["hostnames"] if requiredComponent is not None else [] + + # Client dependencies are not included in validation + # Client dependencies are auto-deployed in both UI deployements and blueprint deployments + if (requiredComponent is None) or \ + (requiredComponent["component_category"] != "CLIENT"): + scope = "cluster" if "scope" not in dependency["Dependencies"] else dependency["Dependencies"]["scope"] + if scope == "host": + componentHosts = component["StackServiceComponents"]["hostnames"] + requiredComponentHostsAbsent = [] + for componentHost in componentHosts: + if componentHost not in requiredComponentHosts: + requiredComponentHostsAbsent.append(componentHost) + if requiredComponentHostsAbsent: + message = "{0} requires {1} to be co-hosted on following host(s): {2}.".format(componentDisplayName, + requiredComponentDisplayName, ', '.join(requiredComponentHostsAbsent)) + items.append({ "type": 'host-component', "level": 'ERROR', "message": message, + "component-name": component["StackServiceComponents"]["component_name"]}) + elif scope == "cluster" and not requiredComponentHosts: + message = "{0} requires {1} to be present in the cluster.".format(componentDisplayName, requiredComponentDisplayName) + items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": component["StackServiceComponents"]["component_name"]}) + return items + + def getConfigurationClusterSummary(self, servicesList, hosts, components, services): """ Copied from HDP 2.0.6 so that it could be used by Service Advisors. @@ -1253,6 +1301,21 @@ class DefaultStackAdvisor(StackAdvisor): def isMasterComponent(self, component): return self.getComponentAttribute(component, "is_master") + def getRequiredComponent(self, services, componentName): + """ + Return Category for component + + :type services dict + :type componentName str + :rtype dict + """ + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] + component = next((component for component in componentsList + if component["component_name"] == componentName), None) + + return component + def getComponentAttribute(self, component, attribute): serviceComponent = component.get("StackServiceComponents", None) if serviceComponent is None: http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/test/python/common-services/configs/services-master_ambari_colo-3-hosts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/configs/services-master_ambari_colo-3-hosts.json b/ambari-server/src/test/python/common-services/configs/services-master_ambari_colo-3-hosts.json index c1b2655..e8c6bcb 100644 --- a/ambari-server/src/test/python/common-services/configs/services-master_ambari_colo-3-hosts.json +++ b/ambari-server/src/test/python/common-services/configs/services-master_ambari_colo-3-hosts.json @@ -1,5 +1,5 @@ { - "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,ZOOKEEPER,HAWQ)", + "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies/Dependencies/scope,services/components/dependencies/Dependencies/conditions,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,ZOOKEEPER,HAWQ)", "Versions" : { "parent_stack_version" : "2.2", "stack_name" : "HDP", @@ -40,7 +40,9 @@ "dependent_component_name" : "HAWQMASTER", "dependent_service_name" : "HAWQ", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/components/HAWQMASTER/dependencies/NAMENODE", @@ -49,7 +51,9 @@ "dependent_component_name" : "HAWQMASTER", "dependent_service_name" : "HAWQ", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3" , + "scope": "cluster", + "conditions": [] } } ] }, { @@ -68,16 +72,7 @@ "stack_version" : "2.3", "hostnames" : [ "c6403.ambari.apache.org" ] }, - "dependencies" : [ { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/components/HAWQSEGMENT/dependencies/DATANODE", - "Dependencies" : { - "component_name" : "DATANODE", - "dependent_component_name" : "HAWQSEGMENT", - "dependent_service_name" : "HAWQ", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - } ] + "dependencies" : [] }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/components/HAWQSTANDBY", "StackServiceComponents" : { @@ -101,7 +96,9 @@ "dependent_component_name" : "HAWQSTANDBY", "dependent_service_name" : "HAWQ", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] } ], @@ -424,7 +421,9 @@ "dependent_component_name" : "JOURNALNODE", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -467,7 +466,9 @@ "dependent_component_name" : "NFS_GATEWAY", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/test/python/common-services/configs/services-master_standby_colo-3-hosts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/configs/services-master_standby_colo-3-hosts.json b/ambari-server/src/test/python/common-services/configs/services-master_standby_colo-3-hosts.json index e4b48da..f95d19b 100644 --- a/ambari-server/src/test/python/common-services/configs/services-master_standby_colo-3-hosts.json +++ b/ambari-server/src/test/python/common-services/configs/services-master_standby_colo-3-hosts.json @@ -1,5 +1,5 @@ { - "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,ZOOKEEPER,HAWQ)", + "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies/Dependencies/scope,services/components/dependencies/Dependencies/conditions,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,ZOOKEEPER,HAWQ)", "Versions" : { "parent_stack_version" : "2.2", "stack_name" : "HDP", @@ -40,7 +40,9 @@ "dependent_component_name" : "HAWQMASTER", "dependent_service_name" : "HAWQ", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/components/HAWQMASTER/dependencies/NAMENODE", @@ -49,7 +51,9 @@ "dependent_component_name" : "HAWQMASTER", "dependent_service_name" : "HAWQ", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "cluster", + "conditions": [] } } ] }, { @@ -68,16 +72,7 @@ "stack_version" : "2.3", "hostnames" : [ "c6403.ambari.apache.org" ] }, - "dependencies" : [ { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/components/HAWQSEGMENT/dependencies/DATANODE", - "Dependencies" : { - "component_name" : "DATANODE", - "dependent_component_name" : "HAWQSEGMENT", - "dependent_service_name" : "HAWQ", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - } ] + "dependencies" : [] }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/components/HAWQSTANDBY", "StackServiceComponents" : { @@ -101,7 +96,9 @@ "dependent_component_name" : "HAWQSTANDBY", "dependent_service_name" : "HAWQ", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] } ], @@ -424,7 +421,9 @@ "dependent_component_name" : "JOURNALNODE", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -467,7 +466,9 @@ "dependent_component_name" : "NFS_GATEWAY", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/test/python/common-services/configs/services-normal-hawq-3-hosts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/configs/services-normal-hawq-3-hosts.json b/ambari-server/src/test/python/common-services/configs/services-normal-hawq-3-hosts.json index b896a71..2ffd5be 100644 --- a/ambari-server/src/test/python/common-services/configs/services-normal-hawq-3-hosts.json +++ b/ambari-server/src/test/python/common-services/configs/services-normal-hawq-3-hosts.json @@ -1,5 +1,5 @@ { - "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,ZOOKEEPER,HAWQ)", + "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies/Dependencies/scope,services/components/dependencies/Dependencies/conditions,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,ZOOKEEPER,HAWQ)", "Versions" : { "parent_stack_version" : "2.2", "stack_name" : "HDP", @@ -40,7 +40,9 @@ "dependent_component_name" : "HAWQMASTER", "dependent_service_name" : "HAWQ", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/components/HAWQMASTER/dependencies/NAMENODE", @@ -49,7 +51,9 @@ "dependent_component_name" : "HAWQMASTER", "dependent_service_name" : "HAWQ", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "cluster", + "conditions": [] } } ] }, { @@ -68,16 +72,7 @@ "stack_version" : "2.3", "hostnames" : [ "c6402.ambari.apache.org", "c6403.ambari.apache.org", "c6401.ambari.apache.org" ] }, - "dependencies" : [ { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/components/HAWQSEGMENT/dependencies/DATANODE", - "Dependencies" : { - "component_name" : "DATANODE", - "dependent_component_name" : "HAWQSEGMENT", - "dependent_service_name" : "HAWQ", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - } ] + "dependencies" : [] }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/components/HAWQSTANDBY", "StackServiceComponents" : { @@ -101,7 +96,9 @@ "dependent_component_name" : "HAWQSTANDBY", "dependent_service_name" : "HAWQ", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] } ], @@ -435,7 +432,9 @@ "dependent_component_name" : "JOURNALNODE", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -478,7 +477,9 @@ "dependent_component_name" : "NFS_GATEWAY", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/test/python/common-services/configs/services-normal-nohawq-3-hosts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/configs/services-normal-nohawq-3-hosts.json b/ambari-server/src/test/python/common-services/configs/services-normal-nohawq-3-hosts.json index 675aacb..119207e 100644 --- a/ambari-server/src/test/python/common-services/configs/services-normal-nohawq-3-hosts.json +++ b/ambari-server/src/test/python/common-services/configs/services-normal-nohawq-3-hosts.json @@ -1,5 +1,5 @@ { - "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,ZOOKEEPER)", + "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies/Dependencies/scope,services/components/dependencies/Dependencies/conditions,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,ZOOKEEPER)", "Versions" : { "parent_stack_version" : "2.2", "stack_name" : "HDP", @@ -74,7 +74,9 @@ "dependent_component_name" : "JOURNALNODE", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -117,7 +119,9 @@ "dependent_component_name" : "NFS_GATEWAY", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/test/python/common-services/configs/services-standby_ambari_colo-3-hosts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/common-services/configs/services-standby_ambari_colo-3-hosts.json b/ambari-server/src/test/python/common-services/configs/services-standby_ambari_colo-3-hosts.json index 489d4df..6d72f48 100644 --- a/ambari-server/src/test/python/common-services/configs/services-standby_ambari_colo-3-hosts.json +++ b/ambari-server/src/test/python/common-services/configs/services-standby_ambari_colo-3-hosts.json @@ -1,5 +1,5 @@ { - "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,ZOOKEEPER,HAWQ)", + "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies/Dependencies/scope,services/components/dependencies/Dependencies/conditions,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,ZOOKEEPER,HAWQ)", "Versions" : { "parent_stack_version" : "2.2", "stack_name" : "HDP", @@ -40,7 +40,9 @@ "dependent_component_name" : "HAWQMASTER", "dependent_service_name" : "HAWQ", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/components/HAWQMASTER/dependencies/NAMENODE", @@ -49,7 +51,9 @@ "dependent_component_name" : "HAWQMASTER", "dependent_service_name" : "HAWQ", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "cluster", + "conditions": [] } } ] }, { @@ -68,16 +72,7 @@ "stack_version" : "2.3", "hostnames" : [ "c6403.ambari.apache.org" ] }, - "dependencies" : [ { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/components/HAWQSEGMENT/dependencies/DATANODE", - "Dependencies" : { - "component_name" : "DATANODE", - "dependent_component_name" : "HAWQSEGMENT", - "dependent_service_name" : "HAWQ", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - } ] + "dependencies" : [] }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/components/HAWQSTANDBY", "StackServiceComponents" : { @@ -101,7 +96,9 @@ "dependent_component_name" : "HAWQSTANDBY", "dependent_service_name" : "HAWQ", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] } ], @@ -424,7 +421,9 @@ "dependent_component_name" : "JOURNALNODE", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -467,7 +466,9 @@ "dependent_component_name" : "NFS_GATEWAY", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index a6931c5..388d7f8 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -191,6 +191,110 @@ class TestHDP206StackAdvisor(TestCase): ] self.assertValidationResult(expectedItems, result) + def test_validateRequiredComponentsPresent(self): + services = { + "Versions": + { + "stack_name":"HDP", + "stack_version":"2.0.6" + }, + "services" : [ + { + "StackServices" : { + "service_name" : "HDFS", + "service_version" : "2.0.6", + }, + "components": [ + { + "StackServiceComponents": { + "stack_version": "2.0.6", + "stack_name": "HDP", + "component_category": "MASTER", + "is_client": False, + "is_master": True, + "service_name": "HDFS", + "cardinality": "1-2", + "hostnames": ["c6401.ambari.apache.org"], + "component_name": "NAMENODE", + "display_name": "NameNode" + }, + "dependencies": [ + { + "Dependencies": { + "stack_name": "HDP", + "stack_version": "2.0.6", + "scope": "cluster", + "conditions": [ + { + "configType": "hdfs-site", + "property": "dfs.nameservices", + "type": "PropertyExists", + } + ], + "dependent_service_name": "HDFS", + "dependent_component_name": "NAMENODE", + "component_name": "ZOOKEEPER_SERVER" + } + } + ] + } + ] + }, + { + "StackServices" : { + "service_name" : "ZOOKEEPER", + "service_version" : "2.0.6", + }, + "components": [ + { + "StackServiceComponents": { + "stack_version": "2.0.6", + "stack_name": "HDP", + "component_category": "MASTER", + "is_client": False, + "is_master": True, + "service_name": "HDFS", + "cardinality": "1-2", + "hostnames": [], + "component_name": "ZOOKEEPER_SERVER", + "display_name": "ZooKeeper Server" + }, + "dependencies": [] + } + ] + } + ] + } + + actualItems = self.stackAdvisor.validateRequiredComponentsPresent(services) + self.assertTrue(len(actualItems) == 0, "Only dependencies that have no associated condition should be accounted") + + services["services"][0]["components"][0]["dependencies"][0]["Dependencies"]["conditions"] = [] + actualItems = self.stackAdvisor.validateRequiredComponentsPresent(services) + self.assertFalse(len(actualItems) == 0, "Dependencies without conditions should be accounted") + expectedItems = [{ "type": 'host-component', "level": 'ERROR', "message": "NameNode requires ZooKeeper Server to be present in the cluster.", "component-name": "NAMENODE"}] + self.assertEqual(expectedItems, actualItems, "Cluster scope dependency error should be raised") + + services["services"][1]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org"] + actualItems = self.stackAdvisor.validateRequiredComponentsPresent(services) + self.assertTrue(len(actualItems) == 0, "Validation error should not be raised when cluster scope dependency is satisfied") + + services["services"][0]["components"][0]["dependencies"][0]["Dependencies"]["scope"] = "host" + actualItems = self.stackAdvisor.validateRequiredComponentsPresent(services) + self.assertFalse(len(actualItems) == 0, "Validation error should be raised when host scope dependency is not satisfied") + expectedItems = [{ "type": 'host-component', "level": 'ERROR', "message": "NameNode requires ZooKeeper Server to be co-hosted on following host(s): c6401.ambari.apache.org.", "component-name": "NAMENODE"}] + self.assertEqual(expectedItems, actualItems, "Host scope dependency error should be raised") + + services["services"][1]["components"][0]["StackServiceComponents"]["component_category"] = "CLIENT" + actualItems = self.stackAdvisor.validateRequiredComponentsPresent(services) + self.assertTrue(len(actualItems) == 0, "Validation error should not be raised when dependency is a client component") + + services["services"][1]["components"][0]["StackServiceComponents"]["component_category"] = "SLAVE" + services["services"][1]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6401.ambari.apache.org"] + actualItems = self.stackAdvisor.validateRequiredComponentsPresent(services) + self.assertTrue(len(actualItems) == 0, "Validation error should not be raised when host scope dependency is satisfied") + + def test_get_components_list(self): servicesInfo = [ { @@ -1034,7 +1138,8 @@ class TestHDP206StackAdvisor(TestCase): "cardinality": component["cardinality"], "component_category": component["category"], "is_master": component["is_master"] - } + }, + "dependencies": [] } try: nextComponent["StackServiceComponents"]["hostnames"] = component["hostnames"] http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json b/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json index fe53614..5d14217 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json +++ b/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json @@ -1,5 +1,5 @@ { - "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,YARN,MAPREDUCE2,ZOOKEEPER,TEZ,HIVE,PIG,SPARK)", + "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies/Dependencies/scope,services/components/dependencies/Dependencies/conditions,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,YARN,MAPREDUCE2,ZOOKEEPER,TEZ,HIVE,PIG,SPARK)", "Versions" : { "parent_stack_version" : "2.2", "stack_name" : "HDP", @@ -74,7 +74,9 @@ "dependent_component_name" : "JOURNALNODE", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -117,7 +119,9 @@ "dependent_component_name" : "NFS_GATEWAY", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -2129,7 +2133,9 @@ "dependent_component_name" : "HIVE_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/components/HIVE_SERVER/dependencies/TEZ_CLIENT", @@ -2138,7 +2144,9 @@ "dependent_component_name" : "HIVE_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/components/HIVE_SERVER/dependencies/YARN_CLIENT", @@ -2147,7 +2155,9 @@ "dependent_component_name" : "HIVE_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/components/HIVE_SERVER/dependencies/ZOOKEEPER_SERVER", @@ -2156,7 +2166,9 @@ "dependent_component_name" : "HIVE_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "cluster", + "conditions": [] } } ] }, { @@ -2199,7 +2211,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/components/WEBHCAT_SERVER/dependencies/HIVE_CLIENT", @@ -2208,7 +2222,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/components/WEBHCAT_SERVER/dependencies/MAPREDUCE2_CLIENT", @@ -2226,7 +2242,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/components/WEBHCAT_SERVER/dependencies/YARN_CLIENT", @@ -2235,7 +2253,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/components/WEBHCAT_SERVER/dependencies/ZOOKEEPER_CLIENT", @@ -2244,7 +2264,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/components/WEBHCAT_SERVER/dependencies/ZOOKEEPER_SERVER", @@ -2253,7 +2275,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "cluster", + "conditions": [] } } ] } ], @@ -5538,7 +5562,9 @@ "dependent_component_name" : "HISTORYSERVER", "dependent_service_name" : "MAPREDUCE2", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/MAPREDUCE2/components/HISTORYSERVER/dependencies/TEZ_CLIENT", @@ -5547,7 +5573,9 @@ "dependent_component_name" : "HISTORYSERVER", "dependent_service_name" : "MAPREDUCE2", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -6334,7 +6362,9 @@ "dependent_component_name" : "SPARK_CLIENT", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_CLIENT/dependencies/MAPREDUCE2_CLIENT", @@ -6343,7 +6373,9 @@ "dependent_component_name" : "SPARK_CLIENT", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_CLIENT/dependencies/YARN_CLIENT", @@ -6352,7 +6384,9 @@ "dependent_component_name" : "SPARK_CLIENT", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -6378,7 +6412,9 @@ "dependent_component_name" : "SPARK_JOBHISTORYSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_JOBHISTORYSERVER/dependencies/MAPREDUCE2_CLIENT", @@ -6387,7 +6423,9 @@ "dependent_component_name" : "SPARK_JOBHISTORYSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_JOBHISTORYSERVER/dependencies/TEZ_CLIENT", @@ -6396,7 +6434,9 @@ "dependent_component_name" : "SPARK_JOBHISTORYSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_JOBHISTORYSERVER/dependencies/YARN_CLIENT", @@ -6405,7 +6445,9 @@ "dependent_component_name" : "SPARK_JOBHISTORYSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -6431,7 +6473,9 @@ "dependent_component_name" : "SPARK_THRIFTSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_THRIFTSERVER/dependencies/MAPREDUCE2_CLIENT", @@ -6440,7 +6484,9 @@ "dependent_component_name" : "SPARK_THRIFTSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_THRIFTSERVER/dependencies/YARN_CLIENT", @@ -6449,7 +6495,9 @@ "dependent_component_name" : "SPARK_THRIFTSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] } ], @@ -7496,7 +7544,9 @@ "dependent_component_name" : "RESOURCEMANAGER", "dependent_service_name" : "YARN", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/test/python/stacks/2.3/common/services-sparkts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/services-sparkts.json b/ambari-server/src/test/python/stacks/2.3/common/services-sparkts.json index 6514f3e..0bbd7c1 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/services-sparkts.json +++ b/ambari-server/src/test/python/stacks/2.3/common/services-sparkts.json @@ -1,5 +1,5 @@ { - "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,YARN,MAPREDUCE2,ZOOKEEPER,SPARK)", + "href" : "/api/v1/stacks/HDP/versions/2.3/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies/Dependencies/scope,services/components/dependencies/Dependencies/conditions,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,YARN,MAPREDUCE2,ZOOKEEPER,SPARK)", "Versions" : { "parent_stack_version" : "2.2", "stack_name" : "HDP", @@ -74,7 +74,9 @@ "dependent_component_name" : "JOURNALNODE", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -117,7 +119,9 @@ "dependent_component_name" : "NFS_GATEWAY", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -2078,18 +2082,11 @@ "dependent_component_name" : "HISTORYSERVER", "dependent_service_name" : "MAPREDUCE2", "stack_name" : "HDP", - "stack_version" : "2.3" - } - }, { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/MAPREDUCE2/components/HISTORYSERVER/dependencies/TEZ_CLIENT", - "Dependencies" : { - "component_name" : "TEZ_CLIENT", - "dependent_component_name" : "HISTORYSERVER", - "dependent_service_name" : "MAPREDUCE2", - "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } - } ] + }] }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT", "StackServiceComponents" : { @@ -2858,7 +2855,9 @@ "dependent_component_name" : "SPARK_JOBHISTORYSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_JOBHISTORYSERVER/dependencies/MAPREDUCE2_CLIENT", @@ -2867,16 +2866,9 @@ "dependent_component_name" : "SPARK_JOBHISTORYSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" - } - }, { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_JOBHISTORYSERVER/dependencies/TEZ_CLIENT", - "Dependencies" : { - "component_name" : "TEZ_CLIENT", - "dependent_component_name" : "SPARK_JOBHISTORYSERVER", - "dependent_service_name" : "SPARK", - "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_JOBHISTORYSERVER/dependencies/YARN_CLIENT", @@ -2885,7 +2877,9 @@ "dependent_component_name" : "SPARK_JOBHISTORYSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] }, { @@ -2911,16 +2905,20 @@ "dependent_component_name" : "SPARK_THRIFTSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_THRIFTSERVER/dependencies/HIVE_SERVER", + "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_THRIFTSERVER/dependencies/HIVE_METASTORE", "Dependencies" : { - "component_name" : "HIVE_SERVER", + "component_name" : "HIVE_METASTORE", "dependent_component_name" : "SPARK_THRIFTSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "cluster", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_THRIFTSERVER/dependencies/MAPREDUCE2_CLIENT", @@ -2929,7 +2927,9 @@ "dependent_component_name" : "SPARK_THRIFTSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/SPARK/components/SPARK_THRIFTSERVER/dependencies/YARN_CLIENT", @@ -2938,7 +2938,9 @@ "dependent_component_name" : "SPARK_THRIFTSERVER", "dependent_service_name" : "SPARK", "stack_name" : "HDP", - "stack_version" : "2.3" + "stack_version" : "2.3", + "scope": "host", + "conditions": [] } } ] } ], @@ -3353,16 +3355,7 @@ "stack_version" : "2.3", "hostnames" : [ "jerryshao-sts-test-1.c.pramod-thangali.internal" ] }, - "dependencies" : [ { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/YARN/components/RESOURCEMANAGER/dependencies/TEZ_CLIENT", - "Dependencies" : { - "component_name" : "TEZ_CLIENT", - "dependent_component_name" : "RESOURCEMANAGER", - "dependent_service_name" : "YARN", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - } ] + "dependencies" : [] }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/YARN/components/YARN_CLIENT", "StackServiceComponents" : { http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 2149877..7e93815 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -106,7 +106,7 @@ class TestHDP23StackAdvisor(TestCase): self.assertEquals(len(hiveMetaStore), 0) validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) - expected = {'component-name': 'SPARK_THRIFTSERVER', 'message': 'SPARK_THRIFTSERVER requires HIVE_METASTORE to be selected/deployed.', 'type': 'host-component', 'level': 'ERROR'} + expected = {'component-name': 'SPARK_THRIFTSERVER', 'message': 'Spark Thrift Server requires HIVE_METASTORE to be present in the cluster.', 'type': 'host-component', 'level': 'ERROR'} self.assertEquals(validations[0], expected) http://git-wip-us.apache.org/repos/asf/ambari/blob/c00399c8/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json index 46a6f4d..44698ed 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json +++ b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-2-hosts.json @@ -1,5 +1,5 @@ { - "href" : "/api/v1/stacks/HDP/versions/2.5/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,YARN,MAPREDUCE2,TEZ,HIVE,PIG,ZOOKEEPER,SLIDER)", + "href" : "/api/v1/stacks/HDP/versions/2.5/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies/Dependencies/scope,services/components/dependencies/Dependencies/conditions,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type&services/StackServices/service_name.in(HDFS,YARN,MAPREDUCE2,TEZ,HIVE,PIG,ZOOKEEPER,SLIDER)", "Versions" : { "parent_stack_version" : "2.4", "stack_name" : "HDP", @@ -92,7 +92,9 @@ "dependent_component_name" : "JOURNALNODE", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } } ] }, { @@ -147,7 +149,9 @@ "dependent_component_name" : "NFS_GATEWAY", "dependent_service_name" : "HDFS", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } } ] }, { @@ -319,7 +323,9 @@ "dependent_component_name" : "HIVE_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER/dependencies/TEZ_CLIENT", @@ -328,7 +334,9 @@ "dependent_component_name" : "HIVE_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER/dependencies/YARN_CLIENT", @@ -337,7 +345,9 @@ "dependent_component_name" : "HIVE_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER/dependencies/ZOOKEEPER_SERVER", @@ -346,7 +356,9 @@ "dependent_component_name" : "HIVE_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "cluster", + "conditions": [] } } ] }, { @@ -378,7 +390,9 @@ "dependent_component_name" : "HIVE_SERVER_INTERACTIVE", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE/dependencies/MAPREDUCE2_CLIENT", @@ -387,7 +401,9 @@ "dependent_component_name" : "HIVE_SERVER_INTERACTIVE", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE/dependencies/PIG", @@ -396,7 +412,9 @@ "dependent_component_name" : "HIVE_SERVER_INTERACTIVE", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE/dependencies/SLIDER", @@ -405,7 +423,9 @@ "dependent_component_name" : "HIVE_SERVER_INTERACTIVE", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE/dependencies/TEZ_CLIENT", @@ -414,7 +434,9 @@ "dependent_component_name" : "HIVE_SERVER_INTERACTIVE", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE/dependencies/YARN_CLIENT", @@ -423,7 +445,9 @@ "dependent_component_name" : "HIVE_SERVER_INTERACTIVE", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE/dependencies/ZOOKEEPER_SERVER", @@ -432,7 +456,9 @@ "dependent_component_name" : "HIVE_SERVER_INTERACTIVE", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "cluster", + "conditions": [] } } ] }, { @@ -487,7 +513,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/WEBHCAT_SERVER/dependencies/HIVE_CLIENT", @@ -496,7 +524,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/WEBHCAT_SERVER/dependencies/MAPREDUCE2_CLIENT", @@ -505,7 +535,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/WEBHCAT_SERVER/dependencies/PIG", @@ -514,7 +546,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/WEBHCAT_SERVER/dependencies/YARN_CLIENT", @@ -523,7 +557,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/WEBHCAT_SERVER/dependencies/ZOOKEEPER_CLIENT", @@ -532,7 +568,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/WEBHCAT_SERVER/dependencies/ZOOKEEPER_SERVER", @@ -541,7 +579,9 @@ "dependent_component_name" : "WEBHCAT_SERVER", "dependent_service_name" : "HIVE", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } } ] } ], @@ -598,7 +638,9 @@ "dependent_component_name" : "HISTORYSERVER", "dependent_service_name" : "MAPREDUCE2", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/MAPREDUCE2/components/HISTORYSERVER/dependencies/TEZ_CLIENT", @@ -607,7 +649,9 @@ "dependent_component_name" : "HISTORYSERVER", "dependent_service_name" : "MAPREDUCE2", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } } ] }, { @@ -749,7 +793,9 @@ "dependent_component_name" : "SLIDER", "dependent_service_name" : "SLIDER", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/SLIDER/components/SLIDER/dependencies/YARN_CLIENT", @@ -758,7 +804,9 @@ "dependent_component_name" : "SLIDER", "dependent_service_name" : "SLIDER", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } } ] } ], @@ -822,7 +870,9 @@ "dependent_component_name" : "TEZ_CLIENT", "dependent_service_name" : "TEZ", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/TEZ/components/TEZ_CLIENT/dependencies/MAPREDUCE2_CLIENT", @@ -831,7 +881,9 @@ "dependent_component_name" : "TEZ_CLIENT", "dependent_service_name" : "TEZ", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } }, { "href" : "/api/v1/stacks/HDP/versions/2.5/services/TEZ/components/TEZ_CLIENT/dependencies/YARN_CLIENT", @@ -840,7 +892,9 @@ "dependent_component_name" : "TEZ_CLIENT", "dependent_service_name" : "TEZ", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } } ] } ], @@ -939,7 +993,9 @@ "dependent_component_name" : "RESOURCEMANAGER", "dependent_service_name" : "YARN", "stack_name" : "HDP", - "stack_version" : "2.5" + "stack_version" : "2.5", + "scope": "host", + "conditions": [] } } ] }, {