Repository: ambari Updated Branches: refs/heads/trunk e8e9781a8 -> 927d5c834
AMBARI-20122 - Stack advisor needs to recommend dependency for slaves and masters Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/927d5c83 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/927d5c83 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/927d5c83 Branch: refs/heads/trunk Commit: 927d5c8340e5a22cd2fb8b8a2d14956b19810b93 Parents: e8e9781 Author: Tim Thorpe <ttho...@apache.org> Authored: Mon Jun 19 09:00:20 2017 -0700 Committer: Tim Thorpe <ttho...@apache.org> Committed: Mon Jun 19 09:00:20 2017 -0700 ---------------------------------------------------------------------- .../src/main/resources/stacks/stack_advisor.py | 116 +++++++++++++- .../stacks/2.0.6/common/test_stack_advisor.py | 153 +++++++++++++++++++ 2 files changed, 264 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/927d5c83/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 4a81dc6..3a39a34 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -756,15 +756,19 @@ class DefaultStackAdvisor(StackAdvisor): if hostName not in hostsComponentsMap: hostsComponentsMap[hostName] = [] + #Sort the services so that the dependent services will be processed before those that depend on them. + sortedServices = self.getServicesSortedByDependencies(services) #extend hostsComponentsMap' with MASTER components - for service in services["services"]: + for service in sortedServices: masterComponents = [component for component in service["components"] if self.isMasterComponent(component)] serviceName = service["StackServices"]["service_name"] serviceAdvisor = self.getServiceAdvisor(serviceName) for component in masterComponents: componentName = component["StackServiceComponents"]["component_name"] advisor = serviceAdvisor if serviceAdvisor is not None else self - hostsForComponent = advisor.getHostsForMasterComponent(services, hosts, component, hostsList) + #Filter the hosts such that only hosts that meet the dependencies are included (if possible) + filteredHosts = self.getFilteredHostsBasedOnDependencies(services, component, hostsList, hostsComponentsMap) + hostsForComponent = advisor.getHostsForMasterComponent(services, hosts, component, filteredHosts) #extend 'hostsComponentsMap' with 'hostsForComponent' for hostName in hostsForComponent: @@ -778,7 +782,7 @@ class DefaultStackAdvisor(StackAdvisor): utilizedHosts = [item for sublist in usedHostsListList for item in sublist] freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts] - for service in services["services"]: + for service in sortedServices: slaveClientComponents = [component for component in service["components"] if self.isSlaveComponent(component) or self.isClientComponent(component)] serviceName = service["StackServices"]["service_name"] @@ -786,7 +790,10 @@ class DefaultStackAdvisor(StackAdvisor): for component in slaveClientComponents: componentName = component["StackServiceComponents"]["component_name"] advisor = serviceAdvisor if serviceAdvisor is not None else self - hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, component, hostsList, freeHosts) + #Filter the hosts and free hosts such that only hosts that meet the dependencies are included (if possible) + filteredHosts = self.getFilteredHostsBasedOnDependencies(services, component, hostsList, hostsComponentsMap) + filteredFreeHosts = self.filterList(freeHosts, filteredHosts) + hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, component, filteredHosts, filteredFreeHosts) #extend 'hostsComponentsMap' with 'hostsForComponent' for hostName in hostsForComponent: @@ -796,7 +803,7 @@ class DefaultStackAdvisor(StackAdvisor): hostsComponentsMap[hostName].append( { "name": componentName } ) #colocate custom services - for service in services["services"]: + for service in sortedServices: serviceName = service["StackServices"]["service_name"] serviceAdvisor = self.getServiceAdvisor(serviceName) if serviceAdvisor is not None: @@ -866,6 +873,105 @@ class DefaultStackAdvisor(StackAdvisor): return hostsForComponent + def getServicesSortedByDependencies(self, services): + """ + Sorts the services based on their dependencies. This is limited to non-conditional host scope dependencies. + Services with no dependencies will go first. Services with dependencies will go after the services they are dependent on. + If there are circular dependencies, the services will go in the order in which they were processed. + """ + processedServices = [] + sortedServices = [] + + for service in services["services"]: + self.sortServicesByDependencies(services, service, processedServices, sortedServices) + + return sortedServices + + def sortServicesByDependencies(self, services, service, processedServices, sortedServices): + """ + Sorts the services based on their dependencies. This is limited to non-conditional host scope dependencies. + Services with no dependencies will go first. Services with dependencies will go after the services they are dependent on. + If there are circular dependencies, the services will go in the order in which they were processed. + """ + if service is None or service in processedServices: + return + + processedServices.append(service) + + components = [] if "components" not in service else service["components"] + for component in components: + dependencies = [] if "dependencies" not in component else component['dependencies'] + for dependency in dependencies: + # accounts only for dependencies that are not conditional + conditionsPresent = "conditions" in dependency["Dependencies"] and dependency["Dependencies"]["conditions"] + scope = "cluster" if "scope" not in dependency["Dependencies"] else dependency["Dependencies"]["scope"] + if not conditionsPresent and scope == "host": + componentName = component["StackServiceComponents"]["component_name"] + requiredComponentName = dependency["Dependencies"]["component_name"] + requiredService = self.getServiceForComponentName(services, requiredComponentName) + self.sortServicesByDependencies(services, requiredService, processedServices, sortedServices) + + sortedServices.append(service) + + def getFilteredHostsBasedOnDependencies(self, services, component, hostsList, hostsComponentsMap): + """ + Returns a list of hosts that only includes the ones which have all host scope dependencies already assigned to them. + If an empty list would be returned, instead the full list of hosts are returned. + In that case, we can't possibly return a valid recommended layout so we will at least return a fully filled layout. + """ + removeHosts = [] + dependencies = [] if "dependencies" not in component else component['dependencies'] + for dependency in dependencies: + # accounts only for dependencies that are not conditional + conditionsPresent = "conditions" in dependency["Dependencies"] and dependency["Dependencies"]["conditions"] + if not conditionsPresent: + componentName = component["StackServiceComponents"]["component_name"] + requiredComponentName = dependency["Dependencies"]["component_name"] + requiredComponent = self.getRequiredComponent(services, requiredComponentName) + + # We only deal with "host" scope. + if (requiredComponent is not None) and (requiredComponent["component_category"] != "CLIENT"): + scope = "cluster" if "scope" not in dependency["Dependencies"] else dependency["Dependencies"]["scope"] + if scope == "host": + for host, hostComponents in hostsComponentsMap.iteritems(): + isRequiredIncluded = False + for component in hostComponents: + currentComponentName = None if "name" not in component else component["name"] + if requiredComponentName == currentComponentName: + isRequiredIncluded = True + if not isRequiredIncluded: + removeHosts.append(host) + + filteredHostsList = [] + for host in hostsList: + if host not in removeHosts: + filteredHostsList.append(host) + return filteredHostsList + + def filterList(self, list, filter): + """ + Returns the union of the two lists passed in (list and filter params). + """ + filteredList = [] + for item in list: + if item in filter: + filteredList.append(item) + return filteredList + + def getServiceForComponentName(self, services, componentName): + """ + Return service for component name + + :type services dict + :type componentName str + """ + for service in services["services"]: + for component in service["components"]: + if self.getComponentName(component) == componentName: + return service + + return None + def isComponentUsingCardinalityForLayout(self, componentName): return False http://git-wip-us.apache.org/repos/asf/ambari/blob/927d5c83/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index 41c57f6..b6f1965 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -191,6 +191,159 @@ class TestHDP206StackAdvisor(TestCase): ] self.assertValidationResult(expectedItems, result) + + def test_handleComponentDependencies(self): + services = { + "Versions": + { + "stack_name":"HDP", + "stack_version":"2.0.6" + }, + "services" : [ + { + "StackServices" : { + "service_name" : "HDFS", + "service_version" : "2.0.6", + }, + "components": [ + { + "StackServiceComponents": { + "stack_version": "2.0.6", + "stack_name": "HDP", + "component_category": "MASTER", + "is_client": False, + "is_master": True, + "service_name": "HDFS", + "cardinality": "1-2", + "hostnames": [], + "component_name": "NAMENODE", + "display_name": "NameNode" + }, + "dependencies": [ + { + "Dependencies": { + "stack_name": "HDP", + "stack_version": "2.0.6", + "scope": "cluster", + "conditions": [ + { + "configType": "hdfs-site", + "property": "dfs.nameservices", + "type": "PropertyExists", + } + ], + "dependent_service_name": "HDFS", + "dependent_component_name": "NAMENODE", + "component_name": "ZOOKEEPER_SERVER" + } + } + ] + } + ] + }, + { + "StackServices" : { + "service_name" : "ZOOKEEPER", + "service_version" : "2.0.6", + }, + "components": [ + { + "StackServiceComponents": { + "stack_version": "2.0.6", + "stack_name": "HDP", + "component_category": "MASTER", + "is_client": False, + "is_master": True, + "service_name": "HDFS", + "cardinality": "1-2", + "hostnames": [], + "component_name": "ZOOKEEPER_SERVER", + "display_name": "ZooKeeper Server" + }, + "dependencies": [] + } + ] + } + ] + } + + nameNodeDependencies = services["services"][0]["components"][0]["dependencies"][0]["Dependencies"] + + # Tests for master component with dependencies + + hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org", "c6404.ambari.apache.org"]) + services["services"][1]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"] + recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts) + # Assert that dependencies are ignored when there are conditions and cluster scope + self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE') + self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1) + + nameNodeDependencies["scope"] = "host" + recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts) + # Assert that dependencies are ignored when there are conditions (even for host scope) + self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE') + self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1) + + nameNodeDependencies["scope"] = "cluster" + originalConditions = nameNodeDependencies["conditions"] + nameNodeDependencies["conditions"] = [] + recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts) + # Assert that dependencies are ignored when scope is cluster + self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE') + self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1) + + nameNodeDependencies["scope"] = "host" + recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts) + # Assert that dependencies are enforced for host scope without conditions + #self.assertEquals(recommendations, "") + self.assertEquals(len(recommendations['blueprint']['host_groups'][0]['components']), 2) + + services["services"][1]["components"][0]["StackServiceComponents"]["is_master"] = False + services["services"][1]["components"][0]["StackServiceComponents"]["component_category"] = "CLIENT" + recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts) + # Assert that dependencies are ignored when depending on client components + self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE') + self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1) + + # Tests for slave component with dependencies + services["services"][0]["components"][0]["StackServiceComponents"]["component_category"] = "SLAVE" + services["services"][0]["components"][0]["StackServiceComponents"]["is_master"] = False + services["services"][1]["components"][0]["StackServiceComponents"]["component_category"] = "MASTER" + services["services"][1]["components"][0]["StackServiceComponents"]["is_master"] = True + + nameNodeDependencies["scope"] = "cluster" + nameNodeDependencies["conditions"] = originalConditions + recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts) + # Assert that dependencies are ignored when there are conditions and cluster scope + self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'], 'NAMENODE') + self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']), 1) + self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE') + self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1) + + nameNodeDependencies["scope"] = "host" + recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts) + # Assert that dependencies are ignored when there are conditions (even for host scope) + self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'], 'NAMENODE') + self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']), 1) + self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE') + self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1) + + nameNodeDependencies["scope"] = "cluster" + nameNodeDependencies["conditions"] = [] + recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts) + # Assert that dependencies are ignored when scope is cluster + self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'], 'NAMENODE') + self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']), 1) + self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE') + self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1) + + nameNodeDependencies["scope"] = "host" + recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts) + # Assert that dependencies are enforced when host scope and no conditions + self.assertEquals(recommendations['blueprint']['host_groups'][1]['components'][1]['name'], 'NAMENODE') + self.assertEquals(len(recommendations['blueprint']['host_groups'][1]['components']), 2) + + def test_validateRequiredComponentsPresent(self): services = { "Versions":