AMBARI-7880. Recommendations API works too slow for 400 node cluster
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4e8a6529 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4e8a6529 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4e8a6529 Branch: refs/heads/trunk Commit: 4e8a6529b4011ba972a58f1627f6e8940acc85b5 Parents: c2e69c4 Author: Srimanth Gunturi <sgunt...@hortonworks.com> Authored: Mon Oct 20 16:43:24 2014 -0700 Committer: Srimanth Gunturi <sgunt...@hortonworks.com> Committed: Mon Oct 20 17:52:05 2014 -0700 ---------------------------------------------------------------------- .../commands/StackAdvisorCommand.java | 2 +- .../src/main/resources/stacks/stack_advisor.py | 37 +- .../test/python/stacks/2.2/common/1/hosts.json | 9743 +++++++ .../python/stacks/2.2/common/1/services.json | 1122 + .../test/python/stacks/2.2/common/2/hosts.json | 25056 +++++++++++++++++ .../python/stacks/2.2/common/2/services.json | 1122 + .../2.2/common/test_stack_advisor_perf.py | 69 + .../test/resources/stacks/old_stack_advisor.py | 581 + 8 files changed, 37715 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4e8a6529/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java index 6353978..eceefef 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java @@ -70,7 +70,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend protected static Log LOG = LogFactory.getLog(StackAdvisorCommand.class); private static final String GET_HOSTS_INFO_URI = "/api/v1/hosts" - + "?fields=*&Hosts/host_name.in(%s)"; + + "?fields=Hosts/host_name,Hosts/total_mem,Hosts/cpu_count,Hosts/disk_info&Hosts/host_name.in(%s)"; private static final String GET_SERVICES_INFO_URI = "/api/v1/stacks/%s/versions/%s" + "?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version" + ",services/StackServices/service_name,services/StackServices/service_version" http://git-wip-us.apache.org/repos/asf/ambari/blob/4e8a6529/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index 8a880e0..faa7b70 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -17,6 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. """ +import time import socket class StackAdvisor(object): @@ -354,25 +355,25 @@ class DefaultStackAdvisor(StackAdvisor): if self.isComponentHostsPopulated(component): hostsForComponent = component["StackServiceComponents"]["hostnames"] else: - availableHosts = hostsList - if len(hostsList) > 1 and self.isComponentNotPreferableOnAmbariServerHost(component): - availableHosts = [hostName for hostName in hostsList if not self.isLocalHost(hostName)] - if self.isMasterComponentWithMultipleInstances(component): + if len(hostsList) > 1 and self.isMasterComponentWithMultipleInstances(component): hostsCount = self.getMinComponentCount(component) if hostsCount > 1: # get first 'hostsCount' available hosts - if len(availableHosts) < hostsCount: - hostsCount = len(availableHosts) - hostsForComponent = availableHosts[:hostsCount] + hostsForComponent = [] + hostIndex = 0 + while hostsCount > len(hostsForComponent) and hostIndex < len(hostsList): + currentHost = hostsList[hostIndex] + if self.isHostSuitableForComponent(currentHost, component): + hostsForComponent.append(currentHost) + hostIndex += 1 else: - hostsForComponent = [self.getHostForComponent(component, availableHosts)] + hostsForComponent = [self.getHostForComponent(component, hostsList)] else: - hostsForComponent = [self.getHostForComponent(component, availableHosts)] + hostsForComponent = [self.getHostForComponent(component, hostsList)] #extend 'hostsComponentsMap' with 'hostsForComponent' for hostName in hostsForComponent: hostsComponentsMap[hostName].append( { "name":componentName } ) - #extend 'hostsComponentsMap' with Slave and Client Components componentsListList = [service["components"] for service in services["services"]] componentsList = [item for sublist in componentsListList for item in sublist] @@ -412,7 +413,7 @@ class DefaultStackAdvisor(StackAdvisor): index += 1 host_group_name = "host-group-{0}".format(index) host_groups.append( { "name": host_group_name, "components": hostsComponentsMap[key] } ) - bindings.append( { "name": host_group_name, "hosts": [{ "fqdn": socket.getfqdn(key) }] } ) + bindings.append( { "name": host_group_name, "hosts": [{ "fqdn": key }] } ) return recommendations pass @@ -538,11 +539,12 @@ class DefaultStackAdvisor(StackAdvisor): if len(hostsList) != 1: scheme = self.getComponentLayoutScheme(componentName) if scheme is not None: - for key in scheme.keys(): - if isinstance(key, ( int, long )): - if len(hostsList) < key: - return hostsList[scheme[key]] - return hostsList[scheme['else']] + hostIndex = next((index for key, index in scheme.iteritems() if isinstance(key, ( int, long )) and len(hostsList) < key), scheme['else']) + else: + hostIndex = 0 + for host in hostsList[hostIndex:]: + if self.isHostSuitableForComponent(host, component): + return host return hostsList[0] def getComponentLayoutScheme(self, componentName): @@ -559,6 +561,9 @@ class DefaultStackAdvisor(StackAdvisor): service = self.getNotPreferableOnServerComponents() return componentName in service + def isHostSuitableForComponent(self, host, component): + return not (self.isComponentNotPreferableOnAmbariServerHost(component) and self.isLocalHost(host)) + def getMastersWithMultipleInstances(self): return []
