AMBARI-14622. Stack Advisor should display warning when PXF component not collocated with NAMENODE and DATANODE(mithmatt via odiachenko).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/93517372 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/93517372 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/93517372 Branch: refs/heads/branch-dev-patch-upgrade Commit: 935173728b2d9344322a98ff4764f69e6ea6c40c Parents: 019d6f4 Author: Oleksandr Diachenko <odiache...@pivotal.io> Authored: Tue Jan 12 14:46:52 2016 -0800 Committer: Nate Cole <nc...@hortonworks.com> Committed: Thu Jan 14 11:43:26 2016 -0500 ---------------------------------------------------------------------- .../stacks/HDP/2.3/services/stack_advisor.py | 45 ++++++-- .../stacks/2.3/common/test_stack_advisor.py | 115 ++++++++++++++++++- 2 files changed, 147 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/93517372/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index b37b95a..ed241ff 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -62,12 +62,12 @@ class HDP23StackAdvisor(HDP22StackAdvisor): hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] hostsCount = len(hostsList) - hawqMasterHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "HAWQMASTER"][0] - hawqStandbyHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "HAWQSTANDBY"][0] - hawqSegmentHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "HAWQSEGMENT"][0] - datanodeHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "DATANODE"][0] + hawqMasterHosts = self.__getHosts(componentsList, "HAWQMASTER") + hawqStandbyHosts = self.__getHosts(componentsList, "HAWQSTANDBY") + hawqSegmentHosts = self.__getHosts(componentsList, "HAWQSEGMENT") + datanodeHosts = self.__getHosts(componentsList, "DATANODE") - # Generate WARNING if any HAWQSEGMENT is not colocated with a DATANODE + # Generate WARNING if any HAWQSEGMENT is not co-located with a DATANODE mismatchHosts = set(hawqSegmentHosts).symmetric_difference(set(datanodeHosts)) if len(mismatchHosts) > 0: hostsString = ', '.join(mismatchHosts) @@ -91,16 +91,40 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "to a value different from the port number used by Ambari Server database." childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } ) + if "PXF" in servicesList: + pxfHosts = self.__getHosts(componentsList, "PXF") + namenodeHosts = self.__getHosts(componentsList, "NAMENODE") + datanodeHosts = self.__getHosts(componentsList, "DATANODE") + + # Generate WARNING if any PXF is not co-located with a NAMENODE + mismatchHosts = sorted(set(namenodeHosts).difference(set(pxfHosts))) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "PXF is not co-located with NAMENODE on the following {0} host(s): {1}".format(len(mismatchHosts), hostsString) + childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PXF' } ) + + # Generate WARNING if any PXF is not co-located with a DATANODE + mismatchHosts = sorted(set(datanodeHosts).difference(set(pxfHosts))) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "PXF is not co-located with DATANODE on the following {0} host(s): {1}".format(len(mismatchHosts), hostsString) + childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PXF' } ) + + # Generate WARNING if any PXF is not co-located with a NAMENODE or a DATANODE + mismatchHosts = sorted(set(pxfHosts).difference(set(namenodeHosts).union(set(datanodeHosts)))) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "PXF is not co-located with NAMENODE or DATANODE on the following {0} host(s): {1}".format(len(mismatchHosts), hostsString) + childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'PXF' } ) + if "SPARK" in servicesList: if "SPARK_THRIFTSERVER" in servicesList: if not "HIVE_SERVER" in servicesList: message = "SPARK_THRIFTSERVER requires HIVE services to be selected." childItems.append( {"type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER'} ) - hmsHosts = [component["hostnames"] for component in componentsList - if component["component_name"] == "HIVE_METASTORE"][0] if "HIVE" in servicesList else [] - sparkTsHosts = [component["hostnames"] for component in componentsList - if component["component_name"] == "SPARK_THRIFTSERVER"][0] if "SPARK" in servicesList else [] + hmsHosts = self.__getHosts(componentsList, "HIVE_METASTORE") if "HIVE" in servicesList else [] + sparkTsHosts = self.__getHosts(componentsList, "SPARK_THRIFTSERVER") if "SPARK" in servicesList else [] # if Spark Thrift Server is deployed but no Hive Server is deployed if len(sparkTsHosts) > 0 and len(hmsHosts) == 0: @@ -110,6 +134,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor): parentItems.extend(childItems) return parentItems + def __getHosts(self, componentsList, componentName): + return [component["hostnames"] for component in componentsList if component["component_name"] == componentName][0] + def getNotPreferableOnServerComponents(self): parentComponents = super(HDP23StackAdvisor, self).getNotPreferableOnServerComponents() parentComponents.extend(['HAWQMASTER', 'HAWQSTANDBY']) http://git-wip-us.apache.org/repos/asf/ambari/blob/93517372/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 7a8d8ce..7e59a6f 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -182,6 +182,10 @@ class TestHDP23StackAdvisor(TestCase): return 'c6401.ambari.apache.org' if value is None else value + def __getHosts(self, componentsList, componentName): + return [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == componentName][0] + + def test_validations_hawqsegment(self): """ Test validation warning for HAWQ segment not colocated with DATANODE """ @@ -190,11 +194,9 @@ class TestHDP23StackAdvisor(TestCase): componentsListList = [service["components"] for service in services["services"]] componentsList = [item for sublist in componentsListList for item in sublist] - hawqsegmentComponent = [component["StackServiceComponents"] for component in componentsList - if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0] + hawqsegmentComponent = self.__getHosts(componentsList, "HAWQSEGMENT") hawqsegmentComponent["hostnames"] = ['c6401.ambari.apache.org'] - datanodeComponent = [component["StackServiceComponents"] for component in componentsList - if component["StackServiceComponents"]["component_name"] == "DATANODE"][0] + datanodeComponent = self.__getHosts(componentsList, "DATANODE") datanodeComponent["hostnames"] = ['c6402.ambari.apache.org'] validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) @@ -206,6 +208,111 @@ class TestHDP23StackAdvisor(TestCase): self.assertEquals(len(validations), 0) + def test_getComponentLayoutValidations_pxf_not_co_located_with_nn(self): + """ Test warning is generated when PXF is not co-located with NAMENODE """ + + services = self.load_json("services-hawq-pxf-hdfs.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + nameNodeComponent = self.__getHosts(componentsList, "NAMENODE") + dataNodeComponent = self.__getHosts(componentsList, "DATANODE") + pxfComponent = self.__getHosts(componentsList, "PXF") + nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"] + dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"] + pxfComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"] + + hosts = self.load_json("hosts-3-hosts.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 3) + + validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"] + self.assertEquals(len(validations), 1) + expected = { + "type": 'host-component', + "level": 'WARN', + "message": 'PXF is not co-located with NAMENODE on the following 1 host(s): c6401.ambari.apache.org', + "component-name": 'PXF' + } + self.assertEquals(validations[0], expected) + + + def test_getComponentLayoutValidations_pxf_not_co_located_with_dn(self): + """ Test warning is generated when PXF is not co-located with NAMENODE or DATANODE """ + + services = self.load_json("services-hawq-pxf-hdfs.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + nameNodeComponent = self.__getHosts(componentsList, "NAMENODE") + dataNodeComponent = self.__getHosts(componentsList, "DATANODE") + pxfComponent = self.__getHosts(componentsList, "PXF") + nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"] + dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"] + pxfComponent["hostnames"] = ["c6401.ambari.apache.org"] + + hosts = self.load_json("hosts-3-hosts.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 3) + + validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"] + self.assertEquals(len(validations), 1) + expected = { + "type": 'host-component', + "level": 'WARN', + "message": 'PXF is not co-located with DATANODE on the following 2 host(s): c6402.ambari.apache.org, c6403.ambari.apache.org', + "component-name": 'PXF' + } + self.assertEquals(validations[0], expected) + + + def test_getComponentLayoutValidations_pxf_not_co_located_with_nn_or_dn(self): + """ Test warning is generated when PXF is not co-located with NAMENODE or DATANODE """ + + services = self.load_json("services-hawq-pxf-hdfs.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + nameNodeComponent = self.__getHosts(componentsList, "NAMENODE") + dataNodeComponent = self.__getHosts(componentsList, "DATANODE") + pxfComponent = self.__getHosts(componentsList, "PXF") + nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"] + dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org"] + pxfComponent["hostnames"] = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"] + + hosts = self.load_json("hosts-3-hosts.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 3) + + validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"] + self.assertEquals(len(validations), 1) + expected = { + "type": 'host-component', + "level": 'WARN', + "message": 'PXF is not co-located with NAMENODE or DATANODE on the following 1 host(s): c6403.ambari.apache.org', + "component-name": 'PXF' + } + self.assertEquals(validations[0], expected) + + + def test_getComponentLayoutValidations_pxf_co_located_with_nn_and_dn(self): + """ Test NO warning is generated when PXF is co-located with NAMENODE and DATANODE """ + + services = self.load_json("services-hawq-pxf-hdfs.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + nameNodeComponent = self.__getHosts(componentsList, "NAMENODE") + dataNodeComponent = self.__getHosts(componentsList, "DATANODE") + pxfComponent = self.__getHosts(componentsList, "PXF") + nameNodeComponent["hostnames"] = ["c6401.ambari.apache.org"] + dataNodeComponent["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"] + pxfComponent["hostnames"] = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"] + + hosts = self.load_json("hosts-3-hosts.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 3) + + validations = [validation for validation in self.stackAdvisor.getComponentLayoutValidations(services, hosts) if validation["component-name"] == "PXF"] + self.assertEquals(len(validations), 0) + + @patch('socket.getfqdn', side_effect=fqdn_mock_result) def test_getComponentLayoutValidations_hawq_3_Hosts(self, socket_mock): """ Test layout validations for HAWQ components on a 3-node cluster """
