Repository: ambari Updated Branches: refs/heads/trunk 3a9ecc092 -> 723333de6
AMBARI-14494: Generate warning on deploy wizard UI when HAWQ segment is not co-located with a datanode. (lavjain via jaoki) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/723333de Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/723333de Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/723333de Branch: refs/heads/trunk Commit: 723333de658ee1070d91938e419d5191f2e07d55 Parents: 3a9ecc0 Author: Jun Aoki <ja...@apache.org> Authored: Mon Jan 11 15:59:42 2016 -0800 Committer: Jun Aoki <ja...@apache.org> Committed: Mon Jan 11 15:59:42 2016 -0800 ---------------------------------------------------------------------- .../stacks/HDP/2.3/services/stack_advisor.py | 93 +++++++++++--------- .../stacks/2.3/common/test_stack_advisor.py | 24 +++++ 2 files changed, 75 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/723333de/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index dc64e24..0da566b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -53,50 +53,59 @@ class HDP23StackAdvisor(HDP22StackAdvisor): def getComponentLayoutValidations(self, services, hosts): parentItems = super(HDP23StackAdvisor, self).getComponentLayoutValidations(services, hosts) - hiveExists = "HIVE" in [service["StackServices"]["service_name"] for service in services["services"]] - sparkExists = "SPARK" in [service["StackServices"]["service_name"] for service in services["services"]] - - if not "HAWQ" in [service["StackServices"]["service_name"] for service in services["services"]] and not sparkExists: - return parentItems - + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] childItems = [] - hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] - hostsCount = len(hostsList) - componentsListList = [service["components"] for service in services["services"]] - componentsList = [item for sublist in componentsListList for item in sublist] - hawqMasterHosts = [hostname for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER" for hostname in component["StackServiceComponents"]["hostnames"]] - hawqStandbyHosts = [hostname for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY" for hostname in component["StackServiceComponents"]["hostnames"]] - - # single node case is not analyzed because HAWQ Standby Master will not be present in single node topology due to logic in createComponentLayoutRecommendations() - if len(hawqMasterHosts) == 1 and len(hawqStandbyHosts) == 1 and hawqMasterHosts == hawqStandbyHosts: - message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same host." - childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } ) - - if len(hawqMasterHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqMasterHosts[0]): - message = "HAWQ Master and Ambari Server should not be deployed on the same host. " \ - "If you leave them collocated, make sure to set HAWQ Master Port property " \ - "to a value different from the port number used by Ambari Server database." - childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQMASTER', "host": hawqMasterHosts[0] } ) - - if len(hawqStandbyHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqStandbyHosts[0]): - message = "HAWQ Standby Master and Ambari Server should not be deployed on the same host. " \ - "If you leave them collocated, make sure to set HAWQ Master Port property " \ - "to a value different from the port number used by Ambari Server database." - childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } ) - - if "SPARK_THRIFTSERVER" in [service["StackServices"]["service_name"] for service in services["services"]]: - if not "HIVE_SERVER" in [service["StackServices"]["service_name"] for service in services["services"]]: - message = "SPARK_THRIFTSERVER requires HIVE services to be selected." - childItems.append( {"type": 'host-component', "level": 'ERROR', "message": messge, "component-name": 'SPARK_THRIFTSERVER'} ) - - hmsHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HIVE_METASTORE"][0] if hiveExists else [] - sparkTsHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "SPARK_THRIFTSERVER"][0] if sparkExists else [] - - # if Spark Thrift Server is deployed but no Hive Server is deployed - if len(sparkTsHosts) > 0 and len(hmsHosts) == 0: - message = "SPARK_THRIFTSERVER requires HIVE_METASTORE to be selected/deployed." - childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER' } ) + if "HAWQ" in servicesList: + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + hostsCount = len(hostsList) + + hawqMasterHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "HAWQMASTER"][0] + hawqStandbyHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "HAWQSTANDBY"][0] + hawqSegmentHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "HAWQSEGMENT"][0] + datanodeHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "DATANODE"][0] + + # Generate WARNING if any HAWQSEGMENT is not colocated with a DATANODE + mismatchHosts = set(hawqSegmentHosts).symmetric_difference(set(datanodeHosts)) + if len(mismatchHosts) > 0: + hostsString = ', '.join(mismatchHosts) + message = "HAWQSEGMENT is not co-located with DATANODE on the following {0} host(s): {1}".format(len(mismatchHosts), hostsString) + childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSEGMENT' } ) + + # single node case is not analyzed because HAWQ Standby Master will not be present in single node topology due to logic in createComponentLayoutRecommendations() + if len(hawqMasterHosts) == 1 and len(hawqStandbyHosts) == 1 and hawqMasterHosts == hawqStandbyHosts: + message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same host." + childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } ) + + if len(hawqMasterHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqMasterHosts[0]): + message = "HAWQ Master and Ambari Server should not be deployed on the same host. " \ + "If you leave them collocated, make sure to set HAWQ Master Port property " \ + "to a value different from the port number used by Ambari Server database." + childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQMASTER', "host": hawqMasterHosts[0] } ) + + if len(hawqStandbyHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqStandbyHosts[0]): + message = "HAWQ Standby Master and Ambari Server should not be deployed on the same host. " \ + "If you leave them collocated, make sure to set HAWQ Master Port property " \ + "to a value different from the port number used by Ambari Server database." + childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } ) + + if "SPARK" in servicesList: + if "SPARK_THRIFTSERVER" in servicesList: + if not "HIVE_SERVER" in servicesList: + message = "SPARK_THRIFTSERVER requires HIVE services to be selected." + childItems.append( {"type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER'} ) + + hmsHosts = [component["hostnames"] for component in componentsList + if component["component_name"] == "HIVE_METASTORE"][0] if "HIVE" in servicesList else [] + sparkTsHosts = [component["hostnames"] for component in componentsList + if component["component_name"] == "SPARK_THRIFTSERVER"][0] if "SPARK" in servicesList else [] + + # if Spark Thrift Server is deployed but no Hive Server is deployed + if len(sparkTsHosts) > 0 and len(hmsHosts) == 0: + message = "SPARK_THRIFTSERVER requires HIVE_METASTORE to be selected/deployed." + childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER' } ) parentItems.extend(childItems) return parentItems http://git-wip-us.apache.org/repos/asf/ambari/blob/723333de/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 22fcb1b..7a8d8ce 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -182,6 +182,30 @@ class TestHDP23StackAdvisor(TestCase): return 'c6401.ambari.apache.org' if value is None else value + def test_validations_hawqsegment(self): + """ Test validation warning for HAWQ segment not colocated with DATANODE """ + + services = self.load_json("services-normal-hawq-3-hosts.json") + hosts = self.load_json("hosts-3-hosts.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + + hawqsegmentComponent = [component["StackServiceComponents"] for component in componentsList + if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0] + hawqsegmentComponent["hostnames"] = ['c6401.ambari.apache.org'] + datanodeComponent = [component["StackServiceComponents"] for component in componentsList + if component["StackServiceComponents"]["component_name"] == "DATANODE"][0] + datanodeComponent["hostnames"] = ['c6402.ambari.apache.org'] + + validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) + expected = {'component-name': 'HAWQSEGMENT', 'message': 'HAWQSEGMENT is not co-located with DATANODE on the following 2 host(s): c6402.ambari.apache.org, c6401.ambari.apache.org', 'type': 'host-component', 'level': 'WARN'} + self.assertEquals(validations[0], expected) + + datanodeComponent["hostnames"] = ['c6401.ambari.apache.org'] + validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) + self.assertEquals(len(validations), 0) + + @patch('socket.getfqdn', side_effect=fqdn_mock_result) def test_getComponentLayoutValidations_hawq_3_Hosts(self, socket_mock): """ Test layout validations for HAWQ components on a 3-node cluster """
