Repository: ambari Updated Branches: refs/heads/branch-2.2 360d5d2d4 -> 3d9f618f9
AMBARI-14911: Populate hawq_site properties when YARN is installed after HAWQ. (adenissov via jaoki) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3d9f618f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3d9f618f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3d9f618f Branch: refs/heads/branch-2.2 Commit: 3d9f618f93dfa4d0044835c9472a24e24a5bd4ca Parents: 360d5d2 Author: Jun Aoki <ja...@apache.org> Authored: Fri Feb 5 15:48:55 2016 -0800 Committer: Jun Aoki <ja...@apache.org> Committed: Fri Feb 5 15:48:55 2016 -0800 ---------------------------------------------------------------------- .../HAWQ/2.0.0/configuration/hawq-site.xml | 12 +++++ .../stacks/HDP/2.3/services/stack_advisor.py | 37 +++++++++++--- .../stacks/2.3/common/test_stack_advisor.py | 52 +++++++++++++++++++- 3 files changed, 93 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/3d9f618f/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml index f034749..f195215 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml @@ -153,6 +153,12 @@ <description> The address of YARN resource manager server. </description> + <depends-on> + <property> + <type>yarn-site</type> + <name>yarn.resourcemanager.address</name> + </property> + </depends-on> </property> <property> @@ -161,6 +167,12 @@ <description> The address of YARN scheduler server. </description> + <depends-on> + <property> + <type>yarn-site</type> + <name>yarn.resourcemanager.scheduler.address</name> + </property> + </depends-on> </property> <property> http://git-wip-us.apache.org/repos/asf/ambari/blob/3d9f618f/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index b9bb8af..71b0e2a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -668,19 +668,32 @@ class HDP23StackAdvisor(HDP22StackAdvisor): def recommendHAWQConfigurations(self, configurations, clusterData, services, hosts): + if "hawq-site" not in services["configurations"]: + return + hawq_site = services["configurations"]["hawq-site"]["properties"] putHawqSiteProperty = self.putProperty(configurations, "hawq-site", services) - if self.isHawqMasterComponentOnAmbariServer(services): - if "hawq-site" in services["configurations"] and "hawq_master_address_port" in services["configurations"]["hawq-site"]["properties"]: - putHawqSiteProperty('hawq_master_address_port', '') - # calculate optimal number of virtual segments componentsListList = [service["components"] for service in services["services"]] componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + + # remove master port when master is colocated with Ambari server + if self.isHawqMasterComponentOnAmbariServer(services) and "hawq_master_address_port" in hawq_site: + putHawqSiteProperty('hawq_master_address_port', '') + + # calculate optimal number of virtual segments numSegments = len(self.__getHosts(componentsList, "HAWQSEGMENT")) # update default if segments are deployed - if numSegments and "hawq-site" in services["configurations"] and "default_segment_num" in services["configurations"]["hawq-site"]["properties"]: + if numSegments and "default_segment_num" in hawq_site: factor = 6 if numSegments < 50 else 4 putHawqSiteProperty('default_segment_num', numSegments * factor) - + + # update YARN RM urls with the values from yarn-site if YARN is installed + if "YARN" in servicesList and "yarn-site" in services["configurations"]: + yarn_site = services["configurations"]["yarn-site"]["properties"] + for hs_prop, ys_prop in self.getHAWQYARNPropertyMapping().items(): + if hs_prop in hawq_site and ys_prop in yarn_site: + putHawqSiteProperty(hs_prop, yarn_site[ys_prop]) + def getServiceConfigurationValidators(self): parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators() childValidators = { @@ -934,8 +947,20 @@ class HDP23StackAdvisor(HDP22StackAdvisor): display_name = 'HAWQ Segment temp directory' self.validateIfRootDir (properties, validationItems, prop_name, display_name) + # 3. Check YARN RM address properties + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if "YARN" in servicesList and "yarn-site" in configurations: + yarn_site = getSiteProperties(configurations, "yarn-site") + for hs_prop, ys_prop in self.getHAWQYARNPropertyMapping().items(): + if hs_prop in hawq_site and ys_prop in yarn_site and hawq_site[hs_prop] != yarn_site[ys_prop]: + message = "Expected value: {0} (this property should have the same value as the property {1} in yarn-site)".format(yarn_site[ys_prop], ys_prop) + validationItems.append({"config-name": hs_prop, "item": self.getWarnItem(message)}) + return self.toConfigurationValidationProblems(validationItems, "hawq-site") def isComponentUsingCardinalityForLayout(self, componentName): return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER'] + + def getHAWQYARNPropertyMapping(self): + return { "hawq_rm_yarn_address": "yarn.resourcemanager.address", "hawq_rm_yarn_scheduler_address": "yarn.resourcemanager.scheduler.address" } http://git-wip-us.apache.org/repos/asf/ambari/blob/3d9f618f/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index cc397c3..12a2970 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1311,8 +1311,14 @@ class TestHDP23StackAdvisor(TestCase): componentsListList = [service["components"] for service in services["services"]] componentsList = [item for sublist in componentsListList for item in sublist] hawqSegmentComponent = [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0] - services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24"}} + # setup default configuration values + services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24", + "hawq_rm_yarn_address": "localhost:8032", + "hawq_rm_yarn_scheduler_address": "localhost:8030"}} + services["configurations"]["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050", + "yarn.resourcemanager.scheduler.address": "host1:8030"}} + services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]}) configurations = {} clusterData = {} @@ -1321,6 +1327,10 @@ class TestHDP23StackAdvisor(TestCase): self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None) self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(3 * 6)) + # check derived properties + self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_address"], "host1:8050") + self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_scheduler_address"], "host1:8030") + # Test 2 - with 49 segments hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(49)] self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None) @@ -1333,6 +1343,7 @@ class TestHDP23StackAdvisor(TestCase): # Test 4 - with no segments configurations = {} + services["configurations"]["hawq-site"] = {"properties":{'hawq-site': {'properties': {}}}} hawqSegmentComponent["hostnames"] = [] self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None) self.assertEquals(configurations, {'hawq-site': {'properties': {}}}) @@ -1602,4 +1613,41 @@ class TestHDP23StackAdvisor(TestCase): # 6 services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"] = "false" problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts) - self.assertEqual(len(problems), 0) \ No newline at end of file + self.assertEqual(len(problems), 0) + + def test_validateHAWQConfigurations(self): + services = self.load_json("services-hawq-3-hosts.json") + # setup default configuration values + configurations = services["configurations"] + configurations["hawq-site"] = {"properties": {"hawq_rm_yarn_address": "localhost:8032", + "hawq_rm_yarn_scheduler_address": "localhost:8030"}} + configurations["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050", + "yarn.resourcemanager.scheduler.address": "host1:8030"}} + services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]}) + properties = configurations["hawq-site"]["properties"] + defaults = {} + hosts = {} + + expected_warnings = { + 'hawq_rm_yarn_address': { + 'config-type': 'hawq-site', + 'message': 'Expected value: host1:8050 (this property should have the same value as the property yarn.resourcemanager.address in yarn-site)', + 'type': 'configuration', + 'config-name': 'hawq_rm_yarn_address', + 'level': 'WARN' + }, + 'hawq_rm_yarn_scheduler_address': { + 'config-type': 'hawq-site', + 'message': 'Expected value: host1:8030 (this property should have the same value as the property yarn.resourcemanager.scheduler.address in yarn-site)', + 'type': 'configuration', + 'config-name': 'hawq_rm_yarn_scheduler_address', + 'level': 'WARN' + } + } + + problems = self.stackAdvisor.validateHAWQConfigurations(properties, defaults, configurations, services, hosts) + problems_dict = {} + for problem in problems: + problems_dict[problem['config-name']] = problem + self.assertEqual(len(problems), 2) + self.assertEqual(problems_dict, expected_warnings)