Repository: ambari Updated Branches: refs/heads/branch-2.2 801e00c2c -> 6460f686c
AMBARI-14840: Expose default_segment_num property for HAWQ (adenissov via jaoki) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6460f686 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6460f686 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6460f686 Branch: refs/heads/branch-2.2 Commit: 6460f686c54bb3d784d1dbbd2d692a1ec3dbfa20 Parents: 801e00c Author: Jun Aoki <ja...@apache.org> Authored: Fri Jan 29 13:35:07 2016 -0800 Committer: Jun Aoki <ja...@apache.org> Committed: Fri Jan 29 13:35:07 2016 -0800 ---------------------------------------------------------------------- .../HAWQ/2.0.0/configuration/hawq-site.xml | 14 ++++++++ .../stacks/HDP/2.3/services/stack_advisor.py | 9 +++++- .../common/services-normal-hawq-3-hosts.json | 11 +++++++ .../stacks/2.3/common/test_stack_advisor.py | 34 ++++++++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6460f686/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml index 90982e9..f034749 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml @@ -205,4 +205,18 @@ </description> </property> + <property> + <name>default_segment_num</name> + <display-name>Default Number of Virtual Segments</display-name> + <value>24</value> + <description> + The default number of virtual segments to use when executing a query statement. When the query is actually executed, + the number of virtual segments may differ from this number depending on the query's needs. + When expanding the cluster, you should adjust this number to reflect the number of nodes in the new cluster times the number of virtual segments per node. + </description> + <value-attributes> + <type>int</type> + </value-attributes> + </property> + </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6460f686/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 0efed8e..b9bb8af 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -672,7 +672,14 @@ class HDP23StackAdvisor(HDP22StackAdvisor): if self.isHawqMasterComponentOnAmbariServer(services): if "hawq-site" in services["configurations"] and "hawq_master_address_port" in services["configurations"]["hawq-site"]["properties"]: putHawqSiteProperty('hawq_master_address_port', '') - + # calculate optimal number of virtual segments + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist] + numSegments = len(self.__getHosts(componentsList, "HAWQSEGMENT")) + # update default if segments are deployed + if numSegments and "hawq-site" in services["configurations"] and "default_segment_num" in services["configurations"]["hawq-site"]["properties"]: + factor = 6 if numSegments < 50 else 4 + putHawqSiteProperty('default_segment_num', numSegments * factor) def getServiceConfigurationValidators(self): parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators() http://git-wip-us.apache.org/repos/asf/ambari/blob/6460f686/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json b/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json index 9fab56d..7a5a7b7 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json +++ b/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json @@ -358,6 +358,17 @@ "type" : "hawq-site.xml" }, "dependencies" : [ ] + }, { + "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/configurations/default_segment_num", + "StackConfigurations" : { + "property_depends_on" : [ ], + "property_name" : "default_segment_num", + "service_name" : "HAWQ", + "stack_name" : "HDP", + "stack_version" : "2.3", + "type" : "hawq-site.xml" + }, + "dependencies" : [ ] } ] }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HDFS", http://git-wip-us.apache.org/repos/asf/ambari/blob/6460f686/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 5267fac..cc397c3 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1304,6 +1304,40 @@ class TestHDP23StackAdvisor(TestCase): self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) + def test_recommendHAWQConfigurations(self): + + # original cluster data with 3 segments + services = self.load_json("services-normal-hawq-3-hosts.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + hawqSegmentComponent = [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0] + services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24"}} + + configurations = {} + clusterData = {} + + # Test 1 - with 3 segments + self.assertEquals(len(hawqSegmentComponent["hostnames"]), 3) + self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None) + self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(3 * 6)) + + # Test 2 - with 49 segments + hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(49)] + self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None) + self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(49 * 6)) + + # Test 3 - with 50 segments (threshold for new factor) + hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(50)] + self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None) + self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(50 * 4)) + + # Test 4 - with no segments + configurations = {} + hawqSegmentComponent["hostnames"] = [] + self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None) + self.assertEquals(configurations, {'hawq-site': {'properties': {}}}) + + def test_validateHiveConfigurations(self): properties = {"hive_security_authorization": "None", "hive.exec.orc.default.stripe.size": "8388608",
