Repository: ambari Updated Branches: refs/heads/branch-2.1 12f9d0764 -> 8857e63b8
http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 02753d4..99cfa6c 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -16,6 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. ''' +import json import os import socket from unittest import TestCase @@ -51,6 +52,12 @@ class TestHDP23StackAdvisor(TestCase): self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic + def load_json(self, filename): + file = os.path.join(self.testDirectory, filename) + with open(file, 'rb') as f: + data = json.load(f) + return data + @patch('__builtin__.open') @patch('os.path.exists') def get_system_min_uid_magic(self, exists_mock, open_mock): @@ -71,6 +78,174 @@ class TestHDP23StackAdvisor(TestCase): open_mock.return_value = MagicFile() return self.get_system_min_uid_real() + + def test_createComponentLayoutRecommendations_hawq_1_Host(self): + """ Test that HAWQSTANDBY is not recommended on a single node cluster """ + + services = self.load_json("services-hawq-1-host.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + componentNames = [component["StackServiceComponents"]["component_name"] for component in componentsList] + self.assertTrue('HAWQSTANDBY' in componentNames) + + hosts = self.load_json("hosts-1-host.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 1) + + recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts) + + recommendedComponentsListList = [hostgroup["components"] for hostgroup in recommendations["blueprint"]["host_groups"]] + recommendedComponents = [item["name"] for sublist in recommendedComponentsListList for item in sublist] + self.assertTrue('HAWQMASTER' in recommendedComponents) + self.assertFalse('HAWQSTANDBY' in recommendedComponents) + self.assertTrue('HAWQSEGMENT' in recommendedComponents) + + + def test_createComponentLayoutRecommendations_hawq_3_Hosts(self): + """ Test that HAWQSTANDBY is recommended on a 3-node cluster """ + + services = self.load_json("services-hawq-3-hosts.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + componentNames = [component["StackServiceComponents"]["component_name"] for component in componentsList] + self.assertTrue('HAWQSTANDBY' in componentNames) + + hosts = self.load_json("hosts-3-hosts.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 3) + + recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts) + + recommendedComponentsListList = [hostgroup["components"] for hostgroup in recommendations["blueprint"]["host_groups"]] + recommendedComponents = [item["name"] for sublist in recommendedComponentsListList for item in sublist] + self.assertTrue('HAWQMASTER' in recommendedComponents) + self.assertTrue('HAWQSTANDBY' in recommendedComponents) + self.assertTrue('HAWQSEGMENT' in recommendedComponents) + + # make sure master components are not collocated + for sublist in recommendedComponentsListList: + hostComponents = [item["name"] for item in sublist] + self.assertFalse(set(['HAWQMASTER', 'HAWQSTANDBY']).issubset(hostComponents)) + + + def test_createComponentLayoutRecommendations_no_hawq_3_Hosts(self): + """ Test no failures when there are no HAWQ components """ + + services = self.load_json("services-nohawq-3-hosts.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + componentNames = [component["StackServiceComponents"]["component_name"] for component in componentsList] + self.assertFalse('HAWQMASTER' in componentNames) + self.assertFalse('HAWQSTANDBY' in componentNames) + self.assertFalse('HAWQSEGMENT' in componentNames) + + hosts = self.load_json("hosts-3-hosts.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 3) + + recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts) + + recommendedComponentsListList = [hostgroup["components"] for hostgroup in recommendations["blueprint"]["host_groups"]] + recommendedComponents = [item["name"] for sublist in recommendedComponentsListList for item in sublist] + self.assertFalse('HAWQMASTER' in recommendedComponents) + self.assertFalse('HAWQSTANDBY' in recommendedComponents) + self.assertFalse('HAWQSEGMENT' in recommendedComponents) + + + def fqdn_mock_result(value=None): + return 'c6401.ambari.apache.org' if value is None else value + + + @patch('socket.getfqdn', side_effect=fqdn_mock_result) + def test_getComponentLayoutValidations_hawq_3_Hosts(self, socket_mock): + """ Test layout validations for HAWQ components on a 3-node cluster """ + + # case-1: normal placement, no warnings + services = self.load_json("services-normal-hawq-3-hosts.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"] + hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"] + self.assertEquals(len(hawqMasterHosts[0]), 1) + self.assertEquals(len(hawqStandbyHosts[0]), 1) + self.assertNotEquals(hawqMasterHosts[0][0], hawqStandbyHosts[0][0]) + + hosts = self.load_json("hosts-3-hosts.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 3) + + validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) + self.assertEquals(len(validations), 0) + + # case-2: HAWQ masters are collocated + services = self.load_json("services-master_standby_colo-3-hosts.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"] + hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"] + self.assertEquals(len(hawqMasterHosts[0]), 1) + self.assertEquals(len(hawqStandbyHosts[0]), 1) + self.assertEquals(hawqMasterHosts[0][0], hawqStandbyHosts[0][0]) + + validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) + self.assertEquals(len(validations), 1) + expected={'component-name': 'HAWQSTANDBY', 'message': 'HAWQ Standby Master and HAWQ Master should not be deployed on the same host.', 'type': 'host-component', 'host': 'c6403.ambari.apache.org', 'level': 'ERROR'} + self.assertEquals(validations[0], expected) + + # case-3: HAWQ Master and Ambari Server are collocated + services = self.load_json("services-master_ambari_colo-3-hosts.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"] + hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"] + self.assertEquals(len(hawqMasterHosts[0]), 1) + self.assertEquals(len(hawqStandbyHosts[0]), 1) + self.assertNotEquals(hawqMasterHosts[0][0], hawqStandbyHosts[0][0]) + self.assertEquals(hawqMasterHosts[0][0], "c6401.ambari.apache.org") + + validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) + self.assertEquals(len(validations), 1) + expected={'component-name': 'HAWQMASTER', 'message': 'HAWQ Master and Ambari Server should not be deployed on the same host. If you leave them collocated, make sure to set HAWQ Master Port property to a value different from the port number used by Ambari Server database.', 'type': 'host-component', 'host': 'c6401.ambari.apache.org', 'level': 'WARN'} + self.assertEquals(validations[0], expected) + + # case-4: HAWQ Standby and Ambari Server are collocated + services = self.load_json("services-standby_ambari_colo-3-hosts.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"] + hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"] + self.assertEquals(len(hawqMasterHosts[0]), 1) + self.assertEquals(len(hawqStandbyHosts[0]), 1) + self.assertNotEquals(hawqMasterHosts[0][0], hawqStandbyHosts[0][0]) + self.assertEquals(hawqStandbyHosts[0][0], "c6401.ambari.apache.org") + + validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) + self.assertEquals(len(validations), 1) + expected={'component-name': 'HAWQSTANDBY', 'message': 'HAWQ Standby Master and Ambari Server should not be deployed on the same host. If you leave them collocated, make sure to set HAWQ Master Port property to a value different from the port number used by Ambari Server database.', 'type': 'host-component', 'host': 'c6401.ambari.apache.org', 'level': 'WARN'} + self.assertEquals(validations[0], expected) + + + @patch('socket.getfqdn', side_effect=fqdn_mock_result) + def test_getComponentLayoutValidations_nohawq_3_Hosts(self, socket_mock): + """ Test no failures when there are no HAWQ components on a 3-node cluster """ + + # normal placement, no warnings + services = self.load_json("services-normal-nohawq-3-hosts.json") + componentsListList = [service["components"] for service in services["services"]] + componentsList = [item for sublist in componentsListList for item in sublist] + hawqMasterHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER"] + hawqStandbyHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY"] + self.assertEquals(len(hawqMasterHosts), 0) + self.assertEquals(len(hawqStandbyHosts), 0) + + hosts = self.load_json("hosts-3-hosts.json") + hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]] + self.assertEquals(len(hostsList), 3) + + validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts) + self.assertEquals(len(validations), 0) + + def test_recommendHDFSConfigurations(self): configurations = {} clusterData = { http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-web/app/data/HDP2.3/site_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/HDP2.3/site_properties.js b/ambari-web/app/data/HDP2.3/site_properties.js index 6bdc373..cdbd9c2 100644 --- a/ambari-web/app/data/HDP2.3/site_properties.js +++ b/ambari-web/app/data/HDP2.3/site_properties.js @@ -231,6 +231,84 @@ hdp23properties.push({ "serviceName": "RANGER", "filename": "ranger-admin-site.xml", "category": "ADSettings" + }, + /*********************************************** HAWQ **********************************************/ + { + "name": "hawq_master_address_host", + "displayType": "masterHost", + "filename": "hawq-site.xml", + "category": "General", + "serviceName": "HAWQ", + "index": 0 + }, + { + "name": "hawq_standby_address_host", + "displayType": "masterHost", + "filename": "hawq-site.xml", + "category": "General", + "serviceName": "HAWQ", + "index": 1 + }, + { + "name": "hawq_master_address_port", + "displayType": "int", + "filename": "hawq-site.xml", + "category": "General", + "serviceName": "HAWQ", + "index": 2 + }, + { + "name": "hawq_segment_address_port", + "displayType": "int", + "filename": "hawq-site.xml", + "category": "General", + "serviceName": "HAWQ", + "index": 3 + }, + { + "name": "hawq_dfs_url", + "filename": "hawq-site.xml", + "category": "General", + "serviceName": "HAWQ", + "index": 4 + }, + { + "name": "hawq_master_directory", + "displayType": "directory", + "filename": "hawq-site.xml", + "category": "General", + "serviceName": "HAWQ", + "index": 5 + }, + { + "name": "hawq_master_temp_directory", + "displayType": "directory", + "filename": "hawq-site.xml", + "category": "General", + "serviceName": "HAWQ", + "index": 6 + }, + { + "name": "hawq_segment_directory", + "displayType": "directory", + "filename": "hawq-site.xml", + "category": "General", + "serviceName": "HAWQ", + "index": 7 + }, + { + "name": "hawq_segment_temp_directory", + "displayType": "directory", + "filename": "hawq-site.xml", + "category": "General", + "serviceName": "HAWQ", + "index": 8 + }, + { + "name": "content", + "serviceName": "HAWQ", + "filename": "gpcheck-env.xml", + "category": "AdvancedGpcheck" } ); http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-web/app/models/stack_service.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/models/stack_service.js b/ambari-web/app/models/stack_service.js index b783bd9..da2d41a 100644 --- a/ambari-web/app/models/stack_service.js +++ b/ambari-web/app/models/stack_service.js @@ -175,7 +175,7 @@ App.StackService = DS.Model.extend({ var configTypes = this.get('configTypes'); var serviceComponents = this.get('serviceComponents'); if (configTypes && Object.keys(configTypes).length) { - var pattern = ["General", "CapacityScheduler", "FaultTolerance", "Isolation", "Performance", "HIVE_SERVER2", "KDC", "Kadmin","^Advanced", "Env$", "^Custom", "Falcon - Oozie integration", "FalconStartupSite", "FalconRuntimeSite", "MetricCollector", "Settings$"]; + var pattern = ["General", "CapacityScheduler", "FaultTolerance", "Isolation", "Performance", "HIVE_SERVER2", "KDC", "Kadmin","^Advanced", "Env$", "^Custom", "Falcon - Oozie integration", "FalconStartupSite", "FalconRuntimeSite", "MetricCollector", "Settings$", "AdvancedGpcheck"]; configCategories = App.StackService.configCategories.call(this).filter(function (_configCategory) { var serviceComponentName = _configCategory.get('name'); var isServiceComponent = serviceComponents.someProperty('componentName', serviceComponentName); @@ -202,8 +202,10 @@ App.StackService.displayOrder = [ 'TEZ', 'GANGLIA', 'HIVE', + 'HAWQ', 'HBASE', 'PIG', + 'PXF', 'SQOOP', 'OOZIE', 'ZOOKEEPER', @@ -360,6 +362,12 @@ App.StackService.configCategories = function () { break; case 'SQOOP': break; + case 'HAWQ': + serviceConfigCategories.pushObjects([ + App.ServiceConfigCategory.create({ name: 'General', displayName: 'General'}), + App.ServiceConfigCategory.create({ name: 'AdvancedGpcheck', displayName: 'Advanced gpcheck'}) + ]); + break; default: serviceConfigCategories.pushObjects([ App.ServiceConfigCategory.create({ name: 'General', displayName: 'General'}) http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-web/app/utils/configs/config_property_helper.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/configs/config_property_helper.js b/ambari-web/app/utils/configs/config_property_helper.js index 2170590..cb1f66b 100644 --- a/ambari-web/app/utils/configs/config_property_helper.js +++ b/ambari-web/app/utils/configs/config_property_helper.js @@ -219,9 +219,35 @@ module.exports = { configProperty.set('isRequired', 'false'); } break; + case 'hawq_master_address_host': + configProperty.set('value', masterComponentHostsInDB.findProperty('component', 'HAWQMASTER').hostName); + break; + case 'hawq_standby_address_host': + var hawqStandby = masterComponentHostsInDB.findProperty('component', 'HAWQSTANDBY'); + if (hawqStandby) { + configProperty.set('value', hawqStandby.hostName); + } + break; + case 'hawq_dfs_url': + var nnHost = masterComponentHostsInDB.findProperty('component', 'NAMENODE').hostName; + this.setRecommendedValue(configProperty, hostWithPort, nnHost); + break; + case 'hawq_rm_yarn_address': + var rmHost = masterComponentHostsInDB.findProperty('component', 'RESOURCEMANAGER'); + if (rmHost) { + this.setRecommendedValue(configProperty, hostWithPort, rmHost.hostName); + } + break; + case 'hawq_rm_yarn_scheduler_address': + var rmHost = masterComponentHostsInDB.findProperty('component', 'RESOURCEMANAGER'); + if (rmHost) { + this.setRecommendedValue(configProperty, hostWithPort, rmHost.hostName); + } + break; } }, + /** * Get hive.metastore.uris initial value * @param hosts http://git-wip-us.apache.org/repos/asf/ambari/blob/8857e63b/ambari-web/test/data/HDP2.3/site_properties_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/data/HDP2.3/site_properties_test.js b/ambari-web/test/data/HDP2.3/site_properties_test.js index 1bc8a7f..48a1988 100644 --- a/ambari-web/test/data/HDP2.3/site_properties_test.js +++ b/ambari-web/test/data/HDP2.3/site_properties_test.js @@ -26,7 +26,7 @@ describe('hdp2SiteProperties', function () { * @stackProperties: All the properties that are derived from stack definition */ var stackProperties = siteProperties.filter(function(item){ - return (!(item.isRequiredByAgent === false || item.filename === 'alert_notification' || item.category === 'Ambari Principals' || item.name === 'oozie_hostname')) + return (!(item.isRequiredByAgent === false || item.filename === 'alert_notification' || item.category === 'Ambari Principals' || item.name === 'oozie_hostname' || item.serviceName === 'HAWQ')) }); stackProperties.forEach(function(siteProperty){ @@ -75,4 +75,4 @@ describe('hdp2SiteProperties', function () { }); }); -}); \ No newline at end of file +});
