Repository: ambari Updated Branches: refs/heads/trunk 562bac816 -> 942b3dba7
AMBARI-18108. [BLUEPRINT] hive.metastore.uris contains single entry when deployed with metastore HA enabled. (Attila Doroszlai via stoader) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/942b3dba Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/942b3dba Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/942b3dba Branch: refs/heads/trunk Commit: 942b3dba72d946bbf82ea90d0f66b6d9c16c0eff Parents: 562bac8 Author: Attila Doroszlai <adorosz...@hortonworks.com> Authored: Fri Oct 21 17:27:02 2016 +0200 Committer: Toader, Sebastian <stoa...@hortonworks.com> Committed: Fri Oct 21 17:31:13 2016 +0200 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 7 +- .../BlueprintConfigurationProcessorTest.java | 103 ++++++++++++------- 2 files changed, 68 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/942b3dba/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 8cc6ec7..f890326 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -2242,16 +2242,17 @@ public class BlueprintConfigurationProcessor { String[] keyValuePairs = origValue.split(","); boolean firstValue = true; for (String keyValuePair : keyValuePairs) { + keyValuePair = keyValuePair.trim(); if (!firstValue) { updatedResult.append(","); } else { firstValue = false; } - String key = keyValuePair.split("=")[0]; + String key = keyValuePair.split("=")[0].trim(); if (mapOfKeysToUpdaters.containsKey(key)) { String result = mapOfKeysToUpdaters.get(key).updateForClusterCreate( - key, keyValuePair.split("=")[1], properties, topology); + key, keyValuePair.split("=")[1].trim(), properties, topology); // append the internal property result, escape out any commas in the internal property, // this is required due to the specific syntax of templeton.hive.properties updatedResult.append(key); http://git-wip-us.apache.org/repos/asf/ambari/blob/942b3dba/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 15fd4e7..5bedb9d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -58,7 +58,7 @@ import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.HostGroupImpl; import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.utils.CollectionPresentationUtils; +import org.apache.commons.lang.StringUtils; import org.easymock.EasyMockRule; import org.easymock.Mock; import org.easymock.MockType; @@ -86,7 +86,7 @@ public class BlueprintConfigurationProcessorTest { public EasyMockRule mocks = new EasyMockRule(this); @Mock(type = MockType.NICE) - private AmbariContext ambariConext; + private AmbariContext ambariContext; @Mock(type = MockType.NICE) private Blueprint bp; @@ -197,7 +197,7 @@ public class BlueprintConfigurationProcessorTest { @After public void tearDown() { - reset(bp, serviceInfo, stack, ambariConext); + reset(bp, serviceInfo, stack, ambariContext); } @Test @@ -2991,56 +2991,81 @@ public class BlueprintConfigurationProcessorTest { } @Test - public void testHiveConfigClusterUpdateDefaultValueWithMetaStoreHA() throws Exception { - final String expectedHostGroupName = "host_group_1"; - final String expectedHostNameOne = "c6401.ambari.apache.org"; - final String expectedHostNameTwo = "c6402.ambari.apache.org"; + public void testHivePropertiesLocalhostReplacedComma() throws Exception { + testHiveMetastoreHA(","); + } - final String expectedPropertyValue = - "hive.metastore.local=false,hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false"; + @Test + public void testHivePropertiesLocalhostReplacedCommaSpace() throws Exception { + testHiveMetastoreHA(", "); + } - Map<String, Map<String, String>> configProperties = - new HashMap<String, Map<String, String>>(); + @Test + public void testHivePropertiesLocalhostReplacedSpaceComma() throws Exception { + testHiveMetastoreHA(" ,"); + } - Map<String, String> webHCatSiteProperties = - new HashMap<String, String>(); + @Test + public void testHivePropertiesLocalhostReplacedSpaceCommaSpace() throws Exception { + testHiveMetastoreHA(" , "); + } - configProperties.put("webhcat-site", webHCatSiteProperties); + private void testHiveMetastoreHA(String separator) throws InvalidTopologyException, ConfigurationTopologyException { + final String[] parts = new String[] { + "hive.metastore.local=false", + "hive.metastore.uris=" + getThriftURI("localhost"), + "hive.metastore.sasl.enabled=false" + }; + final String[] hostNames = new String[] { "c6401.ambari.apache.org", "example.com", "c6402.ambari.apache.org" }; + final Set<String> expectedUris = new HashSet<>(); + for (String hostName : hostNames) { + expectedUris.add(getThriftURI(hostName)); + } - // setup properties that include host information - webHCatSiteProperties.put("templeton.hive.properties", - expectedPropertyValue); + final String initialPropertyValue = StringUtils.join(parts, separator); - Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); + Map<String, Map<String, String>> configProperties = new HashMap<>(); + Map<String, String> webHCatSiteProperties = new HashMap<>(); - Collection<String> hgComponents = new HashSet<String>(); - hgComponents.add("HIVE_METASTORE"); - TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostNameOne)); + configProperties.put("webhcat-site", webHCatSiteProperties); - Collection<String> hgComponents2 = new HashSet<String>(); - hgComponents2.add("HIVE_METASTORE"); - TestHostGroup group2 = new TestHostGroup("host_group_2", hgComponents2, Collections.singleton(expectedHostNameTwo)); + // setup properties that include host information + String propertyKey = "templeton.hive.properties"; + webHCatSiteProperties.put(propertyKey, initialPropertyValue); - Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); - hostGroups.add(group1); - hostGroups.add(group2); + Map<String, Map<String, Map<String, String>>> attributes = Collections.emptyMap(); + Configuration clusterConfig = new Configuration(configProperties, attributes); + + Collection<TestHostGroup> hostGroups = new HashSet<>(); + for (int i = 0; i < hostNames.length; ++i) { + Collection<String> components = new HashSet<>(Collections.singleton("HIVE_METASTORE")); + hostGroups.add(new TestHostGroup("host_group_" + i, components, Collections.singleton(hostNames[i]))); + } ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); + String updatedValue = webHCatSiteProperties.get(propertyKey); // verify that the host name for the metastore.uris property has been updated, and - // that both MetaStore Server URIs are included, using the required Hive Syntax + // that all MetaStore Server URIs are included, using the required Hive Syntax // Depends on hashing, string representation can be different - assertEquals("Unexpected config update for templeton.hive.properties", - "hive.metastore.local=false,hive.metastore.uris=", webHCatSiteProperties.get("templeton.hive.properties").substring(0, 47)); - assertEquals("Unexpected config update for templeton.hive.properties", - ",hive.metastore.sasl.enabled=false", webHCatSiteProperties.get("templeton.hive.properties").substring(123)); - List<String> parts = Arrays.asList(new String[]{"thrift://" + expectedHostNameOne + ":9933", "thrift://" + - expectedHostNameTwo + ":9933"}); - assertTrue("Unexpected config update for templeton.hive.properties", - CollectionPresentationUtils.isStringPermutationOfCollection(webHCatSiteProperties.get("templeton.hive.properties"), parts, "\\,", 47, 34)); + String prefix = parts[0] + ","; + assertTrue(updatedValue, updatedValue.startsWith(prefix)); + + String suffix = "," + parts[2]; + assertTrue(updatedValue, updatedValue.endsWith(suffix)); + + String part1 = updatedValue.replace(prefix, "").replace(suffix, ""); + String key = "hive.metastore.uris="; + assertTrue(part1, part1.startsWith(key)); + + Set<String> updatedUris = new HashSet<>(Arrays.asList(part1.replace(key, "").split("\\\\,"))); + assertEquals(expectedUris, updatedUris); + } + private static String getThriftURI(String hostName) { + return "thrift://" + hostName + ":9933"; } @Test @@ -8042,7 +8067,7 @@ public class BlueprintConfigurationProcessorTest { throws InvalidTopologyException { - replay(stack, serviceInfo, ambariConext); + replay(stack, serviceInfo, ambariContext); Map<String, HostGroupInfo> hostGroupInfo = new HashMap<String, HostGroupInfo>(); Collection<String> allServices = new HashSet<String>(); @@ -8085,7 +8110,7 @@ public class BlueprintConfigurationProcessorTest { replay(bp); ClusterTopology topology = new ClusterTopologyImpl - (ambariConext, 1L, blueprint, configuration, hostGroupInfo); + (ambariContext, 1L, blueprint, configuration, hostGroupInfo); topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.NEVER_APPLY); return topology;
