Repository: ambari Updated Branches: refs/heads/branch-2.5 9ef6a6316 -> 46a528ffb
AMBARI-18546. BP format does not generate property oozie.service.JPAService.jdbc.url.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/46a528ff Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/46a528ff Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/46a528ff Branch: refs/heads/branch-2.5 Commit: 46a528ffbb5e806c10815b7e1555b9a811326c86 Parents: 9ef6a63 Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Authored: Sun Oct 9 23:52:35 2016 +0300 Committer: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Committed: Sun Oct 9 23:52:35 2016 +0300 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 101 ++++++++++++++--- .../BlueprintConfigurationProcessorTest.java | 111 +++++++++++++++++-- 2 files changed, 186 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/46a528ff/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 4776dc6..31d8daf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -83,7 +83,7 @@ public class BlueprintConfigurationProcessor { /** * Single host topology updaters */ - private static Map<String, Map<String, PropertyUpdater>> singleHostTopologyUpdaters = + protected static Map<String, Map<String, PropertyUpdater>> singleHostTopologyUpdaters = new HashMap<String, Map<String, PropertyUpdater>>(); /** @@ -116,7 +116,7 @@ public class BlueprintConfigurationProcessor { * removed from export, but do not require an update during * cluster creation */ - private static Map<String, Map<String, PropertyUpdater>> removePropertyUpdaters = + private Map<String, Map<String, PropertyUpdater>> removePropertyUpdaters = new HashMap<String, Map<String, PropertyUpdater>>(); /** @@ -200,6 +200,81 @@ public class BlueprintConfigurationProcessor { public BlueprintConfigurationProcessor(ClusterTopology clusterTopology) { this.clusterTopology = clusterTopology; + initRemovePropertyUpdaters(); + } + + public Map<String, Map<String, PropertyUpdater>> getRemovePropertyUpdaters() { + return removePropertyUpdaters; + } + + public void initRemovePropertyUpdaters() { + + if (containsHostFromHostGroups("oozie-site", "oozie.service.JPAService.jdbc.url")) { + Map<String, PropertyUpdater> oozieSiteUpdaters = singleHostTopologyUpdaters.get("oozie-site"); + Map<String, PropertyUpdater> oozieEnvUpdaters = singleHostTopologyUpdaters.get("oozie-env"); + if (oozieSiteUpdaters == null) { + oozieSiteUpdaters = new HashMap<>(); + } + if (oozieEnvUpdaters == null) { + oozieEnvUpdaters = new HashMap<>(); + } + oozieEnvUpdaters.put("oozie_existing_mysql_host", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieEnvUpdaters.put("oozie_existing_oracle_host", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieEnvUpdaters.put("oozie_existing_postgresql_host", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieEnvUpdaters.put("oozie_existing_oracle_host", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieEnvUpdaters.put("oozie_existing_postgresql_host", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieSiteUpdaters.put("oozie.service.JPAService.jdbc.url", new SingleHostTopologyUpdater("OOZIE_SERVER")); + + singleHostTopologyUpdaters.put("oozie-env", oozieEnvUpdaters); + singleHostTopologyUpdaters.put("oozie-site", oozieSiteUpdaters); + } else { + Map<String, PropertyUpdater> oozieEnvOriginalValueMap = new HashMap<String, PropertyUpdater>(); + Map<String, PropertyUpdater> oozieSiteOriginalValueMap = new HashMap<String, PropertyUpdater>(); + // register updaters for Oozie properties that may point to an external DB + oozieEnvOriginalValueMap.put("oozie_existing_mysql_host", new OriginalValuePropertyUpdater()); + oozieEnvOriginalValueMap.put("oozie_existing_oracle_host", new OriginalValuePropertyUpdater()); + oozieEnvOriginalValueMap.put("oozie_existing_postgresql_host", new OriginalValuePropertyUpdater()); + oozieEnvOriginalValueMap.put("oozie_existing_oracle_host", new OriginalValuePropertyUpdater()); + oozieEnvOriginalValueMap.put("oozie_existing_postgresql_host", new OriginalValuePropertyUpdater()); + oozieSiteOriginalValueMap.put("oozie.service.JPAService.jdbc.url", new OriginalValuePropertyUpdater()); + + removePropertyUpdaters.put("oozie-env", oozieEnvOriginalValueMap); + removePropertyUpdaters.put("oozie-site", oozieSiteOriginalValueMap); + } + + Map<String, PropertyUpdater> hiveEnvOriginalValueMap = new HashMap<String, PropertyUpdater>(); + // register updaters for Hive properties that may point to an external DB + hiveEnvOriginalValueMap.put("hive_existing_oracle_host", new OriginalValuePropertyUpdater()); + hiveEnvOriginalValueMap.put("hive_existing_mssql_server_2_host", new OriginalValuePropertyUpdater()); + hiveEnvOriginalValueMap.put("hive_existing_mssql_server_host", new OriginalValuePropertyUpdater()); + hiveEnvOriginalValueMap.put("hive_existing_postgresql_host", new OriginalValuePropertyUpdater()); + hiveEnvOriginalValueMap.put("hive_existing_mysql_host", new OriginalValuePropertyUpdater()); + + removePropertyUpdaters.put("hive-env", hiveEnvOriginalValueMap); + + } + + private boolean containsHostFromHostGroups(String configType, String propertyName) { + String propertyValue = clusterTopology.getConfiguration().getPropertyValue(configType, propertyName); + if (StringUtils.isEmpty(propertyValue)) { + return false; + } + // check fir bp import + Matcher m = HOSTGROUP_REGEX.matcher(propertyValue); + if (m.find()) { + return true; + } + + // check for bp export + for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) { + Collection<String> hosts = groupInfo.getHostNames(); + for (String host : hosts) { + if (propertyValue.contains(host)) { + return true; + } + } + } + return false; } public Collection<String> getRequiredHostGroups() { @@ -2277,7 +2352,6 @@ public class BlueprintConfigurationProcessor { Map<String, PropertyUpdater> hiveInteractiveSiteMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> oozieEnvMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> oozieEnvHeapSizeMap = new HashMap<String, PropertyUpdater>(); - Map<String, PropertyUpdater> oozieEnvOriginalValueMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> multiWebhcatSiteMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> multiHbaseSiteMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> multiStormSiteMap = new HashMap<String, PropertyUpdater>(); @@ -2312,7 +2386,6 @@ public class BlueprintConfigurationProcessor { singleHostTopologyUpdaters.put("yarn-site", yarnSiteMap); singleHostTopologyUpdaters.put("hive-site", hiveSiteMap); singleHostTopologyUpdaters.put("hive-interactive-env", hiveInteractiveEnvMap); - singleHostTopologyUpdaters.put("oozie-site", oozieSiteMap); singleHostTopologyUpdaters.put("storm-site", stormSiteMap); singleHostTopologyUpdaters.put("accumulo-site", accumuloSiteMap); singleHostTopologyUpdaters.put("falcon-startup.properties", falconStartupPropertiesMap); @@ -2357,9 +2430,6 @@ public class BlueprintConfigurationProcessor { dbHostTopologyUpdaters.put("hive-site", dbHiveSiteMap); - removePropertyUpdaters.put("oozie-env", oozieEnvOriginalValueMap); - removePropertyUpdaters.put("oozie-site", oozieSiteOriginalValueMap); - nonTopologyUpdaters.put("hive-site", hiveSiteNonTopologyMap); nonTopologyUpdaters.put("kafka-broker", kafkaBrokerNonTopologyMap); nonTopologyUpdaters.put("storm-site", stormSiteNonTopologyMap); @@ -2537,14 +2607,17 @@ public class BlueprintConfigurationProcessor { // OOZIE_SERVER - oozieSiteMap.put("oozie.base.url", new SingleHostTopologyUpdater("OOZIE_SERVER")); - oozieSiteMap.put("oozie.authentication.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); - oozieSiteMap.put("oozie.ha.authentication.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); - oozieSiteMap.put("oozie.service.HadoopAccessorService.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); - multiCoreSiteMap.put("hadoop.proxyuser.oozie.hosts", new MultipleHostTopologyUpdater("OOZIE_SERVER")); + Map<String, PropertyUpdater> oozieStringPropertyUpdaterMap = singleHostTopologyUpdaters.get("oozie-site"); + if (oozieStringPropertyUpdaterMap == null) { + oozieStringPropertyUpdaterMap = new HashMap<>(); + } + oozieStringPropertyUpdaterMap.put("oozie.base.url", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieStringPropertyUpdaterMap.put("oozie.authentication.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieStringPropertyUpdaterMap.put("oozie.ha.authentication.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieStringPropertyUpdaterMap.put("oozie.service.HadoopAccessorService.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); + singleHostTopologyUpdaters.put("oozie-site", oozieStringPropertyUpdaterMap); - // register updaters for Oozie properties that may point to an external DB - oozieEnvOriginalValueMap.put("oozie_existing_mysql_host", new OriginalValuePropertyUpdater()); + multiCoreSiteMap.put("hadoop.proxyuser.oozie.hosts", new MultipleHostTopologyUpdater("OOZIE_SERVER")); oozieSiteOriginalValueMap.put("oozie.service.JPAService.jdbc.url", new OriginalValuePropertyUpdater()); // ZOOKEEPER_SERVER http://git-wip-us.apache.org/repos/asf/ambari/blob/46a528ff/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 13927f8..899b4fe 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -18,6 +18,17 @@ package org.apache.ambari.server.controller.internal; +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertFalse; +import static junit.framework.Assert.assertNotNull; +import static junit.framework.Assert.assertNull; +import static junit.framework.Assert.assertTrue; +import static junit.framework.Assert.fail; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -29,10 +40,9 @@ import java.util.List; import java.util.Map; import java.util.Set; -import com.google.common.collect.*; import org.apache.ambari.server.controller.StackConfigurationResponse; -import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.PropertyDependencyInfo; +import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.ValueAttributesInfo; import org.apache.ambari.server.topology.AdvisedConfiguration; @@ -58,16 +68,11 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertFalse; -import static junit.framework.Assert.assertNotNull; -import static junit.framework.Assert.assertNull; -import static junit.framework.Assert.assertTrue; -import static junit.framework.Assert.fail; -import static org.easymock.EasyMock.anyObject; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.reset; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; /** * BlueprintConfigurationProcessor unit tests. @@ -1693,6 +1698,10 @@ public class BlueprintConfigurationProcessorTest { // call top-level export method configProcessor.doUpdateForBlueprintExport(); + // check that jdbc url and related properties are removed if oozie external db is on host which not included to cluster + assertFalse(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url")); + assertTrue(configProcessor.getRemovePropertyUpdaters().get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url")); + assertEquals("oozie property not exported correctly", createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.base.url")); assertEquals("oozie property not exported correctly", @@ -1717,6 +1726,84 @@ public class BlueprintConfigurationProcessorTest { } @Test + public void testOozieJDBCPropertiesNotRemoved() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedHostNameTwo = "c6402.ambari.apache.org"; + final String expectedHostGroupName = "host_group_1"; + final String expectedHostGroupNameTwo = "host_group_2"; + final String expectedPortNum = "80000"; + + Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); + Map<String, String> oozieSiteProperties = new HashMap<String, String>(); + + configProperties.put("oozie-site", oozieSiteProperties); + + oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql://" + expectedHostNameTwo + "/ooziedb"); + + Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); + Collection<String> hgComponents = new HashSet<String>(); + hgComponents.add("OOZIE_SERVER"); + hgComponents.add("ZOOKEEPER_SERVER"); + TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); + + Collection<String> hgComponents2 = new HashSet<String>(); + hgComponents2.add("OOZIE_SERVER"); + hgComponents2.add("ZOOKEEPER_SERVER"); + TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo)); + + Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); + hostGroups.add(group1); + hostGroups.add(group2); + + expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); + + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + BlueprintConfigurationProcessor blueprintConfigurationProcessor = new BlueprintConfigurationProcessor(topology); + + assertTrue(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url")); + assertNull(blueprintConfigurationProcessor.getRemovePropertyUpdaters().get("oozie-site")); + } + + @Test + public void testOozieJDBCPropertyAddedToSingleHostMapDuringImport() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedHostNameTwo = "c6402.ambari.apache.org"; + final String expectedHostGroupName = "host_group_1"; + final String expectedHostGroupNameTwo = "host_group_2"; + final String expectedPortNum = "80000"; + + Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); + Map<String, String> oozieSiteProperties = new HashMap<String, String>(); + + configProperties.put("oozie-site", oozieSiteProperties); + + oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql://" + "%HOSTGROUP::group1%" + "/ooziedb"); + + Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); + Collection<String> hgComponents = new HashSet<String>(); + hgComponents.add("OOZIE_SERVER"); + hgComponents.add("ZOOKEEPER_SERVER"); + TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); + + Collection<String> hgComponents2 = new HashSet<String>(); + hgComponents2.add("OOZIE_SERVER"); + hgComponents2.add("ZOOKEEPER_SERVER"); + TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo)); + + Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); + hostGroups.add(group1); + hostGroups.add(group2); + + expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); + + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + BlueprintConfigurationProcessor blueprintConfigurationProcessor = new BlueprintConfigurationProcessor(topology); + + assertTrue(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url")); + assertNull(blueprintConfigurationProcessor.getRemovePropertyUpdaters().get("oozie-site")); + } + + @Test public void testZookeeperConfigExported() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.ambari.apache.org";
