Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-18456 5cf5c8349 -> ed2018bf0
AMBARI-18546. BP format does not generate property oozie.service.JPAService.jdbc.url.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/58afcf93 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/58afcf93 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/58afcf93 Branch: refs/heads/branch-feature-AMBARI-18456 Commit: 58afcf93d8445c3b1ffacdd33c1cb12a8e8414cb Parents: 72f1f6f Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Authored: Mon Oct 10 00:22:49 2016 +0300 Committer: Vitaly Brodetskyi <vbrodets...@hortonworks.com> Committed: Mon Oct 10 00:22:49 2016 +0300 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 113 ++++++++++++++----- .../BlueprintConfigurationProcessorTest.java | 82 ++++++++++++++ 2 files changed, 169 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/58afcf93/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index ba09368..8ff568a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -84,7 +84,7 @@ public class BlueprintConfigurationProcessor { /** * Single host topology updaters */ - private static Map<String, Map<String, PropertyUpdater>> singleHostTopologyUpdaters = + protected static Map<String, Map<String, PropertyUpdater>> singleHostTopologyUpdaters = new HashMap<String, Map<String, PropertyUpdater>>(); /** @@ -117,7 +117,7 @@ public class BlueprintConfigurationProcessor { * removed from export, but do not require an update during * cluster creation */ - private static Map<String, Map<String, PropertyUpdater>> removePropertyUpdaters = + private Map<String, Map<String, PropertyUpdater>> removePropertyUpdaters = new HashMap<String, Map<String, PropertyUpdater>>(); /** @@ -196,6 +196,81 @@ public class BlueprintConfigurationProcessor { public BlueprintConfigurationProcessor(ClusterTopology clusterTopology) { this.clusterTopology = clusterTopology; + initRemovePropertyUpdaters(); + } + + public Map<String, Map<String, PropertyUpdater>> getRemovePropertyUpdaters() { + return removePropertyUpdaters; + } + + public void initRemovePropertyUpdaters() { + + if (containsHostFromHostGroups("oozie-site", "oozie.service.JPAService.jdbc.url")) { + Map<String, PropertyUpdater> oozieSiteUpdaters = singleHostTopologyUpdaters.get("oozie-site"); + Map<String, PropertyUpdater> oozieEnvUpdaters = singleHostTopologyUpdaters.get("oozie-env"); + if (oozieSiteUpdaters == null) { + oozieSiteUpdaters = new HashMap<>(); + } + if (oozieEnvUpdaters == null) { + oozieEnvUpdaters = new HashMap<>(); + } + oozieEnvUpdaters.put("oozie_existing_mysql_host", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieEnvUpdaters.put("oozie_existing_oracle_host", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieEnvUpdaters.put("oozie_existing_postgresql_host", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieEnvUpdaters.put("oozie_existing_oracle_host", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieEnvUpdaters.put("oozie_existing_postgresql_host", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieSiteUpdaters.put("oozie.service.JPAService.jdbc.url", new SingleHostTopologyUpdater("OOZIE_SERVER")); + + singleHostTopologyUpdaters.put("oozie-env", oozieEnvUpdaters); + singleHostTopologyUpdaters.put("oozie-site", oozieSiteUpdaters); + } else { + Map<String, PropertyUpdater> oozieEnvOriginalValueMap = new HashMap<String, PropertyUpdater>(); + Map<String, PropertyUpdater> oozieSiteOriginalValueMap = new HashMap<String, PropertyUpdater>(); + // register updaters for Oozie properties that may point to an external DB + oozieEnvOriginalValueMap.put("oozie_existing_mysql_host", new OriginalValuePropertyUpdater()); + oozieEnvOriginalValueMap.put("oozie_existing_oracle_host", new OriginalValuePropertyUpdater()); + oozieEnvOriginalValueMap.put("oozie_existing_postgresql_host", new OriginalValuePropertyUpdater()); + oozieEnvOriginalValueMap.put("oozie_existing_oracle_host", new OriginalValuePropertyUpdater()); + oozieEnvOriginalValueMap.put("oozie_existing_postgresql_host", new OriginalValuePropertyUpdater()); + oozieSiteOriginalValueMap.put("oozie.service.JPAService.jdbc.url", new OriginalValuePropertyUpdater()); + + removePropertyUpdaters.put("oozie-env", oozieEnvOriginalValueMap); + removePropertyUpdaters.put("oozie-site", oozieSiteOriginalValueMap); + } + + Map<String, PropertyUpdater> hiveEnvOriginalValueMap = new HashMap<String, PropertyUpdater>(); + // register updaters for Hive properties that may point to an external DB + hiveEnvOriginalValueMap.put("hive_existing_oracle_host", new OriginalValuePropertyUpdater()); + hiveEnvOriginalValueMap.put("hive_existing_mssql_server_2_host", new OriginalValuePropertyUpdater()); + hiveEnvOriginalValueMap.put("hive_existing_mssql_server_host", new OriginalValuePropertyUpdater()); + hiveEnvOriginalValueMap.put("hive_existing_postgresql_host", new OriginalValuePropertyUpdater()); + hiveEnvOriginalValueMap.put("hive_existing_mysql_host", new OriginalValuePropertyUpdater()); + + removePropertyUpdaters.put("hive-env", hiveEnvOriginalValueMap); + + } + + private boolean containsHostFromHostGroups(String configType, String propertyName) { + String propertyValue = clusterTopology.getConfiguration().getPropertyValue(configType, propertyName); + if (StringUtils.isEmpty(propertyValue)) { + return false; + } + // check fir bp import + Matcher m = HostGroup.HOSTGROUP_REGEX.matcher(propertyValue); + if (m.find()) { + return true; + } + + // check for bp export + for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) { + Collection<String> hosts = groupInfo.getHostNames(); + for (String host : hosts) { + if (propertyValue.contains(host)) { + return true; + } + } + } + return false; } public Collection<String> getRequiredHostGroups() { @@ -2274,7 +2349,6 @@ public class BlueprintConfigurationProcessor { Map<String, PropertyUpdater> hiveInteractiveSiteMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> oozieEnvMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> oozieEnvHeapSizeMap = new HashMap<String, PropertyUpdater>(); - Map<String, PropertyUpdater> oozieEnvOriginalValueMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> multiWebhcatSiteMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> multiHbaseSiteMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> multiStormSiteMap = new HashMap<String, PropertyUpdater>(); @@ -2309,7 +2383,6 @@ public class BlueprintConfigurationProcessor { singleHostTopologyUpdaters.put("yarn-site", yarnSiteMap); singleHostTopologyUpdaters.put("hive-site", hiveSiteMap); singleHostTopologyUpdaters.put("hive-interactive-env", hiveInteractiveEnvMap); - singleHostTopologyUpdaters.put("oozie-site", oozieSiteMap); singleHostTopologyUpdaters.put("storm-site", stormSiteMap); singleHostTopologyUpdaters.put("accumulo-site", accumuloSiteMap); singleHostTopologyUpdaters.put("falcon-startup.properties", falconStartupPropertiesMap); @@ -2354,10 +2427,6 @@ public class BlueprintConfigurationProcessor { dbHostTopologyUpdaters.put("hive-site", dbHiveSiteMap); - removePropertyUpdaters.put("oozie-env", oozieEnvOriginalValueMap); - removePropertyUpdaters.put("oozie-site", oozieSiteOriginalValueMap); - removePropertyUpdaters.put("hive-env", hiveEnvOriginalValueMap); - nonTopologyUpdaters.put("hive-site", hiveSiteNonTopologyMap); nonTopologyUpdaters.put("kafka-broker", kafkaBrokerNonTopologyMap); nonTopologyUpdaters.put("storm-site", stormSiteNonTopologyMap); @@ -2535,25 +2604,17 @@ public class BlueprintConfigurationProcessor { // OOZIE_SERVER - oozieSiteMap.put("oozie.base.url", new SingleHostTopologyUpdater("OOZIE_SERVER")); - oozieSiteMap.put("oozie.authentication.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); - oozieSiteMap.put("oozie.ha.authentication.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); - oozieSiteMap.put("oozie.service.HadoopAccessorService.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); - multiCoreSiteMap.put("hadoop.proxyuser.oozie.hosts", new MultipleHostTopologyUpdater("OOZIE_SERVER")); + Map<String, PropertyUpdater> oozieStringPropertyUpdaterMap = singleHostTopologyUpdaters.get("oozie-site"); + if (oozieStringPropertyUpdaterMap == null) { + oozieStringPropertyUpdaterMap = new HashMap<>(); + } + oozieStringPropertyUpdaterMap.put("oozie.base.url", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieStringPropertyUpdaterMap.put("oozie.authentication.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieStringPropertyUpdaterMap.put("oozie.ha.authentication.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); + oozieStringPropertyUpdaterMap.put("oozie.service.HadoopAccessorService.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); + singleHostTopologyUpdaters.put("oozie-site", oozieStringPropertyUpdaterMap); - // register updaters for Oozie properties that may point to an external DB - oozieEnvOriginalValueMap.put("oozie_existing_mysql_host", new OriginalValuePropertyUpdater()); - oozieEnvOriginalValueMap.put("oozie_existing_oracle_host", new OriginalValuePropertyUpdater()); - oozieEnvOriginalValueMap.put("oozie_existing_postgresql_host", new OriginalValuePropertyUpdater()); - oozieEnvOriginalValueMap.put("oozie_existing_oracle_host", new OriginalValuePropertyUpdater()); - oozieEnvOriginalValueMap.put("oozie_existing_postgresql_host", new OriginalValuePropertyUpdater()); - oozieSiteOriginalValueMap.put("oozie.service.JPAService.jdbc.url", new OriginalValuePropertyUpdater()); - // register updaters for Hive properties that may point to an external DB - hiveEnvOriginalValueMap.put("hive_existing_oracle_host", new OriginalValuePropertyUpdater()); - hiveEnvOriginalValueMap.put("hive_existing_mssql_server_2_host", new OriginalValuePropertyUpdater()); - hiveEnvOriginalValueMap.put("hive_existing_mssql_server_host", new OriginalValuePropertyUpdater()); - hiveEnvOriginalValueMap.put("hive_existing_postgresql_host", new OriginalValuePropertyUpdater()); - hiveEnvOriginalValueMap.put("hive_existing_mysql_host", new OriginalValuePropertyUpdater()); + multiCoreSiteMap.put("hadoop.proxyuser.oozie.hosts", new MultipleHostTopologyUpdater("OOZIE_SERVER")); // ZOOKEEPER_SERVER multiHbaseSiteMap.put("hbase.zookeeper.quorum", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER")); http://git-wip-us.apache.org/repos/asf/ambari/blob/58afcf93/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 8a56da3..b49abe9 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -1700,6 +1700,10 @@ public class BlueprintConfigurationProcessorTest { // call top-level export method configProcessor.doUpdateForBlueprintExport(); + // check that jdbc url and related properties are removed if oozie external db is on host which not included to cluster + assertFalse(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url")); + assertTrue(configProcessor.getRemovePropertyUpdaters().get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url")); + assertEquals("oozie property not exported correctly", createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.base.url")); assertEquals("oozie property not exported correctly", @@ -1726,6 +1730,84 @@ public class BlueprintConfigurationProcessorTest { } @Test + public void testOozieJDBCPropertiesNotRemoved() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedHostNameTwo = "c6402.ambari.apache.org"; + final String expectedHostGroupName = "host_group_1"; + final String expectedHostGroupNameTwo = "host_group_2"; + final String expectedPortNum = "80000"; + + Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); + Map<String, String> oozieSiteProperties = new HashMap<String, String>(); + + configProperties.put("oozie-site", oozieSiteProperties); + + oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql://" + expectedHostNameTwo + "/ooziedb"); + + Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); + Collection<String> hgComponents = new HashSet<String>(); + hgComponents.add("OOZIE_SERVER"); + hgComponents.add("ZOOKEEPER_SERVER"); + TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); + + Collection<String> hgComponents2 = new HashSet<String>(); + hgComponents2.add("OOZIE_SERVER"); + hgComponents2.add("ZOOKEEPER_SERVER"); + TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo)); + + Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); + hostGroups.add(group1); + hostGroups.add(group2); + + expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); + + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + BlueprintConfigurationProcessor blueprintConfigurationProcessor = new BlueprintConfigurationProcessor(topology); + + assertTrue(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url")); + assertNull(blueprintConfigurationProcessor.getRemovePropertyUpdaters().get("oozie-site")); + } + + @Test + public void testOozieJDBCPropertyAddedToSingleHostMapDuringImport() throws Exception { + final String expectedHostName = "c6401.apache.ambari.org"; + final String expectedHostNameTwo = "c6402.ambari.apache.org"; + final String expectedHostGroupName = "host_group_1"; + final String expectedHostGroupNameTwo = "host_group_2"; + final String expectedPortNum = "80000"; + + Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>(); + Map<String, String> oozieSiteProperties = new HashMap<String, String>(); + + configProperties.put("oozie-site", oozieSiteProperties); + + oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql://" + "%HOSTGROUP::group1%" + "/ooziedb"); + + Configuration clusterConfig = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); + Collection<String> hgComponents = new HashSet<String>(); + hgComponents.add("OOZIE_SERVER"); + hgComponents.add("ZOOKEEPER_SERVER"); + TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, Collections.singleton(expectedHostName)); + + Collection<String> hgComponents2 = new HashSet<String>(); + hgComponents2.add("OOZIE_SERVER"); + hgComponents2.add("ZOOKEEPER_SERVER"); + TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, hgComponents2, Collections.singleton(expectedHostNameTwo)); + + Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); + hostGroups.add(group1); + hostGroups.add(group2); + + expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); + + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + BlueprintConfigurationProcessor blueprintConfigurationProcessor = new BlueprintConfigurationProcessor(topology); + + assertTrue(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url")); + assertNull(blueprintConfigurationProcessor.getRemovePropertyUpdaters().get("oozie-site")); + } + + @Test public void testZookeeperConfigExported() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.ambari.apache.org";