Repository: ambari Updated Branches: refs/heads/trunk 2e3082af1 -> 20f112acd
Remove Oozie external DB properties from Blueprint Export This patch resolves bug AMBARI-7507. Some Oozie properties related to using an external MySQL Database (oozie_existing_mysql_host, oozie.service.JPAService.jdbc.url) are currently appearing in an exported Blueprint with hostname information included. This is incorrect, since the hostname information should be completely removed or masked in an exported Blueprint. Having a reference to a given hostname in this scenario is not very useful, since the Blueprint may be applied to a completely different set of machines. These Oozie properties generally will only be set to either a default, in-memory DB, or to an external MYSQL server instance. This means that the normal PropertyUpdater processing won't really apply to these two properties. This patch fixes this problem by creating a new PropertyUpdater type, which is only responsible for tracking these two property types, and marking them as properties that will not be included in an exported Blueprint. This patch also adds a processing method to handle any properties that use this new PropertyUpdater, and this processing method will remove these registered properties during the export process. This patch also adds a unit test to verify this change. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/20f112ac Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/20f112ac Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/20f112ac Branch: refs/heads/trunk Commit: 20f112acd619f58aaad85a5fa65ff82c2e4cc498 Parents: 2e3082a Author: Bob Nettleton <rnettle...@hortonworks.com> Authored: Thu Sep 25 21:37:42 2014 -0400 Committer: John Speidel <jspei...@hortonworks.com> Committed: Fri Sep 26 11:42:07 2014 -0400 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 60 ++++++++++++++++++++ .../BlueprintConfigurationProcessorTest.java | 9 +++ 2 files changed, 69 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/20f112ac/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 3e1bcf0..d33ae65 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -61,6 +61,15 @@ public class BlueprintConfigurationProcessor { new HashMap<String, Map<String, PropertyUpdater>>(); /** + * Updaters that preserve the original property value, functions + * as a placeholder for DB-related properties that need to be + * removed from export, but do not require an update during + * cluster creation + */ + private static Map<String, Map<String, PropertyUpdater>> removePropertyUpdaters = + new HashMap<String, Map<String, PropertyUpdater>>(); + + /** * Collection of all updaters */ private static Collection<Map<String, Map<String, PropertyUpdater>>> allUpdaters = @@ -181,10 +190,37 @@ public class BlueprintConfigurationProcessor { doMultiHostExportUpdate(hostGroups, multiHostTopologyUpdaters); + doRemovePropertyExport(removePropertyUpdaters); + return properties; } /** + * Performs export update for the set of properties that do not + * require update during cluster setup, but should be removed + * during a Blueprint export. + * + * In the case of a service referring to an external DB, any + * properties that contain external host information should + * be removed from the configuration that will be available in + * the exported Blueprint. + * + * @param updaters set of updaters for properties that should + * always be removed during a Blueprint export + */ + private void doRemovePropertyExport(Map<String, Map<String, PropertyUpdater>> updaters) { + for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) { + String type = entry.getKey(); + for (String propertyName : entry.getValue().keySet()) { + Map<String, String> typeProperties = properties.get(type); + if ( (typeProperties != null) && (typeProperties.containsKey(propertyName)) ) { + typeProperties.remove(propertyName); + } + } + } + } + + /** * Perform export update processing for HA configuration for NameNodes. The HA NameNode property * names are based on the nameservices defined when HA is enabled via the Ambari UI, so this method * dynamically determines the property names, and registers PropertyUpdaters to handle the masking of @@ -768,6 +804,21 @@ public class BlueprintConfigurationProcessor { } /** + * PropertyUpdater implementation that will always return the original + * value for the updateForClusterCreate() method. + * This updater type should only be used in cases where a given + * property requires no updates, but may need to be considered + * during the Blueprint export process. + */ + private static class OriginalValuePropertyUpdater implements PropertyUpdater { + @Override + public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, String origValue, Map<String, Map<String, String>> properties) { + // always return the original value, since these properties do not require update handling + return origValue; + } + } + + /** * Register updaters for configuration properties. */ static { @@ -783,6 +834,7 @@ public class BlueprintConfigurationProcessor { Map<String, PropertyUpdater> hbaseSiteMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> yarnSiteMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> hiveSiteMap = new HashMap<String, PropertyUpdater>(); + Map<String, PropertyUpdater> oozieSiteOriginalValueMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> oozieSiteMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> stormSiteMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> falconStartupPropertiesMap = new HashMap<String, PropertyUpdater>(); @@ -793,6 +845,8 @@ public class BlueprintConfigurationProcessor { Map<String, PropertyUpdater> hbaseEnvMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> hiveEnvMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> oozieEnvMap = new HashMap<String, PropertyUpdater>(); + Map<String, PropertyUpdater> oozieEnvOriginalValueMap = new HashMap<String, PropertyUpdater>(); + Map<String, PropertyUpdater> multiWebhcatSiteMap = new HashMap<String, PropertyUpdater>(); Map<String, PropertyUpdater> multiHbaseSiteMap = new HashMap<String, PropertyUpdater>(); @@ -830,6 +884,9 @@ public class BlueprintConfigurationProcessor { dbHostTopologyUpdaters.put("hive-site", dbHiveSiteMap); + removePropertyUpdaters.put("oozie-env", oozieEnvOriginalValueMap); + removePropertyUpdaters.put("oozie-site", oozieSiteOriginalValueMap); + // NAMENODE hdfsSiteMap.put("dfs.http.address", new SingleHostTopologyUpdater("NAMENODE")); hdfsSiteMap.put("dfs.https.address", new SingleHostTopologyUpdater("NAMENODE")); @@ -886,6 +943,9 @@ public class BlueprintConfigurationProcessor { oozieSiteMap.put("oozie.service.HadoopAccessorService.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); oozieEnvMap.put("oozie_hostname", new SingleHostTopologyUpdater("OOZIE_SERVER")); multiCoreSiteMap.put("hadoop.proxyuser.oozie.hosts", new MultipleHostTopologyUpdater("OOZIE_SERVER")); + // register updaters for Oozie properties that may point to an external DB + oozieEnvOriginalValueMap.put("oozie_existing_mysql_host", new OriginalValuePropertyUpdater()); + oozieSiteOriginalValueMap.put("oozie.service.JPAService.jdbc.url", new OriginalValuePropertyUpdater()); // ZOOKEEPER_SERVER multiHbaseSiteMap.put("hbase.zookeeper.quorum", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER")); http://git-wip-us.apache.org/repos/asf/ambari/blob/20f112ac/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 590b9c2..90f83bc 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -1685,6 +1685,7 @@ public class BlueprintConfigurationProcessorTest { public void testOozieConfigExported() throws Exception { final String expectedHostName = "c6401.apache.ambari.org"; final String expectedHostNameTwo = "c6402.ambari.apache.org"; + final String expectedExternalHost = "c6408.ambari.apache.org"; final String expectedHostGroupName = "host_group_1"; final String expectedHostGroupNameTwo = "host_group_2"; @@ -1718,8 +1719,10 @@ public class BlueprintConfigurationProcessorTest { oozieSiteProperties.put("oozie.base.url", expectedHostName); oozieSiteProperties.put("oozie.authentication.kerberos.principal", expectedHostName); oozieSiteProperties.put("oozie.service.HadoopAccessorService.kerberos.principal", expectedHostName); + oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql://" + expectedExternalHost + "/ooziedb"); oozieEnvProperties.put("oozie_hostname", expectedHostName); + oozieEnvProperties.put("oozie_existing_mysql_host", expectedExternalHost); coreSiteProperties.put("hadoop.proxyuser.oozie.hosts", expectedHostName + "," + expectedHostNameTwo); @@ -1740,6 +1743,12 @@ public class BlueprintConfigurationProcessorTest { assertEquals("oozie property not exported correctly", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.oozie.hosts")); + // verify that the oozie properties that can refer to an external DB are not included in the export + assertFalse("oozie_existing_mysql_host should not have been present in the exported configuration", + oozieEnvProperties.containsKey("oozie_existing_mysql_host")); + assertFalse("oozie.service.JPAService.jdbc.url should not have been present in the exported configuration", + oozieSiteProperties.containsKey("oozie.service.JPAService.jdbc.url")); + mockSupport.verifyAll(); }