Repository: ambari
Updated Branches:
  refs/heads/branch-feature-AMBARI-18456 5cf5c8349 -> ed2018bf0


AMBARI-18546. BP format does not generate property 
oozie.service.JPAService.jdbc.url.(vbrodetskyi)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/58afcf93
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/58afcf93
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/58afcf93

Branch: refs/heads/branch-feature-AMBARI-18456
Commit: 58afcf93d8445c3b1ffacdd33c1cb12a8e8414cb
Parents: 72f1f6f
Author: Vitaly Brodetskyi <vbrodets...@hortonworks.com>
Authored: Mon Oct 10 00:22:49 2016 +0300
Committer: Vitaly Brodetskyi <vbrodets...@hortonworks.com>
Committed: Mon Oct 10 00:22:49 2016 +0300

----------------------------------------------------------------------
 .../BlueprintConfigurationProcessor.java        | 113 ++++++++++++++-----
 .../BlueprintConfigurationProcessorTest.java    |  82 ++++++++++++++
 2 files changed, 169 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/58afcf93/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index ba09368..8ff568a 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -84,7 +84,7 @@ public class BlueprintConfigurationProcessor {
   /**
    * Single host topology updaters
    */
-  private static Map<String, Map<String, PropertyUpdater>> 
singleHostTopologyUpdaters =
+  protected static Map<String, Map<String, PropertyUpdater>> 
singleHostTopologyUpdaters =
       new HashMap<String, Map<String, PropertyUpdater>>();
 
   /**
@@ -117,7 +117,7 @@ public class BlueprintConfigurationProcessor {
    * removed from export, but do not require an update during
    * cluster creation
    */
-  private static Map<String, Map<String, PropertyUpdater>> 
removePropertyUpdaters =
+  private Map<String, Map<String, PropertyUpdater>> removePropertyUpdaters =
     new HashMap<String, Map<String, PropertyUpdater>>();
 
   /**
@@ -196,6 +196,81 @@ public class BlueprintConfigurationProcessor {
 
   public BlueprintConfigurationProcessor(ClusterTopology clusterTopology) {
     this.clusterTopology = clusterTopology;
+    initRemovePropertyUpdaters();
+  }
+
+  public Map<String, Map<String, PropertyUpdater>> getRemovePropertyUpdaters() 
{
+    return removePropertyUpdaters;
+  }
+
+  public void initRemovePropertyUpdaters() {
+
+    if (containsHostFromHostGroups("oozie-site", 
"oozie.service.JPAService.jdbc.url")) {
+      Map<String, PropertyUpdater> oozieSiteUpdaters = 
singleHostTopologyUpdaters.get("oozie-site");
+      Map<String, PropertyUpdater> oozieEnvUpdaters = 
singleHostTopologyUpdaters.get("oozie-env");
+      if (oozieSiteUpdaters == null) {
+        oozieSiteUpdaters = new HashMap<>();
+      }
+      if (oozieEnvUpdaters == null) {
+        oozieEnvUpdaters = new HashMap<>();
+      }
+      oozieEnvUpdaters.put("oozie_existing_mysql_host", new 
SingleHostTopologyUpdater("OOZIE_SERVER"));
+      oozieEnvUpdaters.put("oozie_existing_oracle_host", new 
SingleHostTopologyUpdater("OOZIE_SERVER"));
+      oozieEnvUpdaters.put("oozie_existing_postgresql_host", new 
SingleHostTopologyUpdater("OOZIE_SERVER"));
+      oozieEnvUpdaters.put("oozie_existing_oracle_host", new 
SingleHostTopologyUpdater("OOZIE_SERVER"));
+      oozieEnvUpdaters.put("oozie_existing_postgresql_host", new 
SingleHostTopologyUpdater("OOZIE_SERVER"));
+      oozieSiteUpdaters.put("oozie.service.JPAService.jdbc.url",  new 
SingleHostTopologyUpdater("OOZIE_SERVER"));
+
+      singleHostTopologyUpdaters.put("oozie-env", oozieEnvUpdaters);
+      singleHostTopologyUpdaters.put("oozie-site", oozieSiteUpdaters);
+    } else {
+      Map<String, PropertyUpdater> oozieEnvOriginalValueMap = new 
HashMap<String, PropertyUpdater>();
+      Map<String, PropertyUpdater> oozieSiteOriginalValueMap = new 
HashMap<String, PropertyUpdater>();
+      // register updaters for Oozie properties that may point to an external 
DB
+      oozieEnvOriginalValueMap.put("oozie_existing_mysql_host", new 
OriginalValuePropertyUpdater());
+      oozieEnvOriginalValueMap.put("oozie_existing_oracle_host", new 
OriginalValuePropertyUpdater());
+      oozieEnvOriginalValueMap.put("oozie_existing_postgresql_host", new 
OriginalValuePropertyUpdater());
+      oozieEnvOriginalValueMap.put("oozie_existing_oracle_host", new 
OriginalValuePropertyUpdater());
+      oozieEnvOriginalValueMap.put("oozie_existing_postgresql_host", new 
OriginalValuePropertyUpdater());
+      oozieSiteOriginalValueMap.put("oozie.service.JPAService.jdbc.url", new 
OriginalValuePropertyUpdater());
+
+      removePropertyUpdaters.put("oozie-env", oozieEnvOriginalValueMap);
+      removePropertyUpdaters.put("oozie-site", oozieSiteOriginalValueMap);
+    }
+
+    Map<String, PropertyUpdater> hiveEnvOriginalValueMap = new HashMap<String, 
PropertyUpdater>();
+    // register updaters for Hive properties that may point to an external DB
+    hiveEnvOriginalValueMap.put("hive_existing_oracle_host", new 
OriginalValuePropertyUpdater());
+    hiveEnvOriginalValueMap.put("hive_existing_mssql_server_2_host", new 
OriginalValuePropertyUpdater());
+    hiveEnvOriginalValueMap.put("hive_existing_mssql_server_host", new 
OriginalValuePropertyUpdater());
+    hiveEnvOriginalValueMap.put("hive_existing_postgresql_host", new 
OriginalValuePropertyUpdater());
+    hiveEnvOriginalValueMap.put("hive_existing_mysql_host", new 
OriginalValuePropertyUpdater());
+
+    removePropertyUpdaters.put("hive-env", hiveEnvOriginalValueMap);
+
+  }
+
+  private boolean containsHostFromHostGroups(String configType, String 
propertyName) {
+    String propertyValue = 
clusterTopology.getConfiguration().getPropertyValue(configType, propertyName);
+    if (StringUtils.isEmpty(propertyValue)) {
+      return false;
+    }
+    // check fir bp import
+    Matcher m = HostGroup.HOSTGROUP_REGEX.matcher(propertyValue);
+    if (m.find()) {
+      return true;
+    }
+
+    // check for bp export
+    for (HostGroupInfo groupInfo : 
clusterTopology.getHostGroupInfo().values()) {
+      Collection<String> hosts = groupInfo.getHostNames();
+      for (String host : hosts) {
+        if (propertyValue.contains(host)) {
+          return true;
+        }
+      }
+    }
+    return false;
   }
 
   public Collection<String> getRequiredHostGroups() {
@@ -2274,7 +2349,6 @@ public class BlueprintConfigurationProcessor {
     Map<String, PropertyUpdater> hiveInteractiveSiteMap = new HashMap<String, 
PropertyUpdater>();
     Map<String, PropertyUpdater> oozieEnvMap = new HashMap<String, 
PropertyUpdater>();
     Map<String, PropertyUpdater> oozieEnvHeapSizeMap = new HashMap<String, 
PropertyUpdater>();
-    Map<String, PropertyUpdater> oozieEnvOriginalValueMap = new 
HashMap<String, PropertyUpdater>();
     Map<String, PropertyUpdater> multiWebhcatSiteMap = new HashMap<String, 
PropertyUpdater>();
     Map<String, PropertyUpdater> multiHbaseSiteMap = new HashMap<String, 
PropertyUpdater>();
     Map<String, PropertyUpdater> multiStormSiteMap = new HashMap<String, 
PropertyUpdater>();
@@ -2309,7 +2383,6 @@ public class BlueprintConfigurationProcessor {
     singleHostTopologyUpdaters.put("yarn-site", yarnSiteMap);
     singleHostTopologyUpdaters.put("hive-site", hiveSiteMap);
     singleHostTopologyUpdaters.put("hive-interactive-env", 
hiveInteractiveEnvMap);
-    singleHostTopologyUpdaters.put("oozie-site", oozieSiteMap);
     singleHostTopologyUpdaters.put("storm-site", stormSiteMap);
     singleHostTopologyUpdaters.put("accumulo-site", accumuloSiteMap);
     singleHostTopologyUpdaters.put("falcon-startup.properties", 
falconStartupPropertiesMap);
@@ -2354,10 +2427,6 @@ public class BlueprintConfigurationProcessor {
 
     dbHostTopologyUpdaters.put("hive-site", dbHiveSiteMap);
 
-    removePropertyUpdaters.put("oozie-env", oozieEnvOriginalValueMap);
-    removePropertyUpdaters.put("oozie-site", oozieSiteOriginalValueMap);
-    removePropertyUpdaters.put("hive-env", hiveEnvOriginalValueMap);
-
     nonTopologyUpdaters.put("hive-site", hiveSiteNonTopologyMap);
     nonTopologyUpdaters.put("kafka-broker", kafkaBrokerNonTopologyMap);
     nonTopologyUpdaters.put("storm-site", stormSiteNonTopologyMap);
@@ -2535,25 +2604,17 @@ public class BlueprintConfigurationProcessor {
 
 
     // OOZIE_SERVER
-    oozieSiteMap.put("oozie.base.url", new 
SingleHostTopologyUpdater("OOZIE_SERVER"));
-    oozieSiteMap.put("oozie.authentication.kerberos.principal", new 
SingleHostTopologyUpdater("OOZIE_SERVER"));
-    oozieSiteMap.put("oozie.ha.authentication.kerberos.principal", new 
SingleHostTopologyUpdater("OOZIE_SERVER"));
-    oozieSiteMap.put("oozie.service.HadoopAccessorService.kerberos.principal", 
new SingleHostTopologyUpdater("OOZIE_SERVER"));
-    multiCoreSiteMap.put("hadoop.proxyuser.oozie.hosts", new 
MultipleHostTopologyUpdater("OOZIE_SERVER"));
+    Map<String, PropertyUpdater> oozieStringPropertyUpdaterMap = 
singleHostTopologyUpdaters.get("oozie-site");
+    if (oozieStringPropertyUpdaterMap == null) {
+      oozieStringPropertyUpdaterMap = new HashMap<>();
+    }
+    oozieStringPropertyUpdaterMap.put("oozie.base.url", new 
SingleHostTopologyUpdater("OOZIE_SERVER"));
+    
oozieStringPropertyUpdaterMap.put("oozie.authentication.kerberos.principal", 
new SingleHostTopologyUpdater("OOZIE_SERVER"));
+    
oozieStringPropertyUpdaterMap.put("oozie.ha.authentication.kerberos.principal", 
new SingleHostTopologyUpdater("OOZIE_SERVER"));
+    
oozieStringPropertyUpdaterMap.put("oozie.service.HadoopAccessorService.kerberos.principal",
 new SingleHostTopologyUpdater("OOZIE_SERVER"));
+    singleHostTopologyUpdaters.put("oozie-site", 
oozieStringPropertyUpdaterMap);
 
-    // register updaters for Oozie properties that may point to an external DB
-    oozieEnvOriginalValueMap.put("oozie_existing_mysql_host", new 
OriginalValuePropertyUpdater());
-    oozieEnvOriginalValueMap.put("oozie_existing_oracle_host", new 
OriginalValuePropertyUpdater());
-    oozieEnvOriginalValueMap.put("oozie_existing_postgresql_host", new 
OriginalValuePropertyUpdater());
-    oozieEnvOriginalValueMap.put("oozie_existing_oracle_host", new 
OriginalValuePropertyUpdater());
-    oozieEnvOriginalValueMap.put("oozie_existing_postgresql_host", new 
OriginalValuePropertyUpdater());
-    oozieSiteOriginalValueMap.put("oozie.service.JPAService.jdbc.url", new 
OriginalValuePropertyUpdater());
-    // register updaters for Hive properties that may point to an external DB
-    hiveEnvOriginalValueMap.put("hive_existing_oracle_host", new 
OriginalValuePropertyUpdater());
-    hiveEnvOriginalValueMap.put("hive_existing_mssql_server_2_host", new 
OriginalValuePropertyUpdater());
-    hiveEnvOriginalValueMap.put("hive_existing_mssql_server_host", new 
OriginalValuePropertyUpdater());
-    hiveEnvOriginalValueMap.put("hive_existing_postgresql_host", new 
OriginalValuePropertyUpdater());
-    hiveEnvOriginalValueMap.put("hive_existing_mysql_host", new 
OriginalValuePropertyUpdater());
+    multiCoreSiteMap.put("hadoop.proxyuser.oozie.hosts", new 
MultipleHostTopologyUpdater("OOZIE_SERVER"));
 
     // ZOOKEEPER_SERVER
     multiHbaseSiteMap.put("hbase.zookeeper.quorum", new 
MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));

http://git-wip-us.apache.org/repos/asf/ambari/blob/58afcf93/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 8a56da3..b49abe9 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -1700,6 +1700,10 @@ public class BlueprintConfigurationProcessorTest {
     // call top-level export method
     configProcessor.doUpdateForBlueprintExport();
 
+    // check that jdbc url and related properties are removed if oozie 
external db is on host which not included to cluster
+    
assertFalse(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url"));
+    
assertTrue(configProcessor.getRemovePropertyUpdaters().get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url"));
+
     assertEquals("oozie property not exported correctly",
         createExportedHostName(expectedHostGroupName), 
oozieSiteProperties.get("oozie.base.url"));
     assertEquals("oozie property not exported correctly",
@@ -1726,6 +1730,84 @@ public class BlueprintConfigurationProcessorTest {
   }
 
   @Test
+  public void testOozieJDBCPropertiesNotRemoved() throws Exception {
+    final String expectedHostName = "c6401.apache.ambari.org";
+    final String expectedHostNameTwo = "c6402.ambari.apache.org";
+    final String expectedHostGroupName = "host_group_1";
+    final String expectedHostGroupNameTwo = "host_group_2";
+    final String expectedPortNum = "80000";
+
+    Map<String, Map<String, String>> configProperties = new HashMap<String, 
Map<String, String>>();
+    Map<String, String> oozieSiteProperties = new HashMap<String, String>();
+
+    configProperties.put("oozie-site", oozieSiteProperties);
+
+    oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", 
"jdbc:mysql://" + expectedHostNameTwo + "/ooziedb");
+
+    Configuration clusterConfig = new Configuration(configProperties, 
Collections.<String, Map<String, Map<String, String>>>emptyMap());
+    Collection<String> hgComponents = new HashSet<String>();
+    hgComponents.add("OOZIE_SERVER");
+    hgComponents.add("ZOOKEEPER_SERVER");
+    TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, 
hgComponents, Collections.singleton(expectedHostName));
+
+    Collection<String> hgComponents2 = new HashSet<String>();
+    hgComponents2.add("OOZIE_SERVER");
+    hgComponents2.add("ZOOKEEPER_SERVER");
+    TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, 
hgComponents2, Collections.singleton(expectedHostNameTwo));
+
+    Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+    hostGroups.add(group1);
+    hostGroups.add(group2);
+
+    expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new 
Cardinality("1+")).anyTimes();
+
+    ClusterTopology topology = createClusterTopology(bp, clusterConfig, 
hostGroups);
+    BlueprintConfigurationProcessor blueprintConfigurationProcessor = new 
BlueprintConfigurationProcessor(topology);
+
+    
assertTrue(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url"));
+    
assertNull(blueprintConfigurationProcessor.getRemovePropertyUpdaters().get("oozie-site"));
+  }
+
+  @Test
+  public void testOozieJDBCPropertyAddedToSingleHostMapDuringImport() throws 
Exception {
+    final String expectedHostName = "c6401.apache.ambari.org";
+    final String expectedHostNameTwo = "c6402.ambari.apache.org";
+    final String expectedHostGroupName = "host_group_1";
+    final String expectedHostGroupNameTwo = "host_group_2";
+    final String expectedPortNum = "80000";
+
+    Map<String, Map<String, String>> configProperties = new HashMap<String, 
Map<String, String>>();
+    Map<String, String> oozieSiteProperties = new HashMap<String, String>();
+
+    configProperties.put("oozie-site", oozieSiteProperties);
+
+    oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", 
"jdbc:mysql://" + "%HOSTGROUP::group1%" + "/ooziedb");
+
+    Configuration clusterConfig = new Configuration(configProperties, 
Collections.<String, Map<String, Map<String, String>>>emptyMap());
+    Collection<String> hgComponents = new HashSet<String>();
+    hgComponents.add("OOZIE_SERVER");
+    hgComponents.add("ZOOKEEPER_SERVER");
+    TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, 
hgComponents, Collections.singleton(expectedHostName));
+
+    Collection<String> hgComponents2 = new HashSet<String>();
+    hgComponents2.add("OOZIE_SERVER");
+    hgComponents2.add("ZOOKEEPER_SERVER");
+    TestHostGroup group2 = new TestHostGroup(expectedHostGroupNameTwo, 
hgComponents2, Collections.singleton(expectedHostNameTwo));
+
+    Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+    hostGroups.add(group1);
+    hostGroups.add(group2);
+
+    expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new 
Cardinality("1+")).anyTimes();
+
+    ClusterTopology topology = createClusterTopology(bp, clusterConfig, 
hostGroups);
+    BlueprintConfigurationProcessor blueprintConfigurationProcessor = new 
BlueprintConfigurationProcessor(topology);
+
+    
assertTrue(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url"));
+    
assertNull(blueprintConfigurationProcessor.getRemovePropertyUpdaters().get("oozie-site"));
+  }
+
+  @Test
   public void testZookeeperConfigExported() throws Exception {
     final String expectedHostName = "c6401.apache.ambari.org";
     final String expectedHostNameTwo = "c6402.ambari.apache.org";

Reply via email to