Repository: ambari
Updated Branches:
refs/heads/trunk 087d9003e -> b20dbb7fa
AMBARI-11441. Provide better error message from
BlueprintConfigurationProcessor when unable
to match a property component token to a host group
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b20dbb7f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b20dbb7f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b20dbb7f
Branch: refs/heads/trunk
Commit: b20dbb7fae7fc029f353f4eddf280c5f53878421
Parents: 087d900
Author: John Speidel <jspei...@hortonworks.com>
Authored: Wed May 27 15:37:15 2015 -0400
Committer: John Speidel <jspei...@hortonworks.com>
Committed: Thu May 28 10:22:42 2015 -0400
----------------------------------------------------------------------
.../BlueprintConfigurationProcessor.java | 142 +++++++------------
.../BlueprintConfigurationProcessorTest.java | 74 +++++++++-
2 files changed, 123 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b20dbb7f/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index a84bf3d..6fb4807 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -139,7 +139,7 @@ public class BlueprintConfigurationProcessor {
Map<String, String> typeMap = clusterProps.get(type);
if (typeMap != null && typeMap.containsKey(propertyName)) {
requiredHostGroups.addAll(updater.getRequiredHostGroups(
- typeMap.get(propertyName), clusterProps, clusterTopology));
+ propertyName, typeMap.get(propertyName), clusterProps,
clusterTopology));
}
// host group configs
@@ -148,7 +148,7 @@ public class BlueprintConfigurationProcessor {
Map<String, String> hgTypeMap = hgConfigProps.get(type);
if (hgTypeMap != null && hgTypeMap.containsKey(propertyName)) {
requiredHostGroups.addAll(updater.getRequiredHostGroups(
- hgTypeMap.get(propertyName), hgConfigProps,
clusterTopology));
+ propertyName, hgTypeMap.get(propertyName), hgConfigProps,
clusterTopology));
}
}
}
@@ -857,7 +857,7 @@ public class BlueprintConfigurationProcessor {
/**
* Update a property value.
*
- * @param propertyName name of
+ * @param propertyName property name
* @param origValue original value of property
* @param properties all properties
* @param topology cluster topology
@@ -869,9 +869,20 @@ public class BlueprintConfigurationProcessor {
Map<String, Map<String, String>>
properties,
ClusterTopology topology);
- Collection<String> getRequiredHostGroups(String origValue,
- Map<String, Map<String,
String>> properties,
- ClusterTopology topology);
+ /**
+ * Determine the required host groups for the provided property.
+ *
+ * @param propertyName property name
+ * @param origValue original value of property
+ * @param properties all properties
+ * @param topology cluster topology
+ *
+ * @return new property value
+ */
+ Collection<String> getRequiredHostGroups(String propertyName,
+ String origValue,
+ Map<String, Map<String, String>>
properties,
+ ClusterTopology topology);
}
/**
@@ -1001,16 +1012,17 @@ public class BlueprintConfigurationProcessor {
}
}
-
- throw new IllegalArgumentException("Unable to update configuration
property " + "'" + propertyName + "'"+ " with topology information. " +
- "Component '" + component + "' is not mapped to any host group
or is mapped to multiple groups.");
+ throw new IllegalArgumentException(
+ String.format("Unable to update configuration property '%s'
with topology information. " +
+ "Component '%s' is mapped to an invalid number of hosts
'%s'.", propertyName, component, matchingGroupCount));
}
}
}
}
@Override
- public Collection<String> getRequiredHostGroups(String origValue,
+ public Collection<String> getRequiredHostGroups(String propertyName,
+ String origValue,
Map<String, Map<String,
String>> properties,
ClusterTopology topology) {
//todo: getHostStrings
@@ -1021,78 +1033,17 @@ public class BlueprintConfigurationProcessor {
} else {
Collection<String> matchingGroups =
topology.getHostGroupsForComponent(component);
int matchingGroupCount = matchingGroups.size();
- if (matchingGroupCount == 1) {
- return Collections.singleton(matchingGroups.iterator().next());
+ if (matchingGroupCount != 0) {
+ return new HashSet<String>(matchingGroups);
} else {
Cardinality cardinality =
topology.getBlueprint().getStack().getCardinality(component);
// if no matching host groups are found for a component whose
configuration
// is handled by this updater, return an empty set
- if (matchingGroupCount == 0 && cardinality.isValidCount(0)) {
- return Collections.emptySet();
- } else {
- //todo: shouldn't have all of these hard coded HA rules here
- if (topology.isNameNodeHAEnabled() && isComponentNameNode() &&
(matchingGroupCount == 2)) {
- // if this is the defaultFS property, it should reflect the
nameservice name,
- // rather than a hostname (used in non-HA scenarios)
- if (properties.containsKey("core-site") &&
properties.get("core-site").get("fs.defaultFS").equals(origValue)) {
- return Collections.emptySet();
- }
-
- if (properties.containsKey("hbase-site") &&
properties.get("hbase-site").get("hbase.rootdir").equals(origValue)) {
- // hbase-site's reference to the namenode is handled
differently in HA mode, since the
- // reference must point to the logical nameservice, rather
than an individual namenode
- return Collections.emptySet();
- }
-
- if (properties.containsKey("accumulo-site") &&
properties.get("accumulo-site").get("instance.volumes").equals(origValue)) {
- // accumulo-site's reference to the namenode is handled
differently in HA mode, since the
- // reference must point to the logical nameservice, rather
than an individual namenode
- return Collections.emptySet();
- }
-
- if (!origValue.contains("localhost")) {
- // if this NameNode HA property is a FDQN, then simply return
it
- return Collections.emptySet();
- }
- }
-
- if (topology.isNameNodeHAEnabled() &&
isComponentSecondaryNameNode() && (matchingGroupCount == 0)) {
- // if HDFS HA is enabled, then no replacement is necessary for
properties that refer to the SECONDARY_NAMENODE
- // eventually this type of information should be encoded in the
stacks
- return Collections.emptySet();
- }
-
- if (topology.isYarnResourceManagerHAEnabled() &&
isComponentResourceManager() && (matchingGroupCount == 2)) {
- if (!origValue.contains("localhost")) {
- // if this Yarn property is a FQDN, then simply return it
- return Collections.emptySet();
- }
- }
-
- if ((isOozieServerHAEnabled(properties)) &&
isComponentOozieServer() && (matchingGroupCount > 1)) {
- if (!origValue.contains("localhost")) {
- // if this Oozie property is a FQDN, then simply return it
- return Collections.emptySet();
- }
- }
-
- if ((isHiveServerHAEnabled(properties)) && isComponentHiveServer()
&& (matchingGroupCount > 1)) {
- if (!origValue.contains("localhost")) {
- // if this Hive property is a FQDN, then simply return it
- return Collections.emptySet();
- }
- }
-
- if ((isComponentHiveMetaStoreServer()) && matchingGroupCount > 1) {
- if (!origValue.contains("localhost")) {
- // if this Hive MetaStore property is a FQDN, then simply
return it
- return Collections.emptySet();
- }
- }
- //todo: property name
- throw new IllegalArgumentException("Unable to update configuration
property with topology information. " +
- "Component '" + component + "' is not mapped to any host group
or is mapped to multiple groups.");
+ if (! cardinality.isValidCount(0)) {
+ LOG.warn("The property '{}' is associated with the component '{}'
which isn't mapped to any host group. " +
+ "This may affect configuration topology resolution.",
propertyName, component);
}
+ return Collections.emptySet();
}
}
}
@@ -1205,12 +1156,13 @@ public class BlueprintConfigurationProcessor {
}
@Override
- public Collection<String> getRequiredHostGroups(String origValue,
+ public Collection<String> getRequiredHostGroups(String propertyName,
+ String origValue,
Map<String, Map<String,
String>> properties,
ClusterTopology topology) {
try {
- return super.getRequiredHostGroups(origValue, properties, topology);
+ return super.getRequiredHostGroups(propertyName, origValue,
properties, topology);
} catch (IllegalArgumentException e) {
return Collections.emptySet();
}
@@ -1275,9 +1227,12 @@ public class BlueprintConfigurationProcessor {
}
@Override
- public Collection<String> getRequiredHostGroups(String origValue,
Map<String, Map<String, String>> properties, ClusterTopology topology) {
+ public Collection<String> getRequiredHostGroups(String propertyName,
+ String origValue,
+ Map<String, Map<String,
String>> properties,
+ ClusterTopology topology) {
if (isDatabaseManaged(properties)) {
- return super.getRequiredHostGroups(origValue, properties, topology);
+ return super.getRequiredHostGroups(propertyName, origValue,
properties, topology);
} else {
return Collections.emptySet();
}
@@ -1450,7 +1405,8 @@ public class BlueprintConfigurationProcessor {
}
@Override
- public Collection<String> getRequiredHostGroups(String origValue,
+ public Collection<String> getRequiredHostGroups(String propertyName,
+ String origValue,
Map<String, Map<String,
String>> properties,
ClusterTopology topology) {
@@ -1503,8 +1459,10 @@ public class BlueprintConfigurationProcessor {
}
@Override
- public Collection<String> getRequiredHostGroups(String origValue,
- Map<String, Map<String,
String>> properties, ClusterTopology topology) {
+ public Collection<String> getRequiredHostGroups(String propertyName,
+ String origValue,
+ Map<String, Map<String,
String>> properties,
+ ClusterTopology topology) {
return Collections.emptySet();
}
}
@@ -1558,10 +1516,12 @@ public class BlueprintConfigurationProcessor {
public abstract String doFormat(String originalValue);
@Override
- public Collection<String> getRequiredHostGroups(String origValue,
- Map<String, Map<String,
String>> properties, ClusterTopology topology) {
+ public Collection<String> getRequiredHostGroups(String propertyName,
+ String origValue,
+ Map<String, Map<String,
String>> properties,
+ ClusterTopology topology) {
- return propertyUpdater.getRequiredHostGroups(origValue, properties,
topology);
+ return propertyUpdater.getRequiredHostGroups(propertyName, origValue,
properties, topology);
}
/**
@@ -1656,7 +1616,8 @@ public class BlueprintConfigurationProcessor {
}
@Override
- public Collection<String> getRequiredHostGroups(String origValue,
+ public Collection<String> getRequiredHostGroups(String propertyName,
+ String origValue,
Map<String, Map<String,
String>> properties,
ClusterTopology topology) {
@@ -1728,7 +1689,8 @@ public class BlueprintConfigurationProcessor {
}
@Override
- public Collection<String> getRequiredHostGroups(String origValue,
+ public Collection<String> getRequiredHostGroups(String propertyName,
+ String origValue,
Map<String, Map<String,
String>> properties,
ClusterTopology topology) {
@@ -1747,7 +1709,7 @@ public class BlueprintConfigurationProcessor {
String key = keyValuePair.split("=")[0];
if (mapOfKeysToUpdaters.containsKey(key)) {
requiredGroups.addAll(mapOfKeysToUpdaters.get(key).getRequiredHostGroups(
- keyValuePair.split("=")[1], properties, topology));
+ propertyName, keyValuePair.split("=")[1], properties, topology));
}
}
return requiredGroups;
http://git-wip-us.apache.org/repos/asf/ambari/blob/b20dbb7f/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 595cc4a..1de2b29 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -30,6 +30,7 @@ import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.reset;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -2280,7 +2281,7 @@ public class BlueprintConfigurationProcessorTest {
updater.doUpdateForBlueprintExport();
assertEquals("oozie property not updated correctly",
- createExportedHostName(expectedHostGroupName, expectedPortNum),
oozieSiteProperties.get("oozie.base.url"));
+ createExportedHostName(expectedHostGroupName, expectedPortNum),
oozieSiteProperties.get("oozie.base.url"));
assertEquals("oozie property not updated correctly",
createExportedHostName(expectedHostGroupName),
oozieSiteProperties.get("oozie.authentication.kerberos.principal"));
assertEquals("oozie property not updated correctly",
@@ -3907,7 +3908,7 @@ public class BlueprintConfigurationProcessorTest {
}
@Test
- public void testGetRequiredHostGroups___validComponentCountofZero() throws
Exception {
+ public void testGetRequiredHostGroups___validComponentCountOfZero() throws
Exception {
Map<String, Map<String, String>> properties = new HashMap<String,
Map<String, String>>();
Map<String, String> hiveSite = new HashMap<String, String>();
properties.put("hive-site", hiveSite);
@@ -3940,9 +3941,76 @@ public class BlueprintConfigurationProcessorTest {
// call top-level export method
Collection<String> requiredGroups = updater.getRequiredHostGroups();
- System.out.println("Required Groups: " + requiredGroups);
+ assertEquals(0, requiredGroups.size());
+ }
+
+ @Test
+ public void testGetRequiredHostGroups___invalidComponentCountOfZero() throws
Exception {
+ Map<String, Map<String, String>> properties = new HashMap<String,
Map<String, String>>();
+ Map<String, String> coreSiteMap = new HashMap<String, String>();
+ properties.put("core-site", coreSiteMap);
+
+ coreSiteMap.put("fs.defaultFS", "localhost");
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ expect(stack.getCardinality("NAMENODE")).andReturn(new
Cardinality("1")).anyTimes();
+
+ Collection<String> hgComponents1 = new HashSet<String>();
+ hgComponents1.add("HIVE_SERVER");
+ TestHostGroup group1 = new TestHostGroup("group1", hgComponents1,
Collections.singleton("host1"));
+
+ Collection<String> hgComponents2 = new HashSet<String>();
+ hgComponents2.add("DATANODE");
+ TestHostGroup group2 = new TestHostGroup("group2", hgComponents2,
Collections.singleton("host2"));
+
+ Collection<TestHostGroup> hostGroups = new ArrayList<TestHostGroup>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
+ BlueprintConfigurationProcessor updater = new
BlueprintConfigurationProcessor(topology);
+
+ // call top-level export method
+ Collection<String> requiredGroups = updater.getRequiredHostGroups();
+ assertEquals(0, requiredGroups.size());
+ }
+
+ @Test
+ public void testGetRequiredHostGroups___multipleGroups() throws Exception {
+ Map<String, Map<String, String>> properties = new HashMap<String,
Map<String, String>>();
+ Map<String, String> coreSiteMap = new HashMap<String, String>();
+ properties.put("core-site", coreSiteMap);
+
+ coreSiteMap.put("fs.defaultFS", "localhost");
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ expect(stack.getCardinality("NAMENODE")).andReturn(new
Cardinality("1")).anyTimes();
+ Collection<String> hgComponents1 = new HashSet<String>();
+ hgComponents1.add("HIVE_SERVER");
+ hgComponents1.add("NAMENODE");
+ TestHostGroup group1 = new TestHostGroup("group1", hgComponents1,
Collections.singleton("host1"));
+
+ Collection<String> hgComponents2 = new HashSet<String>();
+ hgComponents2.add("NAMENODE");
+ hgComponents2.add("DATANODE");
+ TestHostGroup group2 = new TestHostGroup("group2", hgComponents2,
Collections.singleton("host2"));
+
+ Collection<TestHostGroup> hostGroups = new ArrayList<TestHostGroup>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
+ BlueprintConfigurationProcessor updater = new
BlueprintConfigurationProcessor(topology);
+
+ // call top-level export method
+ Collection<String> requiredGroups = updater.getRequiredHostGroups();
+ assertEquals(2, requiredGroups.size());
+ assertTrue(requiredGroups.containsAll(Arrays.asList("group1", "group2")));
}
private static String createExportedAddress(String expectedPortNum, String
expectedHostGroupName) {
