from:"jspeidel"

[2/2] ambari git commit: AMBARI-14219. Set HBASE_CONF_DIR env var in atlas-env.xml

2015-12-04 Thread jspeidel

AMBARI-14219. Set HBASE_CONF_DIR env var in atlas-env.xml


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/606b84c9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/606b84c9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/606b84c9

Branch: refs/heads/branch-2.2
Commit: 606b84c917b0049b188e56ca501c2254b85fa654
Parents: f8f6e01
Author: John Speidel 
Authored: Fri Dec 4 14:40:50 2015 -0500
Committer: John Speidel 
Committed: Fri Dec 4 17:40:05 2015 -0500

--
 .../common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml   | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/606b84c9/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
--
diff --git 
a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
 
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
index 7d72ee4..d69d618 100644
--- 
a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
+++ 
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
@@ -83,6 +83,9 @@ export METADATA_DATA_DIR={{data_dir}}
 # pid dir
 export METADATA_PID_DIR={{pid_dir}}
 
+# hbase conf dir
+export HBASE_CONF_DIR=/etc/hbase/conf
+
 # Where do you want to expand the war file. By Default it is in /server/webapp 
dir under the base install dir.
 export METADATA_EXPANDED_WEBAPP_DIR={{expanded_war_dir}}

[1/2] ambari git commit: AMBARI-14218. Add Atlas thread pool configuration properties to hive-site.xml

2015-12-04 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-2.2 7ca6fe27d -> 606b84c91


AMBARI-14218. Add Atlas thread pool configuration properties to hive-site.xml


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f8f6e011
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f8f6e011
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f8f6e011

Branch: refs/heads/branch-2.2
Commit: f8f6e01120ad51e137ae8a0c45ae99255067f288
Parents: 7ca6fe2
Author: John Speidel 
Authored: Fri Dec 4 11:53:58 2015 -0500
Committer: John Speidel 
Committed: Fri Dec 4 17:30:19 2015 -0500

--
 .../2.3/services/HIVE/configuration/hive-site.xml   | 16 
 1 file changed, 16 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/f8f6e011/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
--
diff --git 
a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
 
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
index 5d0f5e0..a611386 100644
--- 
a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
+++ 
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
@@ -40,4 +40,20 @@ limitations under the License.
 
   
 
+  
+atlas.hook.hive.minThreads
+1
+
+  Minimum number of threads maintained by Atlas hook.
+
+  
+
+  
+atlas.hook.hive.maxThreads
+1
+
+  Maximum number of threads used by Atlas hook.
+
+  
+

[1/2] ambari git commit: AMBARI-14219. Set HBASE_CONF_DIR env var in atlas-env.xml

2015-12-04 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk b5041a107 -> 6d63960fc


AMBARI-14219. Set HBASE_CONF_DIR env var in atlas-env.xml


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6d63960f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6d63960f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6d63960f

Branch: refs/heads/trunk
Commit: 6d63960fcd365e748a2e62d7b3e71d6b43dda98c
Parents: e763e69
Author: John Speidel 
Authored: Fri Dec 4 14:40:50 2015 -0500
Committer: John Speidel 
Committed: Fri Dec 4 17:20:34 2015 -0500

--
 .../common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml   | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/6d63960f/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
--
diff --git 
a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
 
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
index c365dd5..2935e8f 100644
--- 
a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
+++ 
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
@@ -86,6 +86,9 @@ export METADATA_DATA_DIR={{data_dir}}
 # pid dir
 export METADATA_PID_DIR={{pid_dir}}
 
+# hbase conf dir
+export HBASE_CONF_DIR=/etc/hbase/conf
+
 # Where do you want to expand the war file. By Default it is in /server/webapp 
dir under the base install dir.
 export METADATA_EXPANDED_WEBAPP_DIR={{expanded_war_dir}}

[2/2] ambari git commit: AMBARI-14218. Add Atlas thread pool configuration properties to hive-site.xml

2015-12-04 Thread jspeidel

AMBARI-14218. Add Atlas thread pool configuration properties to hive-site.xml


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e763e693
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e763e693
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e763e693

Branch: refs/heads/trunk
Commit: e763e693099afa726e16e6ff8775492166461af6
Parents: b5041a1
Author: John Speidel 
Authored: Fri Dec 4 11:53:58 2015 -0500
Committer: John Speidel 
Committed: Fri Dec 4 17:20:34 2015 -0500

--
 .../2.3/services/HIVE/configuration/hive-site.xml   | 16 
 1 file changed, 16 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/e763e693/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
--
diff --git 
a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
 
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
index 5d0f5e0..a611386 100644
--- 
a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
+++ 
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
@@ -40,4 +40,20 @@ limitations under the License.
 
   
 
+  
+atlas.hook.hive.minThreads
+1
+
+  Minimum number of threads maintained by Atlas hook.
+
+  
+
+  
+atlas.hook.hive.maxThreads
+1
+
+  Maximum number of threads used by Atlas hook.
+
+  
+

ambari git commit: AMBARI-13129. Integrate Hive with Atlas in Blueprints

2015-10-02 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk bfc35fe8f -> 321d57b0d


AMBARI-13129.  Integrate Hive with Atlas in Blueprints


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/321d57b0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/321d57b0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/321d57b0

Branch: refs/heads/trunk
Commit: 321d57b0de3475977741497d206990dccd4b6f40
Parents: bfc35fe
Author: John Speidel 
Authored: Thu Oct 1 15:54:43 2015 -0400
Committer: John Speidel 
Committed: Fri Oct 2 15:17:17 2015 -0400

--
 .../BlueprintConfigurationProcessor.java| 93 +++-
 .../ATLAS/0.1.0.2.3/metainfo.xml| 27 +-
 .../0.1.0.2.3/package/scripts/atlas_client.py   | 48 ++
 .../BlueprintConfigurationProcessorTest.java| 93 
 4 files changed, 256 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/321d57b0/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 892cf32..5fd5563 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -1077,9 +1077,9 @@ public class BlueprintConfigurationProcessor {
   } else {
 int matchingGroupCount = 
topology.getHostGroupsForComponent(component).size();
 if (matchingGroupCount == 1) {
-  Collection componentHosts = 
topology.getHostAssignmentsForComponent(component);
   //todo: warn if > 1 hosts
-  return origValue.replace("localhost", 
componentHosts.iterator().next());
+  return replacePropertyValue(origValue,
+  
topology.getHostAssignmentsForComponent(component).iterator().next(), 
properties);
 } else {
   //todo: extract all hard coded HA logic
   Cardinality cardinality = 
topology.getBlueprint().getStack().getCardinality(component);
@@ -1159,6 +1159,10 @@ public class BlueprintConfigurationProcessor {
   }
 }
 
+public String replacePropertyValue(String origValue, String host, 
Map> properties) {
+  return origValue.replace("localhost", host);
+}
+
 @Override
 public Collection getRequiredHostGroups(String propertyName,
 String origValue,
@@ -1856,6 +1860,21 @@ public class BlueprintConfigurationProcessor {
   }
 
   /**
+   * A topology independent updater which provides a default implementation of 
getRequiredHostGroups
+   * since no topology related information is required by the updater.
+   */
+  private static abstract class NonTopologyUpdater implements PropertyUpdater {
+@Override
+public Collection getRequiredHostGroups(String propertyName,
+String origValue,
+Map> properties,
+ClusterTopology topology) {
+  return Collections.emptyList();
+}
+  }
+
+
+  /**
* Register updaters for configuration properties.
*/
   static {
@@ -1877,6 +1896,7 @@ public class BlueprintConfigurationProcessor {
 Map accumuloSiteMap = new HashMap();
 Map falconStartupPropertiesMap = new 
HashMap();
 Map kafkaBrokerMap = new HashMap();
+Map atlasPropsMap = new HashMap();
 Map mapredEnvMap = new HashMap();
 Map hadoopEnvMap = new HashMap();
 Map hbaseEnvMap = new HashMap();
@@ -1911,6 +1931,7 @@ public class BlueprintConfigurationProcessor {
 singleHostTopologyUpdaters.put("hive-env", hiveEnvMap);
 singleHostTopologyUpdaters.put("oozie-env", oozieEnvMap);
 singleHostTopologyUpdaters.put("kafka-broker", kafkaBrokerMap);
+singleHostTopologyUpdaters.put("application-properties", atlasPropsMap);
 
 mPropertyUpdaters.put("hadoop-env", hadoopEnvMap);

ambari git commit: AMBARI-12366. Fix issue where the BP configuration is modified on a getter invocation due to an incomplete deep copy

2015-07-09 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 630299fb3 - ba21f8988


AMBARI-12366.  Fix issue where the BP configuration is modified on a getter 
invocation
   due to an incomplete deep copy


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ba21f898
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ba21f898
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ba21f898

Branch: refs/heads/trunk
Commit: ba21f89881d80de8be1256336fa6f655db02fd54
Parents: 630299f
Author: John Speidel jspei...@hortonworks.com
Authored: Thu Jul 9 17:45:13 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Jul 9 19:06:45 2015 -0400

--
 .../ambari/server/topology/Configuration.java   | 12 +--
 .../server/topology/ConfigurationTest.java  | 37 +++-
 2 files changed, 21 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/ba21f898/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
index 91b7736..108ff74 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
@@ -115,7 +115,11 @@ public class Configuration {
*/
   public MapString, MapString, String getFullProperties(int depthLimit) {
 if (depthLimit == 0) {
-  return new HashMapString, MapString, String(properties);
+  HashMapString, MapString, String propertiesCopy = new 
HashMapString, MapString, String();
+  for (Map.EntryString, MapString, String typeProperties : 
properties.entrySet()) {
+propertiesCopy.put(typeProperties.getKey(), new HashMapString, 
String(typeProperties.getValue()));
+  }
+  return propertiesCopy;
 }
 
 MapString, MapString, String mergedProperties = parentConfiguration == 
null ?
@@ -159,8 +163,10 @@ public class Configuration {
 
 for (Map.EntryString, MapString, MapString, String typeEntry : 
attributes.entrySet()) {
   String type = typeEntry.getKey();
-  MapString, MapString, String typeAttributes =
-  new HashMapString, MapString, String(typeEntry.getValue());
+  MapString, MapString, String typeAttributes = new HashMapString, 
MapString, String();
+  for (Map.EntryString, MapString, String attributeEntry : 
typeEntry.getValue().entrySet()) {
+typeAttributes.put(attributeEntry.getKey(), new HashMapString, 
String(attributeEntry.getValue()));
+  }
 
   if (! mergedAttributeMap.containsKey(type)) {
 mergedAttributeMap.put(type, typeAttributes);

http://git-wip-us.apache.org/repos/asf/ambari/blob/ba21f898/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
index 9bab88f..e971e03 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
@@ -106,12 +106,6 @@ public class ConfigurationTest {
   @Test
   public void testGetFullProperties_withParent() {
 Configuration configuration = createConfigurationWithParents_PropsOnly();
-// get prop maps prior to calling getFullProperties
-MapString, MapString, String leafProperties = 
configuration.getProperties();
-MapString, MapString, String parentProperties = 
configuration.getParentConfiguration().getProperties();
-MapString, MapString, String parentParentProperties = 
configuration.getParentConfiguration().getParentConfiguration().getProperties();
-
-// test
 // all parents should be reflected in getFullProperties() result
 MapString, MapString, String fullProperties = 
configuration.getFullProperties();
 
@@ -145,9 +139,10 @@ public class ConfigurationTest {
 assertEquals(val11.3, type4Props.get(prop11));
 
 // ensure that underlying property map is not modified in getFullProperties
-assertEquals(leafProperties, configuration.getProperties());
-assertEquals(parentProperties, 
configuration.getParentConfiguration().getProperties());
-assertEquals(parentParentProperties, 
configuration.getParentConfiguration().getParentConfiguration().getProperties());
+Configuration expectedConfiguration =

ambari git commit: AMBARI-12366. Fix issue where the BP configuration is modified on a getter invocation due to an incomplete deep copy

2015-07-09 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 a27ad20b1 - 719d16cd5


AMBARI-12366.  Fix issue where the BP configuration is modified on a getter 
invocation
   due to an incomplete deep copy


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/719d16cd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/719d16cd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/719d16cd

Branch: refs/heads/branch-2.1
Commit: 719d16cd5d3db25c22d86389e73a82958a5a
Parents: a27ad20
Author: John Speidel jspei...@hortonworks.com
Authored: Thu Jul 9 17:45:13 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Jul 9 19:11:36 2015 -0400

--
 .../ambari/server/topology/Configuration.java   | 12 +--
 .../server/topology/ConfigurationTest.java  | 37 +++-
 2 files changed, 21 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/719d16cd/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
index 91b7736..108ff74 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
@@ -115,7 +115,11 @@ public class Configuration {
*/
   public MapString, MapString, String getFullProperties(int depthLimit) {
 if (depthLimit == 0) {
-  return new HashMapString, MapString, String(properties);
+  HashMapString, MapString, String propertiesCopy = new 
HashMapString, MapString, String();
+  for (Map.EntryString, MapString, String typeProperties : 
properties.entrySet()) {
+propertiesCopy.put(typeProperties.getKey(), new HashMapString, 
String(typeProperties.getValue()));
+  }
+  return propertiesCopy;
 }
 
 MapString, MapString, String mergedProperties = parentConfiguration == 
null ?
@@ -159,8 +163,10 @@ public class Configuration {
 
 for (Map.EntryString, MapString, MapString, String typeEntry : 
attributes.entrySet()) {
   String type = typeEntry.getKey();
-  MapString, MapString, String typeAttributes =
-  new HashMapString, MapString, String(typeEntry.getValue());
+  MapString, MapString, String typeAttributes = new HashMapString, 
MapString, String();
+  for (Map.EntryString, MapString, String attributeEntry : 
typeEntry.getValue().entrySet()) {
+typeAttributes.put(attributeEntry.getKey(), new HashMapString, 
String(attributeEntry.getValue()));
+  }
 
   if (! mergedAttributeMap.containsKey(type)) {
 mergedAttributeMap.put(type, typeAttributes);

http://git-wip-us.apache.org/repos/asf/ambari/blob/719d16cd/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
index 9bab88f..e971e03 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
@@ -106,12 +106,6 @@ public class ConfigurationTest {
   @Test
   public void testGetFullProperties_withParent() {
 Configuration configuration = createConfigurationWithParents_PropsOnly();
-// get prop maps prior to calling getFullProperties
-MapString, MapString, String leafProperties = 
configuration.getProperties();
-MapString, MapString, String parentProperties = 
configuration.getParentConfiguration().getProperties();
-MapString, MapString, String parentParentProperties = 
configuration.getParentConfiguration().getParentConfiguration().getProperties();
-
-// test
 // all parents should be reflected in getFullProperties() result
 MapString, MapString, String fullProperties = 
configuration.getFullProperties();
 
@@ -145,9 +139,10 @@ public class ConfigurationTest {
 assertEquals(val11.3, type4Props.get(prop11));
 
 // ensure that underlying property map is not modified in getFullProperties
-assertEquals(leafProperties, configuration.getProperties());
-assertEquals(parentProperties, 
configuration.getParentConfiguration().getProperties());
-assertEquals(parentParentProperties, 
configuration.getParentConfiguration().getParentConfiguration().getProperties());
+Configuration expectedConfiguration =

ambari git commit: AMBARI-12332. Fix atlas related property names in Hive stack params script

2015-07-08 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 2275ae839 - 8a31b1043


AMBARI-12332. Fix atlas related property names in Hive stack params script


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8a31b104
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8a31b104
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8a31b104

Branch: refs/heads/trunk
Commit: 8a31b1043e9c5bae78b2171937d7a25a1a9bfa37
Parents: 2275ae8
Author: Jon Maron jma...@hortonworks.com
Authored: Wed Jul 8 13:49:29 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed Jul 8 13:51:43 2015 -0400

--
 .../HIVE/0.12.0.2.0/package/scripts/params_linux.py  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/8a31b104/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
--
diff --git 
a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
 
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index bee455f..3f6532c 100644
--- 
a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ 
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -377,7 +377,7 @@ if not has_atlas:
   hive_exclude_packages.append(atlas_plugin_package)
 else:
   # hive-site
-  hive_site_config['hive.cluster.name'] = config['clusterName']
+  hive_site_config['atlas.cluster.name'] = config['clusterName']
   atlas_config = config['configurations']['application-properties']
   metadata_port = config['configurations']['atlas-env']['metadata_port']
   metadata_host = atlas_hosts[0]
@@ -386,7 +386,7 @@ else:
 scheme = https
   else:
 scheme = http
-  hive_site_config['hive.hook.dgi.url'] = 
format('{scheme}://{metadata_host}:{metadata_port}')
+  hive_site_config['atlas.rest.address'] = 
format('{scheme}://{metadata_host}:{metadata_port}')
 
   if not 'hive.exec.post.hooks' in hive_site_config:
 hive_site_config['hive.exec.post.hooks'] = 
'org.apache.atlas.hive.hook.HiveHook'

ambari git commit: AMBARI-12332. Fix atlas related property names in Hive stack params script

2015-07-08 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 5950981b2 - b0b154ac9


AMBARI-12332. Fix atlas related property names in Hive stack params script


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b0b154ac
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b0b154ac
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b0b154ac

Branch: refs/heads/branch-2.1
Commit: b0b154ac9caaa2f55659132233a8ac0738dc42b1
Parents: 5950981
Author: Jon Maron jma...@hortonworks.com
Authored: Wed Jul 8 13:49:29 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed Jul 8 13:52:24 2015 -0400

--
 .../HIVE/0.12.0.2.0/package/scripts/params_linux.py  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/b0b154ac/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
--
diff --git 
a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
 
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index bee455f..3f6532c 100644
--- 
a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ 
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -377,7 +377,7 @@ if not has_atlas:
   hive_exclude_packages.append(atlas_plugin_package)
 else:
   # hive-site
-  hive_site_config['hive.cluster.name'] = config['clusterName']
+  hive_site_config['atlas.cluster.name'] = config['clusterName']
   atlas_config = config['configurations']['application-properties']
   metadata_port = config['configurations']['atlas-env']['metadata_port']
   metadata_host = atlas_hosts[0]
@@ -386,7 +386,7 @@ else:
 scheme = https
   else:
 scheme = http
-  hive_site_config['hive.hook.dgi.url'] = 
format('{scheme}://{metadata_host}:{metadata_port}')
+  hive_site_config['atlas.rest.address'] = 
format('{scheme}://{metadata_host}:{metadata_port}')
 
   if not 'hive.exec.post.hooks' in hive_site_config:
 hive_site_config['hive.exec.post.hooks'] = 
'org.apache.atlas.hive.hook.HiveHook'

ambari git commit: AMBARI-12009. Fix occasional ConcurrentModificationException when provisioning large clusters via blueprints

2015-06-18 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk af7e26704 - 1b6d7a9e1


AMBARI-12009. Fix occasional ConcurrentModificationException when provisioning
  large clusters via blueprints


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1b6d7a9e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1b6d7a9e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1b6d7a9e

Branch: refs/heads/trunk
Commit: 1b6d7a9e1a5484c56107a2957284392ebbbc8272
Parents: af7e267
Author: John Speidel jspei...@hortonworks.com
Authored: Thu Jun 18 16:58:15 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Jun 18 18:47:45 2015 -0400

--
 .../ambari/server/topology/HostGroupInfo.java   | 112 --
 .../server/topology/HostGroupInfoTest.java  | 116 +++
 2 files changed, 219 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/1b6d7a9e/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
index a48f331..8ff3971 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
@@ -29,71 +29,165 @@ import java.util.HashSet;
  * Host Group information specific to a cluster instance.
  */
 public class HostGroupInfo {
-
+  /**
+   * predicate compiler
+   */
   private static PredicateCompiler predicateCompiler = new PredicateCompiler();
 
+  /**
+   * host group name
+   */
   private String hostGroupName;
   /**
-   * Hosts contained associated with the host group
+   * hosts contained associated with the host group
*/
-  private CollectionString hostNames = new HashSetString();
+  private final CollectionString hostNames = new HashSetString();
 
+  /**
+   * explicitly specified host count
+   */
   private int requested_count = 0;
 
+  /**
+   * host group scoped configuration
+   */
   Configuration configuration;
 
+  /**
+   * explicitly specified host predicate string
+   */
   String predicateString;
+
+  /**
+   * compiled host predicate
+   */
   Predicate predicate;
 
 
+  /**
+   * Constructor
+   *
+   * @param hostGroupName  host group name
+   */
   public HostGroupInfo(String hostGroupName) {
 this.hostGroupName = hostGroupName;
   }
 
+  /**
+   * Get the host group name.
+   *
+   * @return host group name
+   */
   public String getHostGroupName() {
 return hostGroupName;
   }
 
+  /**
+   * Get the collection of user specified host names for the host group.
+   * If the user specified a count instead of host names then an empty
+   * collection is returned.
+   *
+   * @return collection of user specified host names; will never be null
+   */
   public CollectionString getHostNames() {
-return new HashSetString(hostNames);
+// needs to be an exclusive lock, not a read lock because collection
+// shouldn't change while copying elements into the new set instance
+synchronized (hostNames) {
+  return new HashSetString(hostNames);
+}
   }
 
+  /**
+   * Get the requested host count.
+   * This is either the user specified value or
+   * the number of explicitly specified host names specified by the user.
+   *
+   * @return number of requested hosts for the group
+   */
   public int getRequestedHostCount() {
-return requested_count == 0 ? hostNames.size() : requested_count;
+synchronized (hostNames) {
+  return requested_count == 0 ? hostNames.size() : requested_count;
+}
   }
 
+  /**
+   * Associate a single host name to the host group.
+   *
+   * @param hostName  the host name to associate with the host group
+   */
   public void addHost(String hostName) {
-hostNames.add(hostName);
+synchronized(hostNames) {
+  hostNames.add(hostName);
+}
   }
 
+  /**
+   * Associate multiple host names to the host group.
+   *
+   * @param hosts  collection of host names to associate with the host group
+   */
   public void addHosts(CollectionString hosts) {
-for (String host : hosts) {
-  addHost(host);
+synchronized (hostNames) {
+  for (String host : hosts) {
+addHost(host);
+  }
 }
   }
 
+  /**
+   * Set the requested host count for the host group.
+   *
+   * @param num  requested host count
+   */
   public void setRequestedCount(int num) {
 requested_count = num;
   }
 
-  //todo: constructor?
+  /**
+   * Set host group configuration for the host group.
+   *
+   * @param configuration configuration instance
+

ambari git commit: AMBARI-12009. Fix occasional ConcurrentModificationException when provisioning large clusters via blueprints

2015-06-18 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 48c02a12a - 527e75c19


AMBARI-12009. Fix occasional ConcurrentModificationException when provisioning
  large clusters via blueprints


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/527e75c1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/527e75c1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/527e75c1

Branch: refs/heads/branch-2.1
Commit: 527e75c1985b068e54e18f6480536a4ed6cb574f
Parents: 48c02a1
Author: John Speidel jspei...@hortonworks.com
Authored: Thu Jun 18 16:58:15 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Jun 18 18:48:39 2015 -0400

--
 .../ambari/server/topology/HostGroupInfo.java   | 112 --
 .../server/topology/HostGroupInfoTest.java  | 116 +++
 2 files changed, 219 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/527e75c1/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
index a48f331..8ff3971 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java
@@ -29,71 +29,165 @@ import java.util.HashSet;
  * Host Group information specific to a cluster instance.
  */
 public class HostGroupInfo {
-
+  /**
+   * predicate compiler
+   */
   private static PredicateCompiler predicateCompiler = new PredicateCompiler();
 
+  /**
+   * host group name
+   */
   private String hostGroupName;
   /**
-   * Hosts contained associated with the host group
+   * hosts contained associated with the host group
*/
-  private CollectionString hostNames = new HashSetString();
+  private final CollectionString hostNames = new HashSetString();
 
+  /**
+   * explicitly specified host count
+   */
   private int requested_count = 0;
 
+  /**
+   * host group scoped configuration
+   */
   Configuration configuration;
 
+  /**
+   * explicitly specified host predicate string
+   */
   String predicateString;
+
+  /**
+   * compiled host predicate
+   */
   Predicate predicate;
 
 
+  /**
+   * Constructor
+   *
+   * @param hostGroupName  host group name
+   */
   public HostGroupInfo(String hostGroupName) {
 this.hostGroupName = hostGroupName;
   }
 
+  /**
+   * Get the host group name.
+   *
+   * @return host group name
+   */
   public String getHostGroupName() {
 return hostGroupName;
   }
 
+  /**
+   * Get the collection of user specified host names for the host group.
+   * If the user specified a count instead of host names then an empty
+   * collection is returned.
+   *
+   * @return collection of user specified host names; will never be null
+   */
   public CollectionString getHostNames() {
-return new HashSetString(hostNames);
+// needs to be an exclusive lock, not a read lock because collection
+// shouldn't change while copying elements into the new set instance
+synchronized (hostNames) {
+  return new HashSetString(hostNames);
+}
   }
 
+  /**
+   * Get the requested host count.
+   * This is either the user specified value or
+   * the number of explicitly specified host names specified by the user.
+   *
+   * @return number of requested hosts for the group
+   */
   public int getRequestedHostCount() {
-return requested_count == 0 ? hostNames.size() : requested_count;
+synchronized (hostNames) {
+  return requested_count == 0 ? hostNames.size() : requested_count;
+}
   }
 
+  /**
+   * Associate a single host name to the host group.
+   *
+   * @param hostName  the host name to associate with the host group
+   */
   public void addHost(String hostName) {
-hostNames.add(hostName);
+synchronized(hostNames) {
+  hostNames.add(hostName);
+}
   }
 
+  /**
+   * Associate multiple host names to the host group.
+   *
+   * @param hosts  collection of host names to associate with the host group
+   */
   public void addHosts(CollectionString hosts) {
-for (String host : hosts) {
-  addHost(host);
+synchronized (hostNames) {
+  for (String host : hosts) {
+addHost(host);
+  }
 }
   }
 
+  /**
+   * Set the requested host count for the host group.
+   *
+   * @param num  requested host count
+   */
   public void setRequestedCount(int num) {
 requested_count = num;
   }
 
-  //todo: constructor?
+  /**
+   * Set host group configuration for the host group.
+   *
+   * @param configuration configuration

ambari git commit: AMBARI-11886. return 400 response and good error msg when user attempts to use API to scale a UI provisioned cluster

2015-06-12 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 4987bfa0a - 80c221bd4


AMBARI-11886.  return 400 response and good error msg when user attempts
   to use API to scale a UI provisioned cluster


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/80c221bd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/80c221bd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/80c221bd

Branch: refs/heads/trunk
Commit: 80c221bd407469cdc47de187aa1d65a43be5c3a4
Parents: 4987bfa
Author: John Speidel jspei...@hortonworks.com
Authored: Fri Jun 12 10:35:04 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Fri Jun 12 13:11:06 2015 -0400

--
 .../org/apache/ambari/server/topology/TopologyManager.java | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/80c221bd/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
index a75cccb..31363b4 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
@@ -117,7 +117,11 @@ public class TopologyManager {
 String clusterName = request.getClusterName();
 ClusterTopology topology = clusterTopologyMap.get(clusterName);
 if (topology == null) {
-  throw new AmbariException(TopologyManager: Unable to retrieve cluster 
topology for cluster:  + clusterName);
+  throw new InvalidTopologyException(Unable to retrieve cluster topology 
for cluster. This is most likely a  +
+ result of trying to scale a cluster 
via the API which was created using  +
+ the Ambari UI. At this time only 
clusters created via the API using a  +
+ blueprint can be scaled with this 
API.  If the cluster was originally created  +
+ via the API as described above, 
please file a Jira for this matter.);
 }
 
 PersistedTopologyRequest persistedRequest = 
persistedState.persistTopologyRequest(request);

ambari git commit: AMBARI-11886. return 400 response and good error msg when user attempts to use API to scale a UI provisioned cluster

2015-06-12 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 f22fd6b76 - 1659d2af4


AMBARI-11886.  return 400 response and good error msg when user attempts
   to use API to scale a UI provisioned cluster


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1659d2af
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1659d2af
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1659d2af

Branch: refs/heads/branch-2.1
Commit: 1659d2af49dc157787bcd9495df676fe7a5930a5
Parents: f22fd6b
Author: John Speidel jspei...@hortonworks.com
Authored: Fri Jun 12 10:35:04 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Fri Jun 12 13:11:52 2015 -0400

--
 .../org/apache/ambari/server/topology/TopologyManager.java | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/1659d2af/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
index a75cccb..31363b4 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
@@ -117,7 +117,11 @@ public class TopologyManager {
 String clusterName = request.getClusterName();
 ClusterTopology topology = clusterTopologyMap.get(clusterName);
 if (topology == null) {
-  throw new AmbariException(TopologyManager: Unable to retrieve cluster 
topology for cluster:  + clusterName);
+  throw new InvalidTopologyException(Unable to retrieve cluster topology 
for cluster. This is most likely a  +
+ result of trying to scale a cluster 
via the API which was created using  +
+ the Ambari UI. At this time only 
clusters created via the API using a  +
+ blueprint can be scaled with this 
API.  If the cluster was originally created  +
+ via the API as described above, 
please file a Jira for this matter.);
 }
 
 PersistedTopologyRequest persistedRequest = 
persistedState.persistTopologyRequest(request);

ambari git commit: AMBARI-11867. Fix adding of host to config group during blueprint provisioning

2015-06-11 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk dc25acbc9 - 043e35a6b


AMBARI-11867.  Fix adding of host to config group during blueprint provisioning


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/043e35a6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/043e35a6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/043e35a6

Branch: refs/heads/trunk
Commit: 043e35a6b5bf970406b1d935edeb7ee06e4e9936
Parents: dc25acb
Author: John Speidel jspei...@hortonworks.com
Authored: Thu Jun 11 17:13:35 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Jun 11 18:28:39 2015 -0400

--
 .../server/state/configgroup/ConfigGroup.java   |  8 +---
 .../state/configgroup/ConfigGroupImpl.java  |  2 +-
 .../ambari/server/topology/AmbariContext.java   | 10 +++---
 .../server/topology/AmbariContextTest.java  | 21 +++-
 4 files changed, 33 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/043e35a6/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroup.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroup.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroup.java
index 4c806e5..7ed7ba5 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroup.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroup.java
@@ -18,7 +18,6 @@
 
 package org.apache.ambari.server.state.configgroup;
 
-import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.controller.ConfigGroupResponse;
 import org.apache.ambari.server.state.Config;
@@ -96,10 +95,14 @@ public interface ConfigGroup {
* Persist the Config group along with the related host and config mapping
* entities to the persistence store
*/
-  @Transactional
   void persist();
 
   /**
+   * Persist the host mapping entity to the persistence store
+   */
+  void persistHostMapping();
+
+  /**
* Delete config group and the related host and config mapping
* entities from the persistence store
*/
@@ -130,7 +133,6 @@ public interface ConfigGroup {
   /**
* Refresh Config group and the host and config mappings for the group
*/
-  @Transactional
   public void refresh();
 
   /**

http://git-wip-us.apache.org/repos/asf/ambari/blob/043e35a6/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroupImpl.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroupImpl.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroupImpl.java
index d322735..3b83f61 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroupImpl.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroupImpl.java
@@ -361,7 +361,7 @@ public class ConfigGroupImpl implements ConfigGroup {
* @throws Exception
*/
   @Transactional
-  void persistHostMapping() {
+  public void persistHostMapping() {
 if (isPersisted) {
   // Delete existing mappings and create new ones
   
configGroupHostMappingDAO.removeAllByGroup(configGroupEntity.getGroupId());

http://git-wip-us.apache.org/repos/asf/ambari/blob/043e35a6/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 091018a..42676aa 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -405,13 +405,17 @@ public class AmbariContext {
 for (ConfigGroup group : configGroups.values()) {
   if (group.getName().equals(qualifiedGroupName)) {
 try {
-  group.addHost(clusters.getHost(hostName));
-  group.persist();
+  Host host = clusters.getHost(hostName);
   addedHost = true;
+  if (! group.getHosts().containsKey(host.getHostId())) {
+group.addHost(host);
+group.persistHostMapping();
+  }
+
 } catch (AmbariException e) {
   // shouldn't occur, this host was just added to the cluster
   throw new RuntimeException(String.format(
-  Unable

ambari git commit: AMBARI-11867. Fix adding of host to config group during blueprint provisioning

2015-06-11 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 f88b30e9e - 0bf83a6d3


AMBARI-11867.  Fix adding of host to config group during blueprint provisioning


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0bf83a6d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0bf83a6d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0bf83a6d

Branch: refs/heads/branch-2.1
Commit: 0bf83a6d3e9efdf90bbe3462af309b10f9af88f9
Parents: f88b30e
Author: John Speidel jspei...@hortonworks.com
Authored: Thu Jun 11 17:13:35 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Jun 11 18:29:36 2015 -0400

--
 .../server/state/configgroup/ConfigGroup.java   |  8 +---
 .../state/configgroup/ConfigGroupImpl.java  |  2 +-
 .../ambari/server/topology/AmbariContext.java   | 10 +++---
 .../server/topology/AmbariContextTest.java  | 21 +++-
 4 files changed, 33 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/0bf83a6d/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroup.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroup.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroup.java
index 4c806e5..7ed7ba5 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroup.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroup.java
@@ -18,7 +18,6 @@
 
 package org.apache.ambari.server.state.configgroup;
 
-import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.controller.ConfigGroupResponse;
 import org.apache.ambari.server.state.Config;
@@ -96,10 +95,14 @@ public interface ConfigGroup {
* Persist the Config group along with the related host and config mapping
* entities to the persistence store
*/
-  @Transactional
   void persist();
 
   /**
+   * Persist the host mapping entity to the persistence store
+   */
+  void persistHostMapping();
+
+  /**
* Delete config group and the related host and config mapping
* entities from the persistence store
*/
@@ -130,7 +133,6 @@ public interface ConfigGroup {
   /**
* Refresh Config group and the host and config mappings for the group
*/
-  @Transactional
   public void refresh();
 
   /**

http://git-wip-us.apache.org/repos/asf/ambari/blob/0bf83a6d/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroupImpl.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroupImpl.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroupImpl.java
index d322735..3b83f61 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroupImpl.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/configgroup/ConfigGroupImpl.java
@@ -361,7 +361,7 @@ public class ConfigGroupImpl implements ConfigGroup {
* @throws Exception
*/
   @Transactional
-  void persistHostMapping() {
+  public void persistHostMapping() {
 if (isPersisted) {
   // Delete existing mappings and create new ones
   
configGroupHostMappingDAO.removeAllByGroup(configGroupEntity.getGroupId());

http://git-wip-us.apache.org/repos/asf/ambari/blob/0bf83a6d/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 091018a..42676aa 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -405,13 +405,17 @@ public class AmbariContext {
 for (ConfigGroup group : configGroups.values()) {
   if (group.getName().equals(qualifiedGroupName)) {
 try {
-  group.addHost(clusters.getHost(hostName));
-  group.persist();
+  Host host = clusters.getHost(hostName);
   addedHost = true;
+  if (! group.getHosts().containsKey(host.getHostId())) {
+group.addHost(host);
+group.persistHostMapping();
+  }
+
 } catch (AmbariException e) {
   // shouldn't occur, this host was just added to the cluster
   throw new RuntimeException(String.format(
-

ambari git commit: AMBARI-11790. Remove all occurrences of a property from a configuration hierarchy

2015-06-08 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk ce33632aa - 6a9d590aa


AMBARI-11790. Remove all occurrences of a property from a configuration 
hierarchy


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6a9d590a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6a9d590a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6a9d590a

Branch: refs/heads/trunk
Commit: 6a9d590aaeeb24045a62fa3780582dbb51a7521f
Parents: ce33632
Author: John Speidel jspei...@hortonworks.com
Authored: Mon Jun 8 11:19:38 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Jun 8 13:11:27 2015 -0400

--
 .../ambari/server/topology/Configuration.java   | 15 +--
 .../server/topology/ConfigurationTest.java  | 44 
 2 files changed, 48 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/6a9d590a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
index b7b9343..91b7736 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
@@ -248,6 +248,8 @@ public class Configuration {
 
   /**
* Remove a property from the configuration hierarchy.
+   * All occurrences of the property are removed from the config hierarchy 
such that
+   * a subsequent call to getPropertyValue() for the removed property will 
return null.
*
* @param configTypeconfiguration type
* @param propertyName  property name
@@ -257,10 +259,15 @@ public class Configuration {
*/
   public String removeProperty(String configType, String propertyName) {
 String previousValue = null;
-if (properties.containsKey(configType)  
properties.get(configType).containsKey(propertyName)) {
-  previousValue =  properties.get(configType).remove(propertyName);
-} else if (parentConfiguration != null) {
-  previousValue =  parentConfiguration.removeProperty(configType, 
propertyName);
+if (properties.containsKey(configType)) {
+  previousValue = properties.get(configType).remove(propertyName);
+}
+
+if (parentConfiguration != null) {
+  String parentPreviousValue =  
parentConfiguration.removeProperty(configType, propertyName);
+  if (previousValue == null) {
+previousValue = parentPreviousValue;
+  }
 }
 return previousValue;
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/6a9d590a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
index 1b9734e..9bab88f 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
@@ -26,6 +26,7 @@ import java.util.Map;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -104,7 +105,7 @@ public class ConfigurationTest {
 
   @Test
   public void testGetFullProperties_withParent() {
-Configuration configuration = createConfigurationWithParentsPropsOnly();
+Configuration configuration = createConfigurationWithParents_PropsOnly();
 // get prop maps prior to calling getFullProperties
 MapString, MapString, String leafProperties = 
configuration.getProperties();
 MapString, MapString, String parentProperties = 
configuration.getParentConfiguration().getProperties();
@@ -157,7 +158,7 @@ public class ConfigurationTest {
 
   @Test
   public void testGetFullProperties_withParent_specifyDepth() {
-Configuration configuration = createConfigurationWithParentsPropsOnly();
+Configuration configuration = createConfigurationWithParents_PropsOnly();
 // get prop maps prior to calling getFullProperties
 MapString, MapString, String leafProperties = 
configuration.getProperties();
 MapString, MapString, String parentProperties = 
configuration.getParentConfiguration().getProperties();
@@ -226,7 +227,7 @@ public class ConfigurationTest {
 
   @Test
   public void testGetFullAttributes_withParent() {
-Configuration configuration = 
createConfigurationWithParentsAttributesOnly();
+Configuration

ambari git commit: AMBARI-11542. Fix database deadlock that occasionally occurs when provisiong clusters using the BP api

2015-05-29 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 07a04c95e - 5767697ca


AMBARI-11542. Fix database deadlock that occasionally occurs when provisiong 
clusters
  using the BP api


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5767697c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5767697c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5767697c

Branch: refs/heads/trunk
Commit: 5767697ca26d0c3d458dd6cdb6128d20d105c6ca
Parents: 07a04c9
Author: John Speidel jspei...@hortonworks.com
Authored: Fri May 29 14:47:41 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Fri May 29 16:06:04 2015 -0400

--
 .../AmbariManagementControllerImpl.java | 68 +++-
 1 file changed, 38 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/5767697c/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 792b6fe..ad76ffa 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -58,6 +58,7 @@ import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.ClusterNotFoundException;
@@ -762,7 +763,7 @@ public class AmbariManagementControllerImpl implements 
AmbariManagementControlle
 
   private Config createConfig(Cluster cluster, String type, MapString, 
String properties,
   String versionTag, MapString, MapString, String 
propertiesAttributes) {
-Config config = configFactory.createNew (cluster, type,
+Config config = configFactory.createNew(cluster, type,
 properties, propertiesAttributes);
 
 if (!StringUtils.isEmpty(versionTag)) {
@@ -2397,51 +2398,58 @@ public class AmbariManagementControllerImpl implements 
AmbariManagementControlle
 
 return response;  }
 
-  @Transactional
   void updateServiceStates(
+  Cluster cluster,
   MapState, ListService changedServices,
   MapState, ListServiceComponent changedComps,
   MapString, MapState, ListServiceComponentHost changedScHosts,
   CollectionServiceComponentHost ignoredScHosts
   ) {
-if (changedServices != null) {
-  for (EntryState, ListService entry : changedServices.entrySet()) {
-State newState = entry.getKey();
-for (Service s : entry.getValue()) {
-  if (s.isClientOnlyService()
-   newState == State.STARTED) {
-continue;
+Lock clusterWriteLock = cluster.getClusterGlobalLock().writeLock();
+
+clusterWriteLock.lock();
+try {
+  if (changedServices != null) {
+for (EntryState, ListService entry : changedServices.entrySet()) {
+  State newState = entry.getKey();
+  for (Service s : entry.getValue()) {
+if (s.isClientOnlyService()
+ newState == State.STARTED) {
+  continue;
+}
+s.setDesiredState(newState);
   }
-  s.setDesiredState(newState);
 }
   }
-}
 
-if (changedComps != null) {
-  for (EntryState, ListServiceComponent entry :
-  changedComps.entrySet()){
-State newState = entry.getKey();
-for (ServiceComponent sc : entry.getValue()) {
-  sc.setDesiredState(newState);
+  if (changedComps != null) {
+for (EntryState, ListServiceComponent entry :
+changedComps.entrySet()) {
+  State newState = entry.getKey();
+  for (ServiceComponent sc : entry.getValue()) {
+sc.setDesiredState(newState);
+  }
 }
   }
-}
 
-for (MapState, ListServiceComponentHost stateScHostMap :
-changedScHosts.values()) {
-  for (EntryState, ListServiceComponentHost entry :
-  stateScHostMap.entrySet()) {
-State newState = entry.getKey();
-for (ServiceComponentHost sch : entry.getValue()) {
-  sch.setDesiredState(newState);
+  for (MapState, ListServiceComponentHost stateScHostMap :
+  changedScHosts.values()) {
+for (EntryState, ListServiceComponentHost entry :
+stateScHostMap.entrySet()) {
+  State newState = entry.getKey();
+  for (ServiceComponentHost sch :

ambari git commit: AMBARI-11441. Provide better error message from BlueprintConfigurationProcessor when unable to match a property component token to a host group

2015-05-28 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 087d9003e - b20dbb7fa


AMBARI-11441.  Provide better error message from 
BlueprintConfigurationProcessor when unable
   to match a property component token to a host group


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b20dbb7f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b20dbb7f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b20dbb7f

Branch: refs/heads/trunk
Commit: b20dbb7fae7fc029f353f4eddf280c5f53878421
Parents: 087d900
Author: John Speidel jspei...@hortonworks.com
Authored: Wed May 27 15:37:15 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu May 28 10:22:42 2015 -0400

--
 .../BlueprintConfigurationProcessor.java| 142 +++
 .../BlueprintConfigurationProcessorTest.java|  74 +-
 2 files changed, 123 insertions(+), 93 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/b20dbb7f/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index a84bf3d..6fb4807 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -139,7 +139,7 @@ public class BlueprintConfigurationProcessor {
   MapString, String typeMap = clusterProps.get(type);
   if (typeMap != null  typeMap.containsKey(propertyName)) {
 requiredHostGroups.addAll(updater.getRequiredHostGroups(
-typeMap.get(propertyName), clusterProps, clusterTopology));
+propertyName, typeMap.get(propertyName), clusterProps, 
clusterTopology));
   }
 
   // host group configs
@@ -148,7 +148,7 @@ public class BlueprintConfigurationProcessor {
 MapString, String hgTypeMap = hgConfigProps.get(type);
 if (hgTypeMap != null  hgTypeMap.containsKey(propertyName)) {
   requiredHostGroups.addAll(updater.getRequiredHostGroups(
-  hgTypeMap.get(propertyName), hgConfigProps, 
clusterTopology));
+  propertyName, hgTypeMap.get(propertyName), hgConfigProps, 
clusterTopology));
 }
   }
 }
@@ -857,7 +857,7 @@ public class BlueprintConfigurationProcessor {
 /**
  * Update a property value.
  *
- * @param propertyNamename of
+ * @param propertyNameproperty name
  * @param origValue   original value of property
  * @param properties  all properties
  * @param topologycluster topology
@@ -869,9 +869,20 @@ public class BlueprintConfigurationProcessor {
  MapString, MapString, String 
properties,
  ClusterTopology topology);
 
-CollectionString getRequiredHostGroups(String origValue,
-MapString, MapString, 
String properties,
-ClusterTopology topology);
+/**
+ * Determine the required host groups for the provided property.
+ *
+ * @param propertyNameproperty name
+ * @param origValue   original value of property
+ * @param properties  all properties
+ * @param topologycluster topology
+ *
+ * @return new property value
+ */
+CollectionString getRequiredHostGroups(String propertyName,
+ String origValue,
+ MapString, MapString, String 
properties,
+ ClusterTopology topology);
   }
 
   /**
@@ -1001,16 +1012,17 @@ public class BlueprintConfigurationProcessor {
   }
 }
 
-
-throw new IllegalArgumentException(Unable to update configuration 
property  + ' + propertyName + '+  with topology information.  +
-  Component ' + component + ' is not mapped to any host group 
or is mapped to multiple groups.);
+throw new IllegalArgumentException(
+String.format(Unable to update configuration property '%s' 
with topology information.  +
+Component '%s' is mapped to an invalid number of hosts 
'%s'., propertyName, component, matchingGroupCount));
   }
 }
   }
 }

ambari git commit: AMBARI-11394. Fix persistence layer which allows out of order database writes when provisioning a cluster via a BP

2015-05-26 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 49403a68d - 988248ca0


AMBARI-11394.  Fix persistence layer which allows out of order database
   writes when provisioning a cluster via a BP


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/988248ca
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/988248ca
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/988248ca

Branch: refs/heads/trunk
Commit: 988248ca061c29ec22d9bb657e47884f47e43e5c
Parents: 49403a6
Author: John Speidel jspei...@hortonworks.com
Authored: Tue May 26 15:30:03 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Tue May 26 16:01:39 2015 -0400

--
 .../ambari/server/state/svccomphost/ServiceComponentHostImpl.java   | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/988248ca/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
index dd06eb5..f6aea66 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
@@ -883,7 +883,6 @@ public class ServiceComponentHostImpl implements 
ServiceComponentHost {
   }
 
   @Override
-  @Transactional
   public void handleEvent(ServiceComponentHostEvent event)
   throws InvalidStateTransitionException {
 if (LOG.isDebugEnabled()) {

ambari git commit: AMBARI-11266. Blueprint API: Move 'host_count' and 'host_predicate' up one level

2015-05-20 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk aeb11c93f - 80e247731


AMBARI-11266.  Blueprint API: Move 'host_count' and 'host_predicate' up one 
level


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/80e24773
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/80e24773
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/80e24773

Branch: refs/heads/trunk
Commit: 80e2477317b2f11b3635872014604a258044288c
Parents: aeb11c9
Author: John Speidel jspei...@hortonworks.com
Authored: Wed May 20 09:57:42 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed May 20 16:44:25 2015 -0400

--
 .../internal/ExportBlueprintRequest.java|   2 +-
 .../internal/ProvisionClusterRequest.java   | 191 +--
 .../internal/ScaleClusterRequest.java   |   2 +-
 .../ambari/server/topology/LogicalRequest.java  |   4 +-
 .../server/topology/PersistedStateImpl.java |   4 +-
 .../ambari/server/topology/TopologyRequest.java |  53 -
 .../internal/ProvisionClusterRequestTest.java   |  40 ++--
 .../internal/ScaleClusterRequestTest.java   |  12 +-
 .../topology/ClusterTopologyImplTest.java   |   2 +-
 .../server/topology/TopologyManagerTest.java|   2 +-
 10 files changed, 209 insertions(+), 103 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/80e24773/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
index 1e71094..8b13826 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
@@ -111,7 +111,7 @@ public class ExportBlueprintRequest implements 
TopologyRequest {
   }
 
   @Override
-  public String getCommandDescription() {
+  public String getDescription() {
 return String.format(Export Command For Cluster '%s', clusterName);
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/80e24773/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
index 1a4520e..5dd25fd 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
@@ -15,9 +15,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-//todo: in which package does this belong?  For now it is co-located with 
resource providers because
-//todo: it needs to understand the syntax of the associated resource provider 
request
 package org.apache.ambari.server.controller.internal;
 
 import org.apache.ambari.server.api.predicate.InvalidQueryException;
@@ -38,7 +35,47 @@ import java.util.Map;
 /**
  * Request for provisioning a cluster.
  */
+@SuppressWarnings(unchecked)
 public class ProvisionClusterRequest extends BaseClusterRequest {
+  /**
+   * host groups property name
+   */
+  public static final String HOSTGROUPS_PROPERTY = host_groups;
+
+  /**
+   * host group name property name
+   */
+  public static final String HOSTGROUP_NAME_PROPERTY = name;
+
+  /**
+   * host group host count property name
+   */
+  public static final String HOSTGROUP_HOST_COUNT_PROPERTY = host_count;
+
+  /**
+   * host group host predicate property name
+   */
+  public static final String HOSTGROUP_HOST_PREDICATE_PROPERTY = 
host_predicate;
+
+  /**
+   * host group host fqdn property name
+   */
+  public static final String HOSTGROUP_HOST_FQDN_PROPERTY = fqdn;
+
+  /**
+   * host group hosts property name
+   */
+  public static final String HOSTGROUP_HOSTS_PROPERTY = hosts;
+
+  /**
+   * configurations property name
+   */
+  public static final String CONFIGURATIONS_PROPERTY = configurations;
+
+  /**
+   * default password property name
+   */
+  public static final String DEFAULT_PASSWORD_PROPERTY = default_password;
 
   /**
* configuration factory
@@ -50,7 +87,6 @@ public class ProvisionClusterRequest extends 
BaseClusterRequest {
*/
   private String defaultPassword;
 
-  @SuppressWarnings(unchecked)

ambari git commit: AMBARI-11226. Fix cluster blueprint export mapping of components to host groups

2015-05-18 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 14bb30125 - 6971baebb


AMBARI-11226. Fix cluster blueprint export mapping of components to host groups


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6971baeb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6971baeb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6971baeb

Branch: refs/heads/trunk
Commit: 6971baebb976d1b316e75a74a88c39350c79b956
Parents: 14bb301
Author: John Speidel jspei...@hortonworks.com
Authored: Mon May 18 16:27:30 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon May 18 18:00:49 2015 -0400

--
 .../internal/ExportBlueprintRequest.java|   8 +-
 .../internal/ExportBlueprintRequestTest.java| 166 +++
 2 files changed, 167 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/6971baeb/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
index 9318db9..1e71094 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
@@ -122,16 +122,10 @@ public class ExportBlueprintRequest implements 
TopologyRequest {
 String bpName = exported-blueprint;
 
 CollectionHostGroup hostGroups = new ArrayListHostGroup();
-int count = 1;
 for (ExportedHostGroup exportedHostGroup : exportedHostGroups) {
-
-  //todo: for now can just get from ExportedHostGroup
-  //String cardinality = 
String.valueOf(hostGroupInfo.get(exportedHostGroup.getName()).getHostNames().size());
-  hostGroups.add(new HostGroupImpl(host_group_ + count++, bpName, stack, 
exportedHostGroup.getComponents(),
+  hostGroups.add(new HostGroupImpl(exportedHostGroup.getName(), bpName, 
stack, exportedHostGroup.getComponents(),
   exportedHostGroup.getConfiguration(), 
String.valueOf(exportedHostGroup.getCardinality(;
 }
-
-
 blueprint = new BlueprintImpl(bpName, hostGroups, stack, configuration);
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/6971baeb/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java
new file mode 100644
index 000..1c3310d
--- /dev/null
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distribut
+ * ed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.controller.internal;
+
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.ambari.server.api.util.TreeNode;
+import org.apache.ambari.server.api.util.TreeNodeImpl;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.StackConfigurationResponse;
+import org.apache.ambari.server.controller.StackLevelConfigurationRequest;
+import org.apache.ambari.server.controller.StackServiceRequest;
+import org.apache.ambari.server.controller.StackServiceResponse;
+import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.topology.Blueprint;
+import org.apache.ambari.server.topology.HostGroup;
+import

ambari git commit: AMBARI-11134. Set service states when provisioning cluster via blueprint

2015-05-14 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk f9cefd231 - 9c570b859


AMBARI-11134. Set service states when provisioning cluster via blueprint


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9c570b85
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9c570b85
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9c570b85

Branch: refs/heads/trunk
Commit: 9c570b859470e68427c399a870af426a3062ada9
Parents: f9cefd2
Author: John Speidel jspei...@hortonworks.com
Authored: Thu May 14 10:22:10 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu May 14 11:00:12 2015 -0400

--
 .../ambari/server/topology/AmbariContext.java   |  26 ++-
 .../ambari/server/topology/TopologyManager.java |  21 +-
 .../server/topology/AmbariContextTest.java  | 229 +++
 .../server/topology/TopologyManagerTest.java|   5 -
 4 files changed, 264 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/9c570b85/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index e6c43ef..0de4b00 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -51,6 +51,8 @@ import 
org.apache.ambari.server.controller.internal.HostResourceProvider;
 import org.apache.ambari.server.controller.internal.RequestImpl;
 import org.apache.ambari.server.controller.internal.ServiceResourceProvider;
 import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.controller.predicate.EqualsPredicate;
+import org.apache.ambari.server.controller.spi.Predicate;
 import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
 import org.apache.ambari.server.state.Cluster;
@@ -81,7 +83,7 @@ public class AmbariContext {
   private static HostResourceProvider hostResourceProvider;
   private static ServiceResourceProvider serviceResourceProvider;
   private static ComponentResourceProvider componentResourceProvider;
-  private HostComponentResourceProvider hostComponentResourceProvider;
+  private static HostComponentResourceProvider hostComponentResourceProvider;
 
   private final static Logger LOG = 
LoggerFactory.getLogger(TopologyManager.class);
 
@@ -158,7 +160,6 @@ public class AmbariContext {
   Cluster cluster = getController().getClusters().getCluster(clusterName);
   services.removeAll(cluster.getServices().keySet());
 } catch (AmbariException e) {
-  e.printStackTrace();
   throw new RuntimeException(Failed to persist service and component 
resources:  + e, e);
 }
 SetServiceRequest serviceRequests = new HashSetServiceRequest();
@@ -173,9 +174,28 @@ public class AmbariContext {
   getServiceResourceProvider().createServices(serviceRequests);
   getComponentResourceProvider().createComponents(componentRequests);
 } catch (AmbariException e) {
-  e.printStackTrace();
   throw new RuntimeException(Failed to persist service and component 
resources:  + e, e);
 }
+// set all services state to INSTALLED-STARTED
+// this is required so the user can start failed services at the service 
level
+MapString, Object installProps = new HashMapString, Object();
+
installProps.put(ServiceResourceProvider.SERVICE_SERVICE_STATE_PROPERTY_ID, 
INSTALLED);
+installProps.put(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, 
clusterName);
+MapString, Object startProps = new HashMapString, Object();
+startProps.put(ServiceResourceProvider.SERVICE_SERVICE_STATE_PROPERTY_ID, 
STARTED);
+startProps.put(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, 
clusterName);
+Predicate predicate = new EqualsPredicateString(
+ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, clusterName);
+try {
+  getServiceResourceProvider().updateResources(
+  new RequestImpl(null, Collections.singleton(installProps), null, 
null), predicate);
+
+  getServiceResourceProvider().updateResources(
+  new RequestImpl(null, Collections.singleton(startProps), null, 
null), predicate);
+} catch (Exception e) {
+  // just log as this won't prevent cluster from being provisioned 
correctly
+  LOG.error(Unable to update state of services during cluster provision: 
 + e, e);
+}
   }
 
   public void createAmbariHostResources(String clusterName, String hostName,

ambari git commit: AMBARI-11093. Implement host predicate property validation and fix issue where host name is specified explicitly in scaling request

2015-05-13 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 597951c1f - 43dd0cddf


AMBARI-11093.  Implement host predicate property validation and fix issue where 
host
name is specified explicitly in scaling request


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/43dd0cdd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/43dd0cdd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/43dd0cdd

Branch: refs/heads/trunk
Commit: 43dd0cddf656bd54cda90e182786e304655b4765
Parents: 597951c
Author: John Speidel jspei...@hortonworks.com
Authored: Tue May 12 22:17:01 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed May 13 12:37:50 2015 -0400

--
 .../ambari/server/controller/AmbariServer.java  |   6 +-
 .../controller/internal/BaseClusterRequest.java | 186 +
 .../internal/HostResourceProvider.java  |  11 +-
 .../internal/ProvisionClusterRequest.java   |  98 ++---
 .../internal/ScaleClusterRequest.java   | 155 +---
 .../internal/ProvisionClusterRequestTest.java   | 263 -
 .../internal/ScaleClusterRequestTest.java   | 394 +++
 7 files changed, 987 insertions(+), 126 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/43dd0cdd/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index 77f6d2c..4a30c0d 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -48,14 +48,13 @@ import 
org.apache.ambari.server.configuration.ComponentSSLConfiguration;
 import org.apache.ambari.server.configuration.Configuration;
 import 
org.apache.ambari.server.controller.internal.AbstractControllerResourceProvider;
 import 
org.apache.ambari.server.controller.internal.AmbariPrivilegeResourceProvider;
+import org.apache.ambari.server.controller.internal.BaseClusterRequest;
 import org.apache.ambari.server.controller.internal.BlueprintResourceProvider;
 import 
org.apache.ambari.server.controller.internal.ClusterPrivilegeResourceProvider;
 import org.apache.ambari.server.controller.internal.ClusterResourceProvider;
 import org.apache.ambari.server.controller.internal.HostResourceProvider;
 import org.apache.ambari.server.controller.internal.PermissionResourceProvider;
 import org.apache.ambari.server.controller.internal.PrivilegeResourceProvider;
-import org.apache.ambari.server.controller.internal.ProvisionClusterRequest;
-import org.apache.ambari.server.controller.internal.ScaleClusterRequest;
 import 
org.apache.ambari.server.controller.internal.StackAdvisorResourceProvider;
 import 
org.apache.ambari.server.controller.internal.StackDefinedPropertyProvider;
 import 
org.apache.ambari.server.controller.internal.StackDependencyResourceProvider;
@@ -616,8 +615,7 @@ public class AmbariServer {
 injector.getInstance(TopologyRequestFactoryImpl.class));
 
HostResourceProvider.setTopologyManager(injector.getInstance(TopologyManager.class));
 BlueprintFactory.init(injector.getInstance(BlueprintDAO.class));
-ProvisionClusterRequest.init(injector.getInstance(BlueprintFactory.class));
-ScaleClusterRequest.init(injector.getInstance(BlueprintFactory.class));
+BaseClusterRequest.init(injector.getInstance(BlueprintFactory.class));
 AmbariContext.init(injector.getInstance(HostRoleCommandFactory.class));
 
 PermissionResourceProvider.init(injector.getInstance(PermissionDAO.class));

http://git-wip-us.apache.org/repos/asf/ambari/blob/43dd0cdd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseClusterRequest.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseClusterRequest.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseClusterRequest.java
new file mode 100644
index 000..7f6a634
--- /dev/null
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseClusterRequest.java
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a

ambari git commit: AMBARI-11038. Fix timing issue regarding setting of topology resolved configuration for clusters provisioned via blueprints

2015-05-11 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk d05c9c287 - 66a4bfb26


AMBARI-11038. Fix timing issue regarding setting of topology resolved 
configuration
for clusters provisioned via blueprints


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/66a4bfb2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/66a4bfb2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/66a4bfb2

Branch: refs/heads/trunk
Commit: 66a4bfb26ad5a59d925e56f465bec3346696
Parents: d05c9c2
Author: John Speidel jspei...@hortonworks.com
Authored: Mon May 11 15:38:25 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon May 11 17:37:03 2015 -0400

--
 .../topology/ClusterConfigurationRequest.java   | 14 ++--
 .../ambari/server/topology/LogicalRequest.java  | 14 
 .../ambari/server/topology/TopologyManager.java | 67 ++--
 .../server/topology/TopologyManagerTest.java| 27 +++-
 4 files changed, 50 insertions(+), 72 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/66a4bfb2/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
index a8c2ff3..eb583fd 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -84,7 +84,7 @@ public class ClusterConfigurationRequest {
*/
   public void setConfigurationsOnCluster(ClusterTopology clusterTopology, 
String tag)  {
 //todo: also handle setting of host group scoped configuration which is 
updated by config processor
-ListBlueprintServiceConfigRequest listofConfigRequests = new 
LinkedListBlueprintServiceConfigRequest();
+ListBlueprintServiceConfigRequest configurationRequests = new 
LinkedListBlueprintServiceConfigRequest();
 
 Blueprint blueprint = clusterTopology.getBlueprint();
 Configuration clusterConfiguration = clusterTopology.getConfiguration();
@@ -108,7 +108,7 @@ public class ClusterConfigurationRequest {
 }
   }
 
-  listofConfigRequests.add(blueprintConfigRequest);
+  configurationRequests.add(blueprintConfigRequest);
 }
 
 // since the stack returns cluster-env with each service's config ensure 
that only one
@@ -118,9 +118,9 @@ public class ClusterConfigurationRequest {
 MapString, MapString, String clusterEnvAttributes = 
clusterConfiguration.getFullAttributes().get(cluster-env);
 
 globalConfigRequest.addConfigElement(cluster-env, 
clusterEnvProps,clusterEnvAttributes);
-listofConfigRequests.add(globalConfigRequest);
+configurationRequests.add(globalConfigRequest);
 
-setConfigurationsOnCluster(listofConfigRequests, tag);
+setConfigurationsOnCluster(configurationRequests, tag);
   }
 
   /**
@@ -131,12 +131,12 @@ public class ClusterConfigurationRequest {
*
* This method will also send these requests to the management controller.
*
-   * @param listOfBlueprintConfigRequests a list of requests to send to the 
AmbariManagementController.
+   * @param configurationRequests a list of requests to send to the 
AmbariManagementController.
*/
-  private void setConfigurationsOnCluster(ListBlueprintServiceConfigRequest 
listOfBlueprintConfigRequests,
+  private void setConfigurationsOnCluster(ListBlueprintServiceConfigRequest 
configurationRequests,
   String tag)  {
 // iterate over services to deploy
-for (BlueprintServiceConfigRequest blueprintConfigRequest : 
listOfBlueprintConfigRequests) {
+for (BlueprintServiceConfigRequest blueprintConfigRequest : 
configurationRequests) {
   ClusterRequest clusterRequest = null;
   // iterate over the config types associated with this service
   ListConfigurationRequest requestsPerService = new 
LinkedListConfigurationRequest();

http://git-wip-us.apache.org/repos/asf/ambari/blob/66a4bfb2/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
index 087ad4c..88c791b 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java
@@ -158,20 +158,6

[4/4] ambari git commit: AMBARI-10990. Implement topology manager persistence

2015-05-06 Thread jspeidel

AMBARI-10990.  Implement topology manager persistence


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/807b3c2d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/807b3c2d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/807b3c2d

Branch: refs/heads/trunk
Commit: 807b3c2df7a9a9fe1d728e75cf80c130848d4756
Parents: 8963501
Author: John Speidel jspei...@hortonworks.com
Authored: Thu May 7 01:17:10 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu May 7 01:17:10 2015 -0400

--
 .../query/render/ClusterBlueprintRenderer.java  |   4 +-
 .../ambari/server/controller/AmbariServer.java  |   4 +-
 .../server/controller/ControllerModule.java |   2 -
 .../internal/ExportBlueprintRequest.java|  10 +
 .../internal/HostComponentResourceProvider.java |  34 +-
 .../internal/HostResourceProvider.java  |   4 +-
 .../internal/ProvisionClusterRequest.java   |  14 +-
 .../internal/ScaleClusterRequest.java   |  94 ++-
 .../server/controller/internal/Stack.java   |  30 +-
 .../server/orm/dao/TopologyHostGroupDAO.java|  12 +
 .../server/orm/dao/TopologyHostTaskDAO.java |  73 ++
 .../server/orm/dao/TopologyLogicalTaskDAO.java  |  63 ++
 .../server/orm/entities/BlueprintEntity.java|   3 -
 .../orm/entities/TopologyHostGroupEntity.java   |  28 +-
 .../orm/entities/TopologyHostInfoEntity.java|   6 +-
 .../orm/entities/TopologyHostRequestEntity.java |  11 +-
 .../orm/entities/TopologyHostTaskEntity.java|   6 +
 .../entities/TopologyLogicalRequestEntity.java  |   4 -
 .../ambari/server/topology/AmbariContext.java   | 499 +
 .../ambari/server/topology/BlueprintImpl.java   |   3 +-
 .../topology/ClusterConfigurationRequest.java   |  30 +-
 .../ambari/server/topology/ClusterTopology.java |  30 +
 .../server/topology/ClusterTopologyImpl.java|  33 +-
 .../ambari/server/topology/HostGroupInfo.java   |  15 +-
 .../server/topology/HostOfferResponse.java  |   9 +-
 .../ambari/server/topology/HostRequest.java | 699 +++
 .../ambari/server/topology/LogicalRequest.java  | 153 ++--
 .../server/topology/LogicalRequestFactory.java  |  40 ++
 .../ambari/server/topology/PersistedState.java  |  69 ++
 .../server/topology/PersistedStateImpl.java | 408 +++
 .../topology/PersistedTopologyRequest.java  |  41 ++
 .../ambari/server/topology/TopologyManager.java | 379 --
 .../ambari/server/topology/TopologyRequest.java |   4 +
 .../ambari/server/topology/TopologyTask.java|   5 +
 .../server/upgrade/UpgradeCatalog210.java   |  15 +-
 .../main/resources/Ambari-DDL-MySQL-CREATE.sql  |  17 +-
 .../main/resources/Ambari-DDL-Oracle-CREATE.sql |  17 +-
 .../resources/Ambari-DDL-Postgres-CREATE.sql|  22 +-
 .../Ambari-DDL-Postgres-EMBEDDED-CREATE.sql |  20 +-
 .../BlueprintConfigurationProcessorTest.java| 142 ++--
 .../internal/StageResourceProviderTest.java |   3 +
 .../orm/dao/TopologyLogicalRequestDAOTest.java  |   8 +-
 .../server/orm/dao/TopologyRequestDAOTest.java  |   3 +-
 .../topology/BlueprintValidatorImplTest.java| 181 +
 .../topology/ClusterTopologyImplTest.java   |  41 +-
 .../server/topology/TopologyManagerTest.java| 298 
 46 files changed, 2595 insertions(+), 991 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/807b3c2d/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
index 351f6b4..cfc9bc0 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
@@ -33,6 +33,7 @@ import 
org.apache.ambari.server.controller.internal.ExportBlueprintRequest;
 import org.apache.ambari.server.controller.internal.ResourceImpl;
 import org.apache.ambari.server.controller.internal.Stack;
 import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.topology.AmbariContext;
 import org.apache.ambari.server.topology.ClusterTopology;
 import org.apache.ambari.server.topology.ClusterTopologyImpl;
 import org.apache.ambari.server.topology.Configuration;
@@ -40,7 +41,6 @@ import org.apache.ambari.server.topology.HostGroup;
 import org.apache.ambari.server.topology.HostGroupInfo;
 import org.apache.ambari.server.topology.InvalidTopologyException;
 import org.apache.ambari.server.topology.InvalidTopologyTemplateException;
-import

[1/4] ambari git commit: AMBARI-10990. Implement topology manager persistence

2015-05-06 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 8963501be - 807b3c2df


http://git-wip-us.apache.org/repos/asf/ambari/blob/807b3c2d/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/TopologyLogicalRequestDAOTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/TopologyLogicalRequestDAOTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/TopologyLogicalRequestDAOTest.java
index bba45eb..fcffe95 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/TopologyLogicalRequestDAOTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/TopologyLogicalRequestDAOTest.java
@@ -34,6 +34,7 @@ import 
org.apache.ambari.server.orm.entities.TopologyLogicalTaskEntity;
 import org.apache.ambari.server.orm.entities.TopologyRequestEntity;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.util.Collection;
@@ -96,11 +97,13 @@ public class TopologyLogicalRequestDAOTest {
 hostGroupEntity = hostGroupEntities.iterator().next();
 
 TopologyLogicalRequestEntity logicalRequestEntity = new 
TopologyLogicalRequestEntity();
+logicalRequestEntity.setId(1L);
 logicalRequestEntity.setDescription(description);
 logicalRequestEntity.setTopologyRequestEntity(requestEntity);
 logicalRequestEntity.setTopologyRequestId(requestEntity.getId());
 
 TopologyHostRequestEntity hostRequestEntity = new 
TopologyHostRequestEntity();
+hostGroupEntity.setId(1L);
 hostRequestEntity.setHostName(h1);
 hostRequestEntity.setStageId(1L);
 hostRequestEntity.setTopologyLogicalRequestEntity(logicalRequestEntity);
@@ -127,6 +130,7 @@ public class TopologyLogicalRequestDAOTest {
   }
 
   @Test
+  @Ignore
   public void testFindAll() throws Exception {
 create();
 ListTopologyLogicalRequestEntity logicalRequestEntities = 
logicalRequestDAO.findAll();
@@ -134,7 +138,7 @@ public class TopologyLogicalRequestDAOTest {
 
 TopologyLogicalRequestEntity logicalRequestEntity = 
logicalRequestEntities.iterator().next();
 Assert.assertNotNull(logicalRequestEntity.getTopologyRequestId());
-Assert.assertNotNull(logicalRequestEntity.getId());
+Assert.assertEquals(Long.valueOf(1), logicalRequestEntity.getId());
 Assert.assertEquals(description, logicalRequestEntity.getDescription());
 Assert.assertNotNull(logicalRequestEntity.getTopologyRequestEntity());
 
@@ -142,7 +146,7 @@ public class TopologyLogicalRequestDAOTest {
 Assert.assertEquals(1, hostRequestEntities.size());
 TopologyHostRequestEntity hostRequestEntity = 
hostRequestEntities.iterator().next();
 Assert.assertNotNull(hostRequestEntity.getTopologyHostGroupEntity());
-Assert.assertEquals(hg1, hostRequestEntity.getHostGroupName());
+
Assert.assertEquals(hostRequestEntity.getTopologyHostGroupEntity().getId(), 
hostRequestEntity.getHostGroupId());
 
 CollectionTopologyHostTaskEntity taskEntities = 
hostRequestEntity.getTopologyHostTaskEntities();
 Assert.assertEquals(1, taskEntities.size());

http://git-wip-us.apache.org/repos/asf/ambari/blob/807b3c2d/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/TopologyRequestDAOTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/TopologyRequestDAOTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/TopologyRequestDAOTest.java
index b426c3f..2dd16c7 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/TopologyRequestDAOTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/TopologyRequestDAOTest.java
@@ -99,7 +99,8 @@ public class TopologyRequestDAOTest {
 CollectionTopologyHostInfoEntity infoEntities = 
hostGroupEntity.getTopologyHostInfoEntities();
 Assert.assertEquals(1, infoEntities.size());
 TopologyHostInfoEntity infoEntity = infoEntities.iterator().next();
-Assert.assertEquals(hostGroupEntity.getName(), infoEntity.getGroupName());
+Assert.assertEquals(hg1, hostGroupEntity.getName());
+Assert.assertEquals(hostGroupEntity.getId(), infoEntity.getGroupId());
 Assert.assertEquals(fqdn, infoEntity.getFqdn());
 Assert.assertEquals(12, infoEntity.getHostCount().intValue());
 Assert.assertEquals(predicate, infoEntity.getPredicate());

http://git-wip-us.apache.org/repos/asf/ambari/blob/807b3c2d/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
new file mode 100644
index 000..0b1573b
--- /dev/null
+++

[2/4] ambari git commit: AMBARI-10990. Implement topology manager persistence

2015-05-06 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/807b3c2d/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
index fb4baec..5ea175f 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
@@ -10,8 +10,7 @@
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distribut
- * ed on an AS IS BASIS,
+ * distributed under the License is distributed on an AS IS BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
@@ -23,21 +22,10 @@ import com.google.inject.Singleton;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.HostRoleCommand;
 import org.apache.ambari.server.actionmanager.Request;
-import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.AmbariServer;
-import org.apache.ambari.server.controller.ClusterRequest;
 import org.apache.ambari.server.controller.RequestStatusResponse;
-import org.apache.ambari.server.controller.ServiceComponentRequest;
-import org.apache.ambari.server.controller.ServiceRequest;
-import org.apache.ambari.server.controller.internal.ComponentResourceProvider;
-import org.apache.ambari.server.controller.internal.ServiceResourceProvider;
 import org.apache.ambari.server.controller.internal.Stack;
-import org.apache.ambari.server.controller.spi.Resource;
-import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
 import org.apache.ambari.server.orm.dao.HostRoleCommandStatusSummaryDTO;
 import org.apache.ambari.server.orm.entities.StageEntity;
-import org.apache.ambari.server.state.Cluster;
-import org.apache.ambari.server.state.SecurityType;
 import org.apache.ambari.server.state.host.HostImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -51,12 +39,8 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.Executors;
 
 /**
  * Manages all cluster provisioning actions on the cluster topology.
@@ -65,77 +49,85 @@ import java.util.concurrent.atomic.AtomicLong;
 @Singleton
 public class TopologyManager {
 
+  public static final String INITIAL_CONFIG_TAG = INITIAL;
+  public static final String TOPOLOGY_RESOLVED_TAG = TOPOLOGY_RESOLVED;
+
+  private PersistedState persistedState;
+  //private ExecutorService executor = getExecutorService();
+  private ExecutorService executor = Executors.newSingleThreadExecutor();
+  private CollectionString hostsToIgnore = new HashSetString();
   private final ListHostImpl availableHosts = new LinkedListHostImpl();
   private final MapString, LogicalRequest reservedHosts = new 
HashMapString, LogicalRequest();
   private final MapLong, LogicalRequest allRequests = new HashMapLong, 
LogicalRequest();
   // priority is given to oldest outstanding requests
   private final CollectionLogicalRequest outstandingRequests = new 
ArrayListLogicalRequest();
+  //todo: currently only support a single cluster
   private MapString, ClusterTopology clusterTopologyMap = new 
HashMapString, ClusterTopology();
-  private final MapTopologyTask.Type, SetTopologyTask pendingTasks = new 
HashMapTopologyTask.Type, SetTopologyTask();
-
-  //todo: proper wait/notify mechanism
-  private final Object configurationFlagLock = new Object();
-  private boolean configureComplete = false;
+  //private final MapTopologyTask.Type, SetTopologyTask pendingTasks = new 
HashMapTopologyTask.Type, SetTopologyTask();
 
-  private AmbariManagementController controller;
-  ExecutorService executor;
-  //todo: task id's.  Use existing mechanism for getting next task id sequence
-  private final static AtomicLong nextTaskId = new AtomicLong(1);
-  private final Object serviceResourceLock = new Object();
+  //todo: inject
+  private static LogicalRequestFactory logicalRequestFactory = new 
LogicalRequestFactory();
+  private static AmbariContext ambariContext = new AmbariContext();
 
-  protected final static Logger LOG = 
LoggerFactory.getLogger(TopologyManager.class);
+  private final Object initializationLock = new Object();
+  private boolean isInitialized;
 
+

ambari git commit: AMBARI-10811. Fix issues with config topology update in certain topologies which include HA and implicit MYSQL_SERVER component inclusion

2015-04-28 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 132cf31bc - 43ae930b0


AMBARI-10811. Fix issues with config topology update in certain topologies
which include HA and implicit MYSQL_SERVER component inclusion


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/43ae930b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/43ae930b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/43ae930b

Branch: refs/heads/trunk
Commit: 43ae930b0188f22913dcc67ec5c8ea168ff4deae
Parents: 132cf31
Author: jspeidel jspei...@hortonworks.com
Authored: Tue Apr 28 20:43:59 2015 -0400
Committer: jspeidel jspei...@hortonworks.com
Committed: Tue Apr 28 20:45:13 2015 -0400

--
 .../BlueprintConfigurationProcessor.java| 96 
 .../topology/ClusterConfigurationRequest.java   |  7 +-
 .../ambari/server/topology/TopologyManager.java | 25 +++--
 .../BlueprintConfigurationProcessorTest.java| 48 +-
 4 files changed, 142 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/43ae930b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 95e9807..7938cc1 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -636,12 +636,12 @@ public class BlueprintConfigurationProcessor {
  *
  * @return new property value
  */
-public String updateForClusterCreate(String propertyName,
+String updateForClusterCreate(String propertyName,
  String origValue,
  MapString, MapString, String 
properties,
  ClusterTopology topology);
 
-public CollectionString getRequiredHostGroups(String origValue,
+CollectionString getRequiredHostGroups(String origValue,
 MapString, MapString, 
String properties,
 ClusterTopology topology);
   }
@@ -792,32 +792,79 @@ public class BlueprintConfigurationProcessor {
 return Collections.singleton(hostGroupName);
   } else {
 CollectionString matchingGroups = 
topology.getHostGroupsForComponent(component);
-if (matchingGroups.size() == 1) {
+int matchingGroupCount = matchingGroups.size();
+if (matchingGroupCount == 1) {
   return Collections.singleton(matchingGroups.iterator().next());
 } else {
-  if (topology.isNameNodeHAEnabled()  isComponentNameNode()  
(matchingGroups.size() == 2)) {
-// if this is the defaultFS property, it should reflect the 
nameservice name,
-// rather than a hostname (used in non-HA scenarios)
-if (properties.containsKey(core-site)  
properties.get(core-site).get(fs.defaultFS).equals(origValue)) {
-  return Collections.emptySet();
+  Cardinality cardinality = 
topology.getBlueprint().getStack().getCardinality(component);
+  // if no matching host groups are found for a component whose 
configuration
+  // is handled by this updater, return an empty set
+  if (matchingGroupCount == 0  cardinality.isValidCount(0)) {
+return Collections.emptySet();
+  } else {
+//todo: shouldn't have all of these hard coded HA rules here
+if (topology.isNameNodeHAEnabled()  isComponentNameNode()  
(matchingGroupCount == 2)) {
+  // if this is the defaultFS property, it should reflect the 
nameservice name,
+  // rather than a hostname (used in non-HA scenarios)
+  if (properties.containsKey(core-site)  
properties.get(core-site).get(fs.defaultFS).equals(origValue)) {
+return Collections.emptySet();
+  }
+
+  if (properties.containsKey(hbase-site)  
properties.get(hbase-site).get(hbase.rootdir).equals(origValue)) {
+// hbase-site's reference to the namenode is handled 
differently in HA mode, since the
+// reference must point to the logical nameservice, rather 
than an individual namenode
+return Collections.emptySet();
+  }
+
+  if (properties.containsKey(accumulo-site)  
properties.get(accumulo-site).get

ambari git commit: AMBARI-10770. Fix blueprint configuration validation regarding password properties.

2015-04-27 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 3a42dfa77 - fa6194265


AMBARI-10770.  Fix blueprint configuration validation regarding password 
properties.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fa619426
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fa619426
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fa619426

Branch: refs/heads/trunk
Commit: fa6194265f8ae0ba98806b50f88497bf659cdead
Parents: 3a42dfa
Author: John Speidel jspei...@hortonworks.com
Authored: Mon Apr 27 12:46:40 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Apr 27 15:09:21 2015 -0400

--
 .../server/controller/internal/Stack.java   |  35 +++---
 .../server/controller/internal/StackTest.java   | 117 +++
 2 files changed, 137 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/fa619426/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
index 82d03fd..7167449 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
@@ -634,25 +634,23 @@ public class Stack {
 
 // shouldn't have any required properties in stack level configuration
 for (StackConfigurationResponse config : serviceConfigs) {
-  String type = config.getType();
-  //strip .xml from type
-  if (type.endsWith(.xml)) {
-type = type.substring(0, type.length() - 4);
-  }
+  ConfigProperty configProperty = new ConfigProperty(config);
+  String type = configProperty.getType();
+
   MapString, ConfigProperty mapTypeConfig = mapServiceConfig.get(type);
   if (mapTypeConfig == null) {
 mapTypeConfig = new HashMapString, ConfigProperty();
 mapServiceConfig.put(type, mapTypeConfig);
   }
-  ConfigProperty property = new ConfigProperty(config);
-  mapTypeConfig.put(config.getPropertyName(), property);
+
+  mapTypeConfig.put(config.getPropertyName(), configProperty);
   if (config.isRequired()) {
 MapString, ConfigProperty requiredTypeConfig = 
mapRequiredServiceConfig.get(type);
 if (requiredTypeConfig == null) {
   requiredTypeConfig = new HashMapString, ConfigProperty();
   mapRequiredServiceConfig.put(type, requiredTypeConfig);
 }
-requiredTypeConfig.put(config.getPropertyName(), property);
+requiredTypeConfig.put(config.getPropertyName(), configProperty);
   }
 }
   }
@@ -663,18 +661,17 @@ public class Stack {
 Collections.singleton(new StackLevelConfigurationRequest(name, 
version, null)));
 
 for (StackConfigurationResponse config : stackLevelConfigs) {
-  String type = config.getType();
-  //strip .xml from type
-  if (type.endsWith(.xml)) {
-type = type.substring(0, type.length() - 4);
-  }
+  ConfigProperty configProperty = new ConfigProperty(config);
+  String type = configProperty.getType();
+
   MapString, ConfigProperty mapTypeConfig = 
stackConfigurations.get(type);
   if (mapTypeConfig == null) {
 mapTypeConfig = new HashMapString, ConfigProperty();
 stackConfigurations.put(type, mapTypeConfig);
   }
+
   mapTypeConfig.put(config.getPropertyName(),
-  new ConfigProperty(config));
+  configProperty);
 }
   }
 
@@ -710,7 +707,7 @@ public class Stack {
   this.value = config.getPropertyValue();
   this.attributes = config.getPropertyAttributes();
   this.propertyTypes = config.getPropertyType();
-  this.type = config.getType();
+  this.type = normalizeType(config.getType());
 }
 
 public ConfigProperty(String type, String name, String value) {
@@ -750,5 +747,13 @@ public class Stack {
 public void setAttributes(MapString, String attributes) {
   this.attributes = attributes;
 }
+
+private String normalizeType(String type) {
+  //strip .xml from type
+  if (type.endsWith(.xml)) {
+type = type.substring(0, type.length() - 4);
+  }
+  return type;
+}
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa619426/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackTest.java

[12/13] ambari git commit: AMBARI-10750. Initial merge of advanced api provisioning work.

2015-04-26 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
deleted file mode 100644
index 73ea1a5..000
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
+++ /dev/null
@@ -1,771 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * License); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.controller.internal;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.StackAccessException;
-import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
-import org.apache.ambari.server.controller.spi.Resource;
-import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException;
-import org.apache.ambari.server.controller.spi.ResourceProvider;
-import org.apache.ambari.server.controller.spi.SystemException;
-import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
-import org.apache.ambari.server.orm.dao.BlueprintDAO;
-import org.apache.ambari.server.orm.dao.StackDAO;
-import org.apache.ambari.server.orm.entities.BlueprintConfigEntity;
-import org.apache.ambari.server.orm.entities.BlueprintEntity;
-import org.apache.ambari.server.orm.entities.HostGroupComponentEntity;
-import org.apache.ambari.server.orm.entities.HostGroupConfigEntity;
-import org.apache.ambari.server.orm.entities.HostGroupEntity;
-import org.apache.ambari.server.state.AutoDeployInfo;
-import org.apache.ambari.server.state.ConfigHelper;
-import org.apache.ambari.server.state.DependencyInfo;
-
-import com.google.gson.Gson;
-
-/**
- * Base blueprint processing resource provider.
- */
-//todo: this class needs to be refactored to a ClusterTopology class which
-//todo: has hostgroup, stack and configuration state specific to a deployment.
-public abstract class BaseBlueprintProcessor extends 
AbstractControllerResourceProvider {
-
-  /**
-   * Data access object used to obtain blueprint entities.
-   */
-  protected static BlueprintDAO blueprintDAO;
-
-  /**
-   * Data access object used to lookup value stacks parsed from the resources.
-   */
-  protected static StackDAO stackDAO;
-
-  /**
-   * Stack related information.
-   */
-  protected static AmbariMetaInfo stackInfo;
-
-  protected static ConfigHelper configHelper;
-
-
-  protected BaseBlueprintProcessor(SetString propertyIds,
-   MapResource.Type, String keyPropertyIds,
-   AmbariManagementController 
managementController) {
-
-super(propertyIds, keyPropertyIds, managementController);
-  }
-
-  /**
-   * Get host groups which contain a component.
-   *
-   * @param component   component name
-   * @param hostGroups  collection of host groups to check
-   *
-   * @return collection of host groups which contain the specified component
-   */
-  protected CollectionHostGroupImpl getHostGroupsForComponent(String 
component, CollectionHostGroupImpl hostGroups) {
-CollectionHostGroupImpl resultGroups = new HashSetHostGroupImpl();
-for (HostGroupImpl group : hostGroups ) {
-  if (group.getComponents().contains(component)) {
-resultGroups.add(group);
-  }
-}
-return resultGroups;
-  }
-
-  /**
-   * Parse blueprint host groups.
-   *
-   * @param blueprint  associated blueprint
-   * @param stack  associated stack
-   *
-   * @return map of host group name to host group
-   */
-  protected MapString, HostGroupImpl 
parseBlueprintHostGroups(BlueprintEntity blueprint, Stack stack) {
-MapString, HostGroupImpl mapHostGroups = new HashMapString,

[01/13] ambari git commit: AMBARI-10750. Initial merge of advanced api provisioning work.

2015-04-26 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 6f67c4f66 - c9f0dd0b8


http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
new file mode 100644
index 000..9d4163a
--- /dev/null
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
@@ -0,0 +1,870 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import org.apache.ambari.server.controller.internal.Stack;
+import org.junit.Test;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.fail;
+
+/**
+ * Blueprint unit tests.
+ */
+public class BlueprintImplTest {
+
+  private static final MapString, MapString, MapString, String 
EMPTY_ATTRIBUTES =
+  new HashMapString, MapString, MapString, String();
+
+  private static final MapString, MapString, String EMPTY_PROPERTIES =
+  new HashMapString, MapString, String();
+
+  private static final Configuration EMPTY_CONFIGURATION = new 
Configuration(EMPTY_PROPERTIES, EMPTY_ATTRIBUTES);
+
+
+
+  @Test
+  public void testValidateConfigurations__basic_positive() throws Exception {
+
+Stack stack = createNiceMock(Stack.class);
+
+HostGroup group1 = createNiceMock(HostGroup.class);
+HostGroup group2 = createNiceMock(HostGroup.class);
+CollectionHostGroup hostGroups = new HashSetHostGroup();
+hostGroups.add(group1);
+hostGroups.add(group2);
+
+SetString group1Components = new HashSetString();
+group1Components.add(c1);
+group1Components.add(c2);
+
+SetString group2Components = new HashSetString();
+group2Components.add(c1);
+group2Components.add(c3);
+
+CollectionStack.ConfigProperty requiredHDFSProperties = new 
HashSetStack.ConfigProperty();
+requiredHDFSProperties.add(new Stack.ConfigProperty(hdfs-site, foo, 
null));
+requiredHDFSProperties.add(new Stack.ConfigProperty(hdfs-site, bar, 
null));
+requiredHDFSProperties.add(new Stack.ConfigProperty(hdfs-site, 
some_password, null));
+
+requiredHDFSProperties.add(new Stack.ConfigProperty(category1, prop1, 
null));
+
+CollectionStack.ConfigProperty requiredService2Properties = new 
HashSetStack.ConfigProperty();
+requiredService2Properties.add(new Stack.ConfigProperty(category2, 
prop2, null));
+
+expect(stack.getServiceForComponent(c1)).andReturn(HDFS).atLeastOnce();
+expect(stack.getServiceForComponent(c2)).andReturn(HDFS).atLeastOnce();
+
expect(stack.getServiceForComponent(c3)).andReturn(SERVICE2).atLeastOnce();
+
+
expect(stack.getRequiredConfigurationProperties(HDFS)).andReturn(requiredHDFSProperties).atLeastOnce();
+
expect(stack.getRequiredConfigurationProperties(SERVICE2)).andReturn(requiredService2Properties).atLeastOnce();
+
+expect(stack.isPasswordProperty(HDFS, hdfs-site, 
foo)).andReturn(false).atLeastOnce();
+expect(stack.isPasswordProperty(HDFS, hdfs-site, 
bar)).andReturn(false).atLeastOnce();
+expect(stack.isPasswordProperty(HDFS, hdfs-site, 
some_password)).andReturn(true).atLeastOnce();
+expect(stack.isPasswordProperty(HDFS, category1, 
prop1)).andReturn(false).atLeastOnce();
+expect(stack.isPasswordProperty(SERVICE2, category2, 
prop2)).andReturn(false).atLeastOnce();
+
+
expect(group1.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce();
+expect(group1.getName()).andReturn(group1).anyTimes();
+expect(group1.getComponents()).andReturn(group1Components).atLeastOnce();
+
+
expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce();
+expect(group2.getName()).andReturn(group2).anyTimes();
+

[04/13] ambari git commit: AMBARI-10750. Initial merge of advanced api provisioning work.

2015-04-26 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
index 1fa4a4d..118a7be 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
@@ -18,7 +18,7 @@
 
 package org.apache.ambari.server.controller.internal;
 
-import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.createStrictMock;
@@ -31,7 +31,6 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -41,21 +40,12 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.StackConfigurationRequest;
-import org.apache.ambari.server.controller.StackConfigurationResponse;
-import org.apache.ambari.server.controller.StackLevelConfigurationRequest;
-import org.apache.ambari.server.controller.StackServiceComponentRequest;
-import org.apache.ambari.server.controller.StackServiceComponentResponse;
-import org.apache.ambari.server.controller.StackServiceRequest;
-import org.apache.ambari.server.controller.StackServiceResponse;
 import 
org.apache.ambari.server.controller.internal.BlueprintResourceProvider.BlueprintConfigPopulationStrategy;
 import 
org.apache.ambari.server.controller.internal.BlueprintResourceProvider.BlueprintConfigPopulationStrategyV1;
 import 
org.apache.ambari.server.controller.internal.BlueprintResourceProvider.BlueprintConfigPopulationStrategyV2;
@@ -65,7 +55,6 @@ import 
org.apache.ambari.server.controller.spi.NoSuchResourceException;
 import org.apache.ambari.server.controller.spi.Predicate;
 import org.apache.ambari.server.controller.spi.Request;
 import org.apache.ambari.server.controller.spi.Resource;
-import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException;
 import org.apache.ambari.server.controller.spi.ResourceProvider;
 import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
@@ -79,13 +68,10 @@ import 
org.apache.ambari.server.orm.entities.HostGroupComponentEntity;
 import org.apache.ambari.server.orm.entities.HostGroupConfigEntity;
 import org.apache.ambari.server.orm.entities.HostGroupEntity;
 import org.apache.ambari.server.orm.entities.StackEntity;
-import org.apache.ambari.server.state.AutoDeployInfo;
-import org.apache.ambari.server.state.ComponentInfo;
-import org.apache.ambari.server.state.DependencyInfo;
-import org.apache.ambari.server.state.ServiceInfo;
 import org.apache.ambari.server.utils.StageUtils;
-import org.easymock.Capture;
-import org.easymock.EasyMock;
+import org.apache.ambari.server.topology.Blueprint;
+import org.apache.ambari.server.topology.BlueprintFactory;
+import org.apache.ambari.server.topology.InvalidTopologyException;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
@@ -105,75 +91,79 @@ public class BlueprintResourceProviderTest {
   @Rule
   public ExpectedException expectedException = ExpectedException.none();
 
-  private final static BlueprintResourceProvider provider = createProvider();
   private final static BlueprintDAO dao = createStrictMock(BlueprintDAO.class);
   private final static StackDAO stackDAO = createNiceMock(StackDAO.class);
-  private final static Gson gson = new Gson();
+  private final static BlueprintEntity entity = 
createStrictMock(BlueprintEntity.class);
+  private final static Blueprint blueprint = createMock(Blueprint.class);
   private final static AmbariMetaInfo metaInfo = 
createMock(AmbariMetaInfo.class);
+  private final static BlueprintFactory blueprintFactory = 
createMock(BlueprintFactory.class);
+  private final static BlueprintResourceProvider provider = createProvider();
+  private final static Gson gson = new Gson();
+

[06/13] ambari git commit: AMBARI-10750. Initial merge of advanced api provisioning work.

2015-04-26 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequestFactoryImpl.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequestFactoryImpl.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequestFactoryImpl.java
new file mode 100644
index 000..7c6a8ce
--- /dev/null
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequestFactoryImpl.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distribut
+ * ed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import org.apache.ambari.server.controller.internal.ProvisionClusterRequest;
+
+import java.util.Map;
+
+/**
+ * Factory for creating topology requests.
+ */
+public class TopologyRequestFactoryImpl implements TopologyRequestFactory {
+  @Override
+  public TopologyRequest createProvisionClusterRequest(MapString, Object 
properties) throws InvalidTopologyTemplateException {
+return new ProvisionClusterRequest(properties);
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyTask.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyTask.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyTask.java
new file mode 100644
index 000..99783dd
--- /dev/null
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyTask.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distribut
+ * ed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+/**
+ * Task which is executed by the TopologyManager.
+ */
+public interface TopologyTask extends Runnable {
+  /**
+   * Task type.
+   */
+  public enum Type {
+RESOURCE_CREATION,
+CONFIGURE,
+INSTALL,
+START
+  }
+
+  /**
+   * Get the task type.
+   *
+   * @return the type of task
+   */
+  public Type getType();
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyValidator.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyValidator.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyValidator.java
new file mode 100644
index 000..146b424
--- /dev/null
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyValidator.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the

[05/13] ambari git commit: AMBARI-10750. Initial merge of advanced api provisioning work.

2015-04-26 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 6c26b06..34b239b 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -23,21 +23,33 @@ import static junit.framework.Assert.assertFalse;
 import static junit.framework.Assert.assertNotNull;
 import static junit.framework.Assert.assertTrue;
 import static junit.framework.Assert.fail;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.isA;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
 
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.StackServiceResponse;
-import org.easymock.EasyMockSupport;
+import org.apache.ambari.server.state.ServiceInfo;
+import org.apache.ambari.server.topology.Blueprint;
+import org.apache.ambari.server.topology.Cardinality;
+import org.apache.ambari.server.topology.ClusterTopology;
+import org.apache.ambari.server.topology.ClusterTopologyImpl;
+import org.apache.ambari.server.topology.Configuration;
+import org.apache.ambari.server.topology.HostGroup;
+import org.apache.ambari.server.topology.HostGroupImpl;
+import org.apache.ambari.server.topology.HostGroupInfo;
+import org.apache.ambari.server.topology.InvalidTopologyException;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 /**
@@ -45,100 +57,207 @@ import org.junit.Test;
  */
 public class BlueprintConfigurationProcessorTest {
 
+  private static final Configuration EMPTY_CONFIG = new 
Configuration(Collections.String, MapString, StringemptyMap(),
+  Collections.String, MapString, MapString, StringemptyMap());
+
+  private final MapString, CollectionString serviceComponents = new 
HashMapString, CollectionString();
+
+  private final Blueprint bp = createNiceMock(Blueprint.class);
+  //private final AmbariMetaInfo metaInfo = 
createNiceMock(AmbariMetaInfo.class);
+  private final ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class);
+  private final Stack stack = createNiceMock(Stack.class);
+
+  @Before
+  public void init() throws Exception {
+expect(bp.getStack()).andReturn(stack).anyTimes();
+expect(bp.getName()).andReturn(test-bp).anyTimes();
+
+expect(stack.getName()).andReturn(testStack).anyTimes();
+expect(stack.getVersion()).andReturn(1).anyTimes();
+// return false for all components since for this test we don't care about 
the value
+expect(stack.isMasterComponent((String) 
anyObject())).andReturn(false).anyTimes();
+
+expect(serviceInfo.getRequiredProperties()).andReturn(
+Collections.String, 
org.apache.ambari.server.state.PropertyInfoemptyMap()).anyTimes();
+
expect(serviceInfo.getRequiredServices()).andReturn(Collections.StringemptyList()).anyTimes();
+
+CollectionString hdfsComponents = new HashSetString();
+hdfsComponents.add(NAMENODE);
+hdfsComponents.add(SECONDARY_NAMENODE);
+hdfsComponents.add(DATANODE);
+hdfsComponents.add(HDFS_CLIENT);
+serviceComponents.put(HDFS, hdfsComponents);
+
+CollectionString yarnComponents = new HashSetString();
+yarnComponents.add(RESOURCEMANAGER);
+yarnComponents.add(NODEMANAGER);
+yarnComponents.add(YARN_CLIENT);
+yarnComponents.add(APP_TIMELINE_SERVER);
+serviceComponents.put(YARN, yarnComponents);
+
+CollectionString mrComponents = new HashSetString();
+mrComponents.add(MAPREDUCE2_CLIENT);
+mrComponents.add(HISTORY_SERVER);
+serviceComponents.put(MAPREDUCE2, mrComponents);
+
+CollectionString zkComponents = new HashSetString();
+zkComponents.add(ZOOKEEPER_SERVER);
+zkComponents.add(ZOOKEEPER_CLIENT);
+serviceComponents.put(ZOOKEEPER, zkComponents);
+
+CollectionString hiveComponents = new HashSetString();
+hiveComponents.add(MYSQL_SERVER);
+hiveComponents.add(HIVE_METASTORE);
+serviceComponents.put(HIVE, hiveComponents);
+
+CollectionString falconComponents = new HashSetString();
+

[02/13] ambari git commit: AMBARI-10750. Initial merge of advanced api provisioning work.

2015-04-26 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostComponentResourceProviderTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostComponentResourceProviderTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostComponentResourceProviderTest.java
index 8d1d20d..03ed000 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostComponentResourceProviderTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostComponentResourceProviderTest.java
@@ -332,131 +332,13 @@ public class HostComponentResourceProviderTest {
 Resource responseResource = requestStatus.getRequestResource();
 assertEquals(response msg, 
responseResource.getPropertyValue(PropertyHelper.getPropertyId(Requests, 
message)));
 assertEquals(1000L, 
responseResource.getPropertyValue(PropertyHelper.getPropertyId(Requests, 
id)));
-assertEquals(InProgress, 
responseResource.getPropertyValue(PropertyHelper.getPropertyId(Requests, 
status)));
+assertEquals(Accepted, 
responseResource.getPropertyValue(PropertyHelper.getPropertyId(Requests, 
status)));
 assertTrue(requestStatus.getAssociatedResources().isEmpty());
 
 // verify
 verify(managementController, response, resourceProviderFactory, 
stageContainer);
   }
 
-  @Test
-  public void testInstallAndStart() throws Exception {
-Resource.Type type = Resource.Type.HostComponent;
-
-AmbariManagementController managementController = 
createMock(AmbariManagementController.class);
-RequestStatusResponse response = 
createNiceMock(RequestStatusResponse.class);
-ResourceProviderFactory resourceProviderFactory = 
createNiceMock(ResourceProviderFactory.class);
-Injector injector = createNiceMock(Injector.class);
-Clusters clusters = createNiceMock(Clusters.class);
-Cluster cluster = createNiceMock(Cluster.class);
-Service service = createNiceMock(Service.class);
-ServiceComponent component = createNiceMock(ServiceComponent.class);
-ServiceComponent clientComponent = createNiceMock(ServiceComponent.class);
-ServiceComponentHost componentHost = 
createNiceMock(ServiceComponentHost.class);
-ServiceComponentHost clientComponentHost = 
createNiceMock(ServiceComponentHost.class);
-RequestStageContainer stageContainer = 
createNiceMock(RequestStageContainer.class);
-MaintenanceStateHelper maintenanceStateHelper = 
createNiceMock(MaintenanceStateHelper.class);
-HostVersionDAO hostVersionDAO = createMock(HostVersionDAO.class);
-
-CollectionString hosts = new HashSetString();
-hosts.add(Host100);
-
-MapString, String mapRequestProps = new HashMapString, String();
-mapRequestProps.put(context, Install and start components on added 
hosts);
-
-SetServiceComponentHostResponse nameResponse = new 
HashSetServiceComponentHostResponse();
-nameResponse.add(new ServiceComponentHostResponse(
-Cluster102, Service100, Component100, Host100, INIT, , 
INIT, , null));
-nameResponse.add(new ServiceComponentHostResponse(
-Cluster102, Service100, some-client, Host100, INIT, , 
INIT, , null));
-SetServiceComponentHostResponse nameResponse2 = new 
HashSetServiceComponentHostResponse();
-nameResponse2.add(new ServiceComponentHostResponse(
-Cluster102, Service100, Component100, Host100, INIT, , 
INSTALLED, , null));
-nameResponse2.add(new ServiceComponentHostResponse(
-Cluster102, Service100, some-client, Host100, INIT, , 
INSTALLED, , null));
-
-
-// set expectations
-expect(managementController.getClusters()).andReturn(clusters).anyTimes();
-expect(managementController.findServiceName(cluster, 
Component100)).andReturn(Service100).anyTimes();
-expect(managementController.findServiceName(cluster, 
some-client)).andReturn(Service100).anyTimes();
-expect(clusters.getCluster(Cluster102)).andReturn(cluster).anyTimes();
-expect(cluster.getService(Service100)).andReturn(service).anyTimes();
-
expect(service.getServiceComponent(Component100)).andReturn(component).anyTimes();
-
expect(service.getServiceComponent(some-client)).andReturn(clientComponent).anyTimes();
-
expect(component.getServiceComponentHost(Host100)).andReturn(componentHost).anyTimes();
-expect(component.getName()).andReturn(Component100).anyTimes();
-
expect(clientComponent.getServiceComponentHost(Host100)).andReturn(clientComponentHost).anyTimes();
-expect(clientComponent.getName()).andReturn(some-client).anyTimes();
-expect(clientComponent.isClientComponent()).andReturn(true).anyTimes();
-// actual state is always INIT until stages actually execute
-expect(componentHost.getState()).andReturn(State.INIT).anyTimes();
-

[10/13] ambari git commit: AMBARI-10750. Initial merge of advanced api provisioning work.

2015-04-26 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
new file mode 100644
index 000..e4acea2
--- /dev/null
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
@@ -0,0 +1,531 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distribut
+ * ed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.controller.internal;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.api.util.TreeNode;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.AmbariServer;
+import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.apache.ambari.server.state.DesiredConfig;
+import org.apache.ambari.server.state.HostConfig;
+import org.apache.ambari.server.topology.Blueprint;
+import org.apache.ambari.server.topology.BlueprintImpl;
+import org.apache.ambari.server.topology.Configuration;
+import org.apache.ambari.server.topology.HostGroup;
+import org.apache.ambari.server.topology.HostGroupImpl;
+import org.apache.ambari.server.topology.HostGroupInfo;
+import org.apache.ambari.server.topology.InvalidTopologyTemplateException;
+import org.apache.ambari.server.topology.TopologyRequest;
+import org.apache.ambari.server.topology.TopologyValidator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Request to export a blueprint from an existing cluster.
+ */
+public class ExportBlueprintRequest implements TopologyRequest {
+
+  private final static Logger LOG = 
LoggerFactory.getLogger(ExportBlueprintRequest.class);
+  private static AmbariManagementController controller = 
AmbariServer.getController();
+
+  private String clusterName;
+  private Blueprint blueprint;
+  private Configuration configuration;
+  //todo: Should this map be represented by a new class?
+  private MapString, HostGroupInfo hostGroupInfo = new HashMapString, 
HostGroupInfo();
+
+
+  public ExportBlueprintRequest(TreeNodeResource clusterNode) throws 
InvalidTopologyTemplateException {
+Resource clusterResource = clusterNode.getObject();
+clusterName = String.valueOf(clusterResource.getPropertyValue(
+ClusterResourceProvider.CLUSTER_NAME_PROPERTY_ID));
+
+
+createConfiguration(clusterNode);
+//todo: should be parsing Configuration from the beginning
+//createConfiguration(configurations);
+
+CollectionExportedHostGroup exportedHostGroups = 
processHostGroups(clusterNode.getChild(hosts));
+createHostGroupInfo(exportedHostGroups);
+createBlueprint(exportedHostGroups, parseStack(clusterResource));
+  }
+
+  @Override
+  public String getClusterName() {
+return clusterName;
+  }
+
+  @Override
+  public Blueprint getBlueprint() {
+return blueprint;
+  }
+
+  @Override
+  public Configuration getConfiguration() {
+return configuration;
+  }
+
+  @Override
+  public MapString, HostGroupInfo getHostGroupInfo() {
+return hostGroupInfo;
+  }
+
+  @Override
+  public ListTopologyValidator getTopologyValidators() {
+return Collections.emptyList();
+  }
+
+  // - private instance methods --
+
+
+  private void createBlueprint(CollectionExportedHostGroup 
exportedHostGroups, Stack stack) {
+String bpName = exported-blueprint;
+
+CollectionHostGroup hostGroups = new ArrayListHostGroup();
+int count = 1;
+for (ExportedHostGroup exportedHostGroup : exportedHostGroups) {
+
+  //todo: for now can just get

[13/13] ambari git commit: AMBARI-10750. Initial merge of advanced api provisioning work.

2015-04-26 Thread jspeidel

AMBARI-10750. Initial merge of advanced api provisioning work.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c9f0dd0b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c9f0dd0b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c9f0dd0b

Branch: refs/heads/trunk
Commit: c9f0dd0b8206956a7a71fa0adca64160710d2910
Parents: 6f67c4f
Author: John Speidel jspei...@hortonworks.com
Authored: Mon Apr 27 01:48:16 2015 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Apr 27 01:51:01 2015 -0400

--
 .../server/actionmanager/ActionManager.java |   30 +-
 .../server/actionmanager/HostRoleCommand.java   |   27 +-
 .../query/render/ClusterBlueprintRenderer.java  |  601 +-
 .../controller/AmbariActionExecutionHelper.java |2 +-
 .../AmbariCustomCommandExecutionHelper.java |6 +-
 .../AmbariManagementControllerImpl.java |   18 +-
 .../ambari/server/controller/AmbariServer.java  |   25 +-
 .../server/controller/ControllerModule.java |4 +
 .../server/controller/KerberosHelper.java   |6 +-
 .../server/controller/ShortTaskStatus.java  |9 +
 .../internal/AbstractResourceProvider.java  |8 +-
 .../internal/BaseBlueprintProcessor.java|  771 ---
 .../BlueprintConfigurationProcessor.java|  670 ++-
 .../internal/BlueprintResourceProvider.java |  279 +-
 .../server/controller/internal/Cardinality.java |   86 -
 .../internal/ClientConfigResourceProvider.java  |2 +-
 .../internal/ClusterResourceProvider.java   |  907 +--
 .../internal/ComponentResourceProvider.java |5 +-
 .../ConfigurationTopologyException.java |   39 +
 .../internal/ExportBlueprintRequest.java|  531 ++
 .../internal/HostComponentResourceProvider.java |   75 +-
 .../server/controller/internal/HostGroup.java   |   56 -
 .../internal/HostResourceProvider.java  |  237 +-
 .../internal/ProvisionClusterRequest.java   |  180 +
 .../internal/RequestResourceProvider.java   |   25 +
 .../internal/ScaleClusterRequest.java   |  156 +
 .../internal/ServiceResourceProvider.java   |   63 +-
 .../server/controller/internal/Stack.java   |  374 +-
 .../internal/StageResourceProvider.java |   70 +-
 .../ambari/server/orm/dao/BlueprintDAO.java |   15 +
 .../server/orm/entities/BlueprintEntity.java|  123 +-
 .../server/stack/NoSuchStackException.java  |   28 +
 .../org/apache/ambari/server/state/Cluster.java |6 +
 .../server/state/cluster/ClusterImpl.java   |   19 +
 .../ambari/server/state/host/HostImpl.java  |   18 +
 .../ambari/server/topology/Blueprint.java   |  126 +
 .../server/topology/BlueprintFactory.java   |  199 +
 .../ambari/server/topology/BlueprintImpl.java   |  397 ++
 .../server/topology/BlueprintValidator.java |   41 +
 .../server/topology/BlueprintValidatorImpl.java |  318 ++
 .../ambari/server/topology/Cardinality.java |   90 +
 .../topology/ClusterConfigurationRequest.java   |  271 +
 .../ambari/server/topology/ClusterTopology.java |  116 +
 .../server/topology/ClusterTopologyImpl.java|  245 +
 .../ambari/server/topology/Configuration.java   |  187 +
 .../server/topology/ConfigurationFactory.java   |  121 +
 .../ambari/server/topology/HostGroup.java   |  119 +
 .../ambari/server/topology/HostGroupImpl.java   |  239 +
 .../ambari/server/topology/HostGroupInfo.java   |   91 +
 .../server/topology/HostOfferResponse.java  |   62 +
 .../ambari/server/topology/HostRequest.java |  814 +++
 .../topology/InvalidTopologyException.java  |   32 +
 .../InvalidTopologyTemplateException.java   |   34 +
 .../ambari/server/topology/LogicalRequest.java  |  307 +
 .../topology/NoSuchBlueprintException.java  |   29 +
 .../topology/NoSuchHostGroupException.java  |   37 +
 .../topology/RequiredPasswordValidator.java |  155 +
 .../ambari/server/topology/TopologyManager.java |  610 ++
 .../ambari/server/topology/TopologyRequest.java |   40 +
 .../server/topology/TopologyRequestFactory.java |   30 +
 .../topology/TopologyRequestFactoryImpl.java|   34 +
 .../ambari/server/topology/TopologyTask.java|   42 +
 .../server/topology/TopologyValidator.java  |   26 +
 .../apache/ambari/server/utils/StageUtils.java  |   69 +-
 .../render/ClusterBlueprintRendererTest.java|  127 +-
 .../AmbariCustomCommandExecutionHelperTest.java |3 +
 .../AmbariManagementControllerTest.java |   30 +-
 .../BackgroundCustomCommandExecutionTest.java   |3 +
 .../server/controller/KerberosHelperTest.java   |   19 +
 .../internal/BaseBlueprintProcessorTest.java|  494 +-
 .../BlueprintConfigurationProcessorTest.java| 5255 +-
 .../internal/BlueprintResourceProviderTest.java | 1051 +---
 .../ClientConfigResourceProviderTest.java   |   11 +-

[08/13] ambari git commit: AMBARI-10750. Initial merge of advanced api provisioning work.

2015-04-26 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
new file mode 100644
index 000..70d1907
--- /dev/null
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -0,0 +1,318 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.state.AutoDeployInfo;
+import org.apache.ambari.server.state.DependencyInfo;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+/**
+ * Default blueprint validator.
+ */
+public class BlueprintValidatorImpl implements BlueprintValidator {
+
+  private final Blueprint blueprint;
+  private final Stack stack;
+
+  public BlueprintValidatorImpl(Blueprint blueprint) {
+this.blueprint = blueprint;
+this.stack = blueprint.getStack();
+  }
+  @Override
+  public void validateTopology() throws InvalidTopologyException {
+CollectionHostGroup hostGroups = blueprint.getHostGroups().values();
+MapString, MapString, CollectionDependencyInfo missingDependencies =
+new HashMapString, MapString, CollectionDependencyInfo();
+
+CollectionString services = blueprint.getServices();
+for (HostGroup group : hostGroups) {
+  MapString, CollectionDependencyInfo missingGroupDependencies = 
validateHostGroup(group);
+  if (! missingGroupDependencies.isEmpty()) {
+missingDependencies.put(group.getName(), missingGroupDependencies);
+  }
+}
+
+CollectionString cardinalityFailures = new HashSetString();
+for (String service : services) {
+  for (String component : stack.getComponents(service)) {
+Cardinality cardinality = stack.getCardinality(component);
+AutoDeployInfo autoDeploy = stack.getAutoDeployInfo(component);
+if (cardinality.isAll()) {
+  cardinalityFailures.addAll(verifyComponentInAllHostGroups(component, 
autoDeploy));
+} else {
+  cardinalityFailures.addAll(verifyComponentCardinalityCount(
+  component, cardinality, autoDeploy));
+}
+  }
+}
+
+if (! missingDependencies.isEmpty() || ! cardinalityFailures.isEmpty()) {
+  generateInvalidTopologyException(missingDependencies, 
cardinalityFailures);
+}
+  }
+
+  @Override
+  public void validateRequiredProperties() throws InvalidTopologyException {
+//todo: combine with RequiredPasswordValidator
+MapString, MapString, CollectionString missingProperties =
+new HashMapString, MapString, CollectionString();
+
+// we don't want to include default stack properties so we can't just use 
hostGroup full properties
+MapString, MapString, String clusterConfigurations = 
blueprint.getConfiguration().getProperties();
+
+for (HostGroup hostGroup : blueprint.getHostGroups().values()) {
+  CollectionString processedServices = new HashSetString();
+  MapString, CollectionString allRequiredProperties = new 
HashMapString, CollectionString();
+  MapString, MapString, String operationalConfiguration = new 
HashMapString, MapString, String(clusterConfigurations);
+
+  
operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties());
+  for (String component : hostGroup.getComponents()) {
+//check that MYSQL_SERVER component is not available while hive is 
using existing db
+if (component.equals(MYSQL_SERVER)) {
+  MapString, String hiveEnvConfig = 
clusterConfigurations.get(hive-env);
+  if (hiveEnvConfig != null  !hiveEnvConfig.isEmpty()  
hiveEnvConfig.get(hive_database) != null
+   hiveEnvConfig.get(hive_database).startsWith(Existing)) {
+throw new IllegalArgumentException(Incorrect configuration: 
MYSQL_SERVER

[09/13] ambari git commit: AMBARI-10750. Initial merge of advanced api provisioning work.

2015-04-26 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/c9f0dd0b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
index 9ef13ba..82d03fd 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
@@ -37,11 +37,14 @@ import 
org.apache.ambari.server.controller.StackServiceResponse;
 import org.apache.ambari.server.orm.entities.StackEntity;
 import org.apache.ambari.server.state.AutoDeployInfo;
 import org.apache.ambari.server.state.DependencyInfo;
+import org.apache.ambari.server.state.PropertyInfo;
+import org.apache.ambari.server.topology.Cardinality;
+import org.apache.ambari.server.topology.Configuration;
 
 /**
  * Encapsulates stack information.
  */
-class Stack {
+public class Stack {
   /**
* Stack name
*/
@@ -88,6 +91,10 @@ class Stack {
*/
   private MapString, String cardinalityRequirements = new HashMapString, 
String();
 
+  //todo: instead of all these maps from component - * ,
+  //todo: we should use a Component object with all of these attributes
+  private SetString masterComponents = new HashSetString();
+
   /**
* Map of component to auto-deploy information
*/
@@ -101,45 +108,28 @@ class Stack {
   new HashMapString, MapString, MapString, ConfigProperty();
 
   /**
-   * Map of service to set of excluded config types
+   * Map of service to required type properties
*/
-  private MapString, SetString excludedConfigurationTypes =
-new HashMapString, SetString();
+  private MapString, MapString, MapString, ConfigProperty 
requiredServiceConfigurations =
+  new HashMapString, MapString, MapString, ConfigProperty();
 
   /**
-   * Ambari Management Controller, used to obtain Stack definitions
+   * Map of service to config type properties
*/
-  private final AmbariManagementController ambariManagementController;
+  private MapString, MapString, ConfigProperty stackConfigurations =
+  new HashMapString, MapString, ConfigProperty();
 
   /**
-   * Contains a configuration property's value and attributes.
+   * Map of service to set of excluded config types
*/
-  private class ConfigProperty {
-
-private ConfigProperty(String value, MapString, String attributes) {
-  this.value = value;
-  this.attributes = attributes;
-}
-
-private String value;
-private MapString, String attributes;
-
-public String getValue() {
-  return value;
-}
-
-public void setValue(String value) {
-  this.value = value;
-}
+  private MapString, SetString excludedConfigurationTypes =
+new HashMapString, SetString();
 
-public MapString, String getAttributes() {
-  return attributes;
-}
+  /**
+   * Ambari Management Controller, used to obtain Stack definitions
+   */
+  private final AmbariManagementController controller;
 
-public void setAttributes(MapString, String attributes) {
-  this.attributes = attributes;
-}
-  }
 
   /**
* Constructor.
@@ -163,15 +153,16 @@ class Stack {
* @param name stack name
* @param version  stack version
*
-   * @throws org.apache.ambari.server.AmbariException an exception occurred 
getting stack information
+   * @throws AmbariException an exception occurred getting stack information
* for the specified name and version
*/
-  public Stack(String name, String version, AmbariManagementController 
ambariManagementController) throws AmbariException {
+  //todo: don't pass management controller in constructor
+  public Stack(String name, String version, AmbariManagementController 
controller) throws AmbariException {
 this.name = name;
 this.version = version;
-this.ambariManagementController = ambariManagementController;
+this.controller = controller;
 
-SetStackServiceResponse stackServices = 
ambariManagementController.getStackServices(
+SetStackServiceResponse stackServices = controller.getStackServices(
 Collections.singleton(new StackServiceRequest(name, version, null)));
 
 for (StackServiceResponse stackService : stackServices) {
@@ -181,6 +172,9 @@ class Stack {
   parseConfigurations(serviceName);
   registerConditionalDependencies();
 }
+
+//todo: already done for each service
+parseStackConfigurations();
   }
 
   /**
@@ -227,26 +221,57 @@ class Stack {
   }
 
   /**
+   * Get all service components
+   *
+   * @return map of service to associated components
+   */
+  public MapString, CollectionString getComponents() {
+MapString, CollectionString serviceComponents = new HashMapString, 
CollectionString();
+for (String service :

ambari git commit: AMBARI-9870. Provide ability to test KDC connection over UDP

2015-03-02 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 121924874 - 86d658938


AMBARI-9870. Provide ability to test KDC connection over UDP


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/86d65893
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/86d65893
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/86d65893

Branch: refs/heads/trunk
Commit: 86d6589382ac0467640c2f286c950877387fd41a
Parents: 1219248
Author: John Speidel jspei...@hortonworks.com
Authored: Mon Mar 2 11:23:50 2015 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Mar 2 15:26:44 2015 -0500

--
 ambari-project/pom.xml  |   5 +
 ambari-server/pom.xml   |   4 +
 .../server/KdcServerConnectionVerification.java | 128 -
 .../KdcServerConnectionVerificationTest.java| 190 +++
 4 files changed, 321 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/86d65893/ambari-project/pom.xml
--
diff --git a/ambari-project/pom.xml b/ambari-project/pom.xml
index 0577bee..48c7214 100644
--- a/ambari-project/pom.xml
+++ b/ambari-project/pom.xml
@@ -247,6 +247,11 @@ instead of a SNAPSHOT. --
   /dependency
   dependency
 groupIdorg.apache.directory.server/groupId
+artifactIdkerberos-client/artifactId
+version2.0.0-M19/version
+  /dependency
+  dependency
+groupIdorg.apache.directory.server/groupId
 artifactIdapacheds-protocol-ldap/artifactId
 version2.0.0-M19/version
 exclusions

http://git-wip-us.apache.org/repos/asf/ambari/blob/86d65893/ambari-server/pom.xml
--
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index c57a2d0..4c56d78 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -1515,6 +1515,10 @@
   scopetest/scope
 /dependency
 dependency
+  groupIdorg.apache.directory.server/groupId
+  artifactIdkerberos-client/artifactId
+/dependency
+dependency
   groupIdorg.apache.directory.shared/groupId
   artifactIdshared-ldap/artifactId
   scopetest/scope

http://git-wip-us.apache.org/repos/asf/ambari/blob/86d65893/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
index 8bfbc5f..b7bfef9 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
@@ -22,9 +22,18 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.net.UnknownHostException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.commons.lang.StringUtils;
+import org.apache.directory.kerberos.client.KdcConfig;
+import org.apache.directory.kerberos.client.KdcConnection;
+import org.apache.directory.shared.kerberos.exceptions.ErrorType;
+import org.apache.directory.shared.kerberos.exceptions.KerberosException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -53,6 +62,11 @@ public class KdcServerConnectionVerification {
   private static Logger LOG = 
LoggerFactory.getLogger(KdcServerConnectionVerification.class);
 
   private Configuration config;
+
+  /**
+   * UDP connection timeout in seconds.
+   */
+  private int udpTimeout = 10;
   
   @Inject
   public KdcServerConnectionVerification(Configuration config) {
@@ -84,15 +98,30 @@ public class KdcServerConnectionVerification {
   return false;
 }
   }
+
   /**
-   * Given server IP or hostname, checks if server is reachable i.e.
-   * we can make a socket connection to it.
-   * 
+   * Given a host and port, checks if server is reachable meaning that we
+   * can communicate with it.  First we attempt to connect via TCP and if
+   * that is unsuccessful, attempt via UDP. It is important to understand that
+   * we are not validating credentials, only attempting to communicate with 
server
+   * process for the give host and port.
+   *
* @param server KDC server IP or hostname
* @param portKDC port
* @return   true, if server is accepting connection given

ambari git commit: AMBARI-9870. Provide ability to test KDC connection over UDP

2015-03-02 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-2.0.0 7bf692d93 - f84775853


AMBARI-9870. Provide ability to test KDC connection over UDP


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f8477585
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f8477585
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f8477585

Branch: refs/heads/branch-2.0.0
Commit: f847758538594ab44c16467b569ba1036e8cdbec
Parents: 7bf692d
Author: John Speidel jspei...@hortonworks.com
Authored: Mon Mar 2 11:23:50 2015 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Mar 2 15:34:09 2015 -0500

--
 ambari-project/pom.xml  |   5 +
 ambari-server/pom.xml   |   4 +
 .../server/KdcServerConnectionVerification.java | 128 -
 .../KdcServerConnectionVerificationTest.java| 190 +++
 4 files changed, 321 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/f8477585/ambari-project/pom.xml
--
diff --git a/ambari-project/pom.xml b/ambari-project/pom.xml
index 0577bee..48c7214 100644
--- a/ambari-project/pom.xml
+++ b/ambari-project/pom.xml
@@ -247,6 +247,11 @@ instead of a SNAPSHOT. --
   /dependency
   dependency
 groupIdorg.apache.directory.server/groupId
+artifactIdkerberos-client/artifactId
+version2.0.0-M19/version
+  /dependency
+  dependency
+groupIdorg.apache.directory.server/groupId
 artifactIdapacheds-protocol-ldap/artifactId
 version2.0.0-M19/version
 exclusions

http://git-wip-us.apache.org/repos/asf/ambari/blob/f8477585/ambari-server/pom.xml
--
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index c57a2d0..4c56d78 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -1515,6 +1515,10 @@
   scopetest/scope
 /dependency
 dependency
+  groupIdorg.apache.directory.server/groupId
+  artifactIdkerberos-client/artifactId
+/dependency
+dependency
   groupIdorg.apache.directory.shared/groupId
   artifactIdshared-ldap/artifactId
   scopetest/scope

http://git-wip-us.apache.org/repos/asf/ambari/blob/f8477585/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
index 8bfbc5f..b7bfef9 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/KdcServerConnectionVerification.java
@@ -22,9 +22,18 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.net.UnknownHostException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.commons.lang.StringUtils;
+import org.apache.directory.kerberos.client.KdcConfig;
+import org.apache.directory.kerberos.client.KdcConnection;
+import org.apache.directory.shared.kerberos.exceptions.ErrorType;
+import org.apache.directory.shared.kerberos.exceptions.KerberosException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -53,6 +62,11 @@ public class KdcServerConnectionVerification {
   private static Logger LOG = 
LoggerFactory.getLogger(KdcServerConnectionVerification.class);
 
   private Configuration config;
+
+  /**
+   * UDP connection timeout in seconds.
+   */
+  private int udpTimeout = 10;
   
   @Inject
   public KdcServerConnectionVerification(Configuration config) {
@@ -84,15 +98,30 @@ public class KdcServerConnectionVerification {
   return false;
 }
   }
+
   /**
-   * Given server IP or hostname, checks if server is reachable i.e.
-   * we can make a socket connection to it.
-   * 
+   * Given a host and port, checks if server is reachable meaning that we
+   * can communicate with it.  First we attempt to connect via TCP and if
+   * that is unsuccessful, attempt via UDP. It is important to understand that
+   * we are not validating credentials, only attempting to communicate with 
server
+   * process for the give host and port.
+   *
* @param server KDC server IP or hostname
* @param portKDC port
* @return   true, if server is accepting

ambari git commit: AMBARI-9640. Allow the KDC admin credentials stored in session to be validated via the REST API

2015-02-14 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 4b54c366f - 140213f2d


AMBARI-9640. Allow the KDC admin credentials stored in session to be validated 
via the REST API


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/140213f2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/140213f2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/140213f2

Branch: refs/heads/trunk
Commit: 140213f2da3273ca3fe012ab55d231b6622dc369
Parents: 4b54c36
Author: John Speidel jspei...@hortonworks.com
Authored: Fri Feb 13 19:50:03 2015 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Sat Feb 14 16:32:21 2015 -0500

--
 .../server/controller/KerberosHelper.java   | 211 ++---
 .../internal/ServiceResourceProvider.java   |  61 +
 .../KerberosInvalidConfigurationException.java  |  43 
 ...erberosMissingAdminCredentialsException.java |  43 
 .../server/controller/KerberosHelperTest.java   | 102 -
 .../internal/ServiceResourceProviderTest.java   | 224 +++
 6 files changed, 548 insertions(+), 136 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/140213f2/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index 2e68c7d..fa829a4 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -57,8 +57,10 @@ import 
org.apache.ambari.server.serveraction.kerberos.KerberosAdminAuthenticatio
 import org.apache.ambari.server.serveraction.kerberos.KerberosConfigDataFile;
 import 
org.apache.ambari.server.serveraction.kerberos.KerberosConfigDataFileBuilder;
 import org.apache.ambari.server.serveraction.kerberos.KerberosCredential;
+import 
org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException;
 import 
org.apache.ambari.server.serveraction.kerberos.KerberosKDCConnectionException;
 import 
org.apache.ambari.server.serveraction.kerberos.KerberosLDAPContainerException;
+import 
org.apache.ambari.server.serveraction.kerberos.KerberosMissingAdminCredentialsException;
 import 
org.apache.ambari.server.serveraction.kerberos.KerberosOperationException;
 import org.apache.ambari.server.serveraction.kerberos.KerberosOperationHandler;
 import 
org.apache.ambari.server.serveraction.kerberos.KerberosOperationHandlerFactory;
@@ -188,7 +190,12 @@ public class KerberosHelper {
   RequestStageContainer 
requestStageContainer)
   throws AmbariException {
 
-KerberosDetails kerberosDetails = getKerberosDetails(cluster);
+KerberosDetails kerberosDetails;
+try {
+  kerberosDetails = getKerberosDetails(cluster);
+} catch (KerberosInvalidConfigurationException e) {
+  throw new IllegalArgumentException(e.getMessage(), e);
+}
 
 // Update KerberosDetails with the new security type - the current one in 
the cluster is the old value
 kerberosDetails.setSecurityType(securityType);
@@ -234,12 +241,18 @@ public class KerberosHelper {
 throw new AmbariException(String.format(Custom operation %s 
can only be requested with the security type cluster property: %s, 
operation.name(), SecurityType.KERBEROS.name()));
   }
 
-  if (true.equalsIgnoreCase(value) || 
all.equalsIgnoreCase(value)) {
-requestStageContainer = handle(cluster, 
getKerberosDetails(cluster), null, null, requestStageContainer, new 
CreatePrincipalsAndKeytabsHandler(true));
-  } else if (missing.equalsIgnoreCase(value)) {
-requestStageContainer = handle(cluster, 
getKerberosDetails(cluster), null, null, requestStageContainer, new 
CreatePrincipalsAndKeytabsHandler(false));
-  } else {
-throw new AmbariException(String.format(Unexpected directive 
value: %s, value));
+  try {
+if (true.equalsIgnoreCase(value) || 
all.equalsIgnoreCase(value)) {
+  requestStageContainer = handle(cluster, 
getKerberosDetails(cluster), null, null,
+  requestStageContainer, new 
CreatePrincipalsAndKeytabsHandler(true));
+} else if (missing.equalsIgnoreCase(value)) {
+  requestStageContainer = handle(cluster, 
getKerberosDetails(cluster), null, null,
+  requestStageContainer, new 
CreatePrincipalsAndKeytabsHandler(false));
+} else {
+

ambari git commit: AMBARI-9022. Preserve existing kerberos auth_to_local properties when scaling cluster

2015-02-11 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk bb99a97bd - 2ddd1004e


AMBARI-9022.  Preserve existing kerberos auth_to_local properties when scaling 
cluster


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2ddd1004
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2ddd1004
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2ddd1004

Branch: refs/heads/trunk
Commit: 2ddd1004ea82c59a247310f7e48d63540b5685a4
Parents: bb99a97
Author: John Speidel jspei...@hortonworks.com
Authored: Tue Feb 10 17:36:24 2015 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Feb 12 00:52:08 2015 -0500

--
 .../server/controller/AuthToLocalBuilder.java   | 475 ---
 .../server/controller/KerberosHelper.java   |   7 +-
 .../HDFS/2.1.0.2.0/configuration/core-site.xml  |   5 -
 .../controller/AuthToLocalBuilderTest.java  | 247 --
 4 files changed, 623 insertions(+), 111 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/2ddd1004/ambari-server/src/main/java/org/apache/ambari/server/controller/AuthToLocalBuilder.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AuthToLocalBuilder.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AuthToLocalBuilder.java
index 6017bed..a22c759 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AuthToLocalBuilder.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AuthToLocalBuilder.java
@@ -18,54 +18,72 @@
 
 package org.apache.ambari.server.controller;
 
-import java.util.Map;
-import java.util.TreeMap;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
  * AuthToLocalBuilder helps to create auth_to_local rules for use in 
configuration files like
- * core-site.xml.
+ * core-site.xml.  No duplicate rules will be generated.
  * p/
- * For each principal appended to the rule set, parse out the primary value 
and match it to a local
- * username.  Then when done appending all principals, generate the rules 
where each entry yields
- * one of the following rule:
+ * Allows previously existing rules to be added verbatim.  Also allows new 
rules to be generated
+ * based on a principal and local username.  For each principal added to the 
builder, generate
+ * a rule conforming to one of the following formats:
  * p/
- * Qualified Principal: RULE:[2:$1@$0](PRIMARY@REALM)s/.*\/LOCAL_USERNAME/
+ * Qualified Principal (the principal contains a user and host):
+ * RULE:[2:$1@$0](PRIMARY@REALM)s/.*\/LOCAL_USERNAME/
  * p/
- * Unqualified Principal: RULE:[1:$1@$0](PRIMARY@REALM)s/.*\/LOCAL_USERNAME/
+ * Unqualified Principal (only user is specified):
+ * RULE:[1:$1@$0](PRIMARY@REALM)s/.*\/LOCAL_USERNAME/
+ * p
+ * Additionally, for each realm included in the rule set, generate a default 
realm rule
+ * in the format: RULE:[1:$1@$0](.*@REALM)s/@.{@literal *}//
+ * p
+ * Ordering guarantees for the generated rule string are as follows:
+ * ul
+ *   liRules with the same expected component count are ordered according to 
match component count/li
+ *   liRules with different expected component count are ordered according 
to the default string ordering/li
+ *   liRules in the form of .*@REALM are ordered after all other rules with 
the same expected component count/li
+ * /ul
+ *
  */
 public class AuthToLocalBuilder {
 
   /**
-   * A Regular expression declaring a qualified principal such that the 
principal is in the following format:
-   * primary/instance@REALM
-   */
-  private static final Pattern PATTERN_QUALIFIED_PRINCIPAL = 
Pattern.compile((\\w+)/.*@.*);
-
-  /**
-   * A Regular expression declaring an un qualified principal such that the 
principal is in the following format:
-   * primary@REALM
+   * Ordered set of rules which have been added to the builder.
*/
-  private static final Pattern PATTERN_UNQUALIFIED_PRINCIPAL = 
Pattern.compile((\\w+)@.*);
+  private SetRule setRules = new TreeSetRule();
 
-  /**
-   * A map of qualified principal names (primary/instance@REALM, with instance 
and @REALM removed).
-   * p/
-   * A TreeMap is used to help generate deterministic ordering of rules for 
testing.
-   */
-  private MapString, String qualifiedAuthToLocalMap = new TreeMapString, 
String();
 
   /**
-   * A map of unqualified principal names (primary@REALM, with @REALM removed).
-   * p/
-   * A TreeMap is used to help generate deterministic ordering of rules for 
testing.
+   * Add existing rules from the given authToLocal configuration property.
+   * The rules are added

ambari git commit: AMBARI-9367. Fix regression of new high level add hosts api. Also, ensure that kerberos client is added to hostgroups if kerberos is enabled.

2015-02-02 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 015d89b7c - 44b2a5cca


AMBARI-9367. Fix regression of new high level add hosts api.
 Also, ensure that kerberos client is added to hostgroups if 
kerberos is enabled.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/44b2a5cc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/44b2a5cc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/44b2a5cc

Branch: refs/heads/trunk
Commit: 44b2a5ccaa7b58878d19634acf364eb90a9ef34e
Parents: 015d89b
Author: John Speidel jspei...@hortonworks.com
Authored: Mon Feb 2 16:44:10 2015 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Feb 2 16:52:26 2015 -0500

--
 .../internal/HostComponentResourceProvider.java | 60 +---
 .../internal/HostResourceProvider.java  | 28 -
 .../HostComponentResourceProviderTest.java  | 31 +++---
 3 files changed, 90 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/44b2a5cc/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
index b1e05cc..b513de7 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
@@ -264,7 +264,7 @@ public class HostComponentResourceProvider extends 
AbstractControllerResourcePro
   throw new IllegalArgumentException(Received an update request with no 
properties);
 }
 
-RequestStageContainer requestStages = doUpdateResources(null, request, 
predicate);
+RequestStageContainer requestStages = doUpdateResources(null, request, 
predicate, false);
 
 RequestStatusResponse response = null;
 if (requestStages != null) {
@@ -343,7 +343,7 @@ public class HostComponentResourceProvider extends 
AbstractControllerResourcePro
 
 try {
   LOG.info(Installing all components on added hosts);
-  requestStages = doUpdateResources(null, installRequest, 
installPredicate);
+  requestStages = doUpdateResources(null, installRequest, 
installPredicate, true);
   notifyUpdate(Resource.Type.HostComponent, installRequest, 
installPredicate);
 
   MapString, Object startProperties = new HashMapString, Object();
@@ -362,7 +362,7 @@ public class HostComponentResourceProvider extends 
AbstractControllerResourcePro
   LOG.info(Starting all non-client components on added hosts);
   //todo: if a host in in state HEARTBEAT_LOST, no stage will be created, 
so if this occurs during INSTALL
   //todo: then no INSTALL stage will exist which will result in invalid 
state transition INIT-STARTED
-  doUpdateResources(requestStages, startRequest, startPredicate);
+  doUpdateResources(requestStages, startRequest, startPredicate, true);
   notifyUpdate(Resource.Type.HostComponent, startRequest, startPredicate);
   try {
 requestStages.persist();
@@ -616,8 +616,25 @@ public class HostComponentResourceProvider extends 
AbstractControllerResourcePro
 return serviceComponentHostRequest;
   }
 
-  private RequestStageContainer doUpdateResources(final RequestStageContainer 
stages, final Request request, Predicate predicate)
-  throws UnsupportedPropertyException, SystemException, 
NoSuchResourceException, NoSuchParentResourceException {
+  /**
+   * Update resources.
+   *
+   * @param stages  request stage container
+   * @param request request
+   * @param predicate   request predicate
+   * @param performQueryEvaluation  should query be evaluated for matching 
resource set
+   * @return
+   * @throws UnsupportedPropertyException   an unsupported property was 
specified in the request
+   * @throws SystemExceptionan unknown exception occurred
+   * @throws NoSuchResourceExceptionthe query didn't match any 
resources
+   * @throws NoSuchParentResourceException  a specified parent resource 
doesn't exist
+   */
+  private RequestStageContainer doUpdateResources(final RequestStageContainer 
stages, final Request request,
+  Predicate predicate, boolean 
performQueryEvaluation)
+  throws 
UnsupportedPropertyException,
+ SystemException,
+

ambari git commit: AMBARI-9410. Allow cluster and service artifacts to be updated and deleted

2015-01-30 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk b2f392d8f - ee4e786a4


AMBARI-9410. Allow cluster and service artifacts to be updated and deleted


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ee4e786a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ee4e786a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ee4e786a

Branch: refs/heads/trunk
Commit: ee4e786a4daf639da34ea81a7ff1a11f3dad378f
Parents: b2f392d
Author: John Speidel jspei...@hortonworks.com
Authored: Thu Jan 29 17:27:14 2015 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Fri Jan 30 11:54:39 2015 -0500

--
 .../server/api/services/ClusterService.java | 124 --
 .../server/api/services/ServiceService.java |  92 
 .../internal/ArtifactResourceProvider.java  |  84 ++-
 .../ambari/server/orm/dao/ArtifactDAO.java  |  22 +-
 .../server/api/services/ClusterServiceTest.java |  50 +++-
 .../server/api/services/ServiceServiceTest.java |  59 +
 .../internal/ArtifactResourceProviderTest.java  | 235 ---
 7 files changed, 607 insertions(+), 59 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/ee4e786a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ClusterService.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ClusterService.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ClusterService.java
index 5f44b2f..a794693 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ClusterService.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ClusterService.java
@@ -185,10 +185,10 @@ public class ClusterService extends BaseService {
   @GET
   @Path({clusterName}/artifacts)
   @Produces(text/plain)
-  public Response getClusterArtifacts(String body,
-  @Context HttpHeaders headers,
-  @Context UriInfo ui,
-  @PathParam(clusterName) String 
clusterName) {
+  public Response getArtifacts(String body,
+   @Context HttpHeaders headers,
+   @Context UriInfo ui,
+   @PathParam(clusterName) String clusterName) {
 
 hasPermission(Request.Type.GET, clusterName);
 return handleRequest(headers, body, ui, Request.Type.GET,
@@ -210,11 +210,11 @@ public class ClusterService extends BaseService {
   @GET
   @Path({clusterName}/artifacts/{artifactName})
   @Produces(text/plain)
-  public Response getClusterArtifact(String body,
-  @Context HttpHeaders headers,
-  @Context UriInfo ui,
-  @PathParam(clusterName) String 
clusterName,
-  @PathParam(artifactName) String 
artifactName) {
+  public Response getArtifact(String body,
+  @Context HttpHeaders headers,
+  @Context UriInfo ui,
+  @PathParam(clusterName) String clusterName,
+  @PathParam(artifactName) String artifactName) {
 
 hasPermission(Request.Type.GET, clusterName);
 return handleRequest(headers, body, ui, Request.Type.GET,
@@ -235,11 +235,11 @@ public class ClusterService extends BaseService {
   @POST
   @Path({clusterName}/artifacts/{artifactName})
   @Produces(text/plain)
-  public Response createClusterArtifact(String body,
-  @Context HttpHeaders headers,
-  @Context UriInfo ui,
-  @PathParam(clusterName) String 
clusterName,
-  @PathParam(artifactName) String 
artifactName) {
+  public Response createArtifact(String body,
+ @Context HttpHeaders headers,
+ @Context UriInfo ui,
+ @PathParam(clusterName) String clusterName,
+ @PathParam(artifactName) String 
artifactName) {
 
 hasPermission(Request.Type.POST, clusterName);
 return handleRequest(headers, body, ui, Request.Type.POST,
@@ -247,6 +247,102 @@ public class ClusterService extends BaseService {
   }
 
   /**
+   * Handles: PUT /clusters/{clusterID}/artifacts
+   * Update all artifacts matching the provided predicate.
+   *
+   * @param headers  http headers
+   * @param ui   uri info
+   * @param clusterName  cluster name
+   *
+   * @return information regarding the updated artifacts
+   */
+

[1/2] ambari git commit: AMBARI-9364. Handle hosts/services/components being added to a kerberized cluster.

2015-01-27 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk c242beeac - 5e930e44c


AMBARI-9364. Handle hosts/services/components being added to a kerberized 
cluster.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1dd3ad21
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1dd3ad21
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1dd3ad21

Branch: refs/heads/trunk
Commit: 1dd3ad2178dc1b1a11511ce163a64aa78d3dfdd7
Parents: c242bee
Author: John Speidel jspei...@hortonworks.com
Authored: Tue Jan 27 18:44:59 2015 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Tue Jan 27 19:37:16 2015 -0500

--
 .../server/controller/KerberosHelper.java   | 115 -
 .../internal/HostComponentResourceProvider.java |  47 +-
 .../server/controller/KerberosHelperTest.java   | 169 +--
 .../HostComponentResourceProviderTest.java  | 129 +-
 4 files changed, 436 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/1dd3ad21/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index c537498..6bb9bf1 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -29,8 +29,21 @@ import org.apache.ambari.server.actionmanager.Stage;
 import org.apache.ambari.server.actionmanager.StageFactory;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.controller.internal.ArtifactResourceProvider;
+import org.apache.ambari.server.controller.internal.RequestImpl;
 import org.apache.ambari.server.controller.internal.RequestResourceFilter;
 import org.apache.ambari.server.controller.internal.RequestStageContainer;
+import org.apache.ambari.server.controller.spi.ClusterController;
+import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
+import org.apache.ambari.server.controller.spi.NoSuchResourceException;
+import org.apache.ambari.server.controller.spi.Predicate;
+import org.apache.ambari.server.controller.spi.Request;
+import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.spi.ResourceProvider;
+import org.apache.ambari.server.controller.spi.SystemException;
+import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
+import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
+import org.apache.ambari.server.controller.utilities.PredicateBuilder;
 import org.apache.ambari.server.metadata.RoleCommandOrder;
 import org.apache.ambari.server.serveraction.ServerAction;
 import org.apache.ambari.server.serveraction.kerberos.*;
@@ -64,6 +77,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -76,6 +90,16 @@ public class KerberosHelper {
 
   private static final Logger LOG = 
LoggerFactory.getLogger(KerberosHelper.class);
 
+  /**
+   * config type which contains the property used to determine if keberos is 
enabled
+   */
+  private static final String SECURITY_ENABLED_CONFIG_TYPE = cluster-env;
+
+  /**
+   * name of property which states whether kerberos is enabled
+   */
+  private static final String SECURITY_ENABLED_PROPERTY_NAME = 
security_enabled;
+
   @Inject
   private AmbariCustomCommandExecutionHelper customCommandExecutionHelper;
 
@@ -107,6 +131,12 @@ public class KerberosHelper {
   private KerberosOperationHandlerFactory kerberosOperationHandlerFactory;
 
   /**
+   * Used to get kerberos descriptors associated with the cluster or stack.
+   * Currently not available via injection.
+   */
+  private static ClusterController clusterController = null;
+
+  /**
* The Handler implementation that provides the logic to enable Kerberos
*/
   private Handler enableKerberosHandler = new EnableKerberosHandler();
@@ -159,6 +189,11 @@ public class KerberosHelper {
 // Update KerberosDetails with the new security type - the current one in 
the cluster is the old value
 kerberosDetails.setSecurityType(securityType);
 
+//todo: modify call from cluster state transition to not include descriptor
+if (kerberosDescriptor == null) {
+  kerberosDescriptor = getClusterDescriptor(cluster);
+}
+
 if (securityType ==

[2/2] ambari git commit: AMBARI-9366. Fix create statement for artifact table in external Postgres DB script

2015-01-27 Thread jspeidel

AMBARI-9366.  Fix create statement for artifact table in external Postgres DB 
script


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5e930e44
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5e930e44
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5e930e44

Branch: refs/heads/trunk
Commit: 5e930e44c8412cab73554a8ab17c13b32f5bb630
Parents: 1dd3ad2
Author: John Speidel jspei...@hortonworks.com
Authored: Tue Jan 27 19:49:27 2015 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Tue Jan 27 19:49:27 2015 -0500

--
 ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/5e930e44/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
--
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql 
b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
index 64ffc4f..b589993 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
@@ -509,7 +509,7 @@ CREATE TABLE repo_version (
   PRIMARY KEY(repo_version_id)
 );
 
-CREATE TABLE ambari.artifact (
+CREATE TABLE artifact (
   artifact_name VARCHAR(255) NOT NULL,
   artifact_data TEXT NOT NULL,
   foreign_keys VARCHAR(255) NOT NULL,

ambari git commit: AMBARI-9243. Preserve nested map structures for artifact resource data

2015-01-21 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 07c78ba39 - fcc6a4ccf


AMBARI-9243.  Preserve nested map structures for artifact resource data


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fcc6a4cc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fcc6a4cc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fcc6a4cc

Branch: refs/heads/trunk
Commit: fcc6a4ccf7bd4e49e5ad0d6062c0ce727183b4b2
Parents: 07c78ba
Author: John Speidel jspei...@hortonworks.com
Authored: Wed Jan 21 13:23:45 2015 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed Jan 21 17:45:00 2015 -0500

--
 .../ambari/server/api/services/RequestBody.java |   2 +
 .../internal/ArtifactResourceProvider.java  |  37 +++---
 .../ambari/server/controller/spi/Request.java   |   4 +
 .../parsers/JsonRequestBodyParserTest.java  |   3 +-
 .../internal/ArtifactResourceProviderTest.java  | 120 ++-
 5 files changed, 146 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/fcc6a4cc/ambari-server/src/main/java/org/apache/ambari/server/api/services/RequestBody.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/RequestBody.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/RequestBody.java
index f4c5647..548922d 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/RequestBody.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/RequestBody.java
@@ -144,12 +144,14 @@ public class RequestBody {
 
   /**
* Set the body from the request.
+   * Also sets the body as a request info property.
*
* @param body the request body
*/
   public void setBody(String body) {
 if (body != null  ! body.isEmpty()) {
   m_body = body;
+  m_requestInfoProps.put(RAW_REQUEST_BODY, body);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/fcc6a4cc/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ArtifactResourceProvider.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ArtifactResourceProvider.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ArtifactResourceProvider.java
index b3eb159..e476f62 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ArtifactResourceProvider.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ArtifactResourceProvider.java
@@ -18,6 +18,7 @@
 
 package org.apache.ambari.server.controller.internal;
 
+import com.google.gson.Gson;
 import com.google.inject.Inject;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.DuplicateResourceException;
@@ -105,6 +106,11 @@ public class ArtifactResourceProvider extends 
AbstractResourceProvider {
   new HashMapString, TypeRegistration();
 
   /**
+   * serializer used to convert json to map
+   */
+  private static final Gson jsonSerializer = new Gson();
+
+  /**
* artifact data access object
*/
   @Inject
@@ -181,7 +187,7 @@ public class ArtifactResourceProvider extends 
AbstractResourceProvider {
  NoSuchParentResourceException {
 
 for (MapString, Object properties : request.getProperties()) {
-  createResources(getCreateCommand(properties));
+  createResources(getCreateCommand(properties, 
request.getRequestInfoProperties()));
 }
 notifyCreate(Resource.Type.Artifact, request);
 
@@ -232,11 +238,13 @@ public class ArtifactResourceProvider extends 
AbstractResourceProvider {
   /**
* Create a command to create the resource.
*
-   * @param properties  request properties
+   * @param propertiesrequest properties
+   * @param requestInfoProps  request info properties
*
* @return a new create command
*/
-  private CommandVoid getCreateCommand(final MapString, Object properties) 
{
+  private CommandVoid getCreateCommand(final MapString, Object properties,
+ final MapString, String 
requestInfoProps) {
 return new CommandVoid() {
   @Override
   public Void invoke() throws AmbariException {
@@ -255,7 +263,7 @@ public class ArtifactResourceProvider extends 
AbstractResourceProvider {
 LOG.debug(Creating Artifact Resource with name '{}'. Parent 
information: {},
 artifactName, getRequestForeignKeys(properties));
 
-artifactDAO.create(toEntity(properties));
+artifactDAO.create(toEntity(properties, 
requestInfoProps.get(Request.REQUEST_INFO_BODY_PROPERTY)));
 
 return null;

ambari git commit: AMBARI-8867. Ensure that bluepint deployment sets each config type on cluster no more than once

2014-12-22 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk c40a9dec2 - 03463912a


AMBARI-8867.  Ensure that bluepint deployment sets each config type on cluster
  no more than once


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/03463912
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/03463912
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/03463912

Branch: refs/heads/trunk
Commit: 03463912a95bdddab9c78a120b1de75b34c03395
Parents: c40a9de
Author: John Speidel jspei...@hortonworks.com
Authored: Mon Dec 22 16:34:22 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Dec 22 18:19:53 2014 -0500

--
 .../internal/ClusterResourceProvider.java | 18 --
 1 file changed, 12 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/03463912/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
index f9aca1d..2d6ad8f 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
@@ -815,17 +815,23 @@ public class ClusterResourceProvider extends 
BaseBlueprintProcessor {
 // create a list of config requests on a per-service basis, in order
 // to properly support the new service configuration versioning mechanism
 // in Ambari
+CollectionString encounteredConfigTypes = new HashSetString();
 for (String service : getServicesToDeploy(stack, blueprintHostGroups)) {
   BlueprintServiceConfigRequest blueprintConfigRequest =
 new BlueprintServiceConfigRequest(service);
 
   for (String serviceConfigType : stack.getConfigurationTypes(service)) {
-// skip handling of cluster-env here
-if (!serviceConfigType.equals(cluster-env)) {
-  if (mapClusterConfigurations.containsKey(serviceConfigType)) {
-blueprintConfigRequest.addConfigElement(serviceConfigType,
-  mapClusterConfigurations.get(serviceConfigType),
-  mapClusterAttributes.get(serviceConfigType));
+//todo: This is a temporary fix to ensure that we don't try to add the 
same
+//todo: config type multiple times.
+//todo: This is to unblock BUG-28939 and will be correctly fixed as 
part of BUG-29145.
+if (encounteredConfigTypes.add(serviceConfigType)) {
+  // skip handling of cluster-env here
+  if (!serviceConfigType.equals(cluster-env)) {
+if (mapClusterConfigurations.containsKey(serviceConfigType)) {
+  blueprintConfigRequest.addConfigElement(serviceConfigType,
+mapClusterConfigurations.get(serviceConfigType),
+mapClusterAttributes.get(serviceConfigType));
+}
   }
 }
   }

[1/2] ambari git commit: AMBARI-8700. Create orchestrator to manage enabling and disabling Kerberos on a cluster

2014-12-17 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 191232e29 - ef341466e


http://git-wip-us.apache.org/repos/asf/ambari/blob/ef341466/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 9dd967a..eb14c20 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -9353,6 +9353,7 @@ public class AmbariManagementControllerTest {
 injector.injectMembers(capture(controllerCapture));
 expect(injector.getInstance(Gson.class)).andReturn(null);
 
expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(maintHelper);
+
expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class));
 
 // getServices
 expect(clusters.getCluster(cluster1)).andReturn(cluster);
@@ -9396,6 +9397,7 @@ public class AmbariManagementControllerTest {
 injector.injectMembers(capture(controllerCapture));
 expect(injector.getInstance(Gson.class)).andReturn(null);
 
expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(maintHelper);
+
expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class));
 
 // getServices
 expect(clusters.getCluster(cluster1)).andReturn(cluster);
@@ -9454,6 +9456,7 @@ public class AmbariManagementControllerTest {
 injector.injectMembers(capture(controllerCapture));
 expect(injector.getInstance(Gson.class)).andReturn(null);
 
expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(maintHelper);
+
expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class));
 
 // getServices
 expect(clusters.getCluster(cluster1)).andReturn(cluster).times(4);

http://git-wip-us.apache.org/repos/asf/ambari/blob/ef341466/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
new file mode 100644
index 000..8804550
--- /dev/null
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
@@ -0,0 +1,350 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.controller;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.ActionManager;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.actionmanager.RequestFactory;
+import org.apache.ambari.server.actionmanager.Stage;
+import org.apache.ambari.server.actionmanager.StageFactory;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.controller.internal.RequestStageContainer;
+import org.apache.ambari.server.metadata.RoleCommandOrder;
+import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.security.SecurityHelper;
+import org.apache.ambari.server.serveraction.kerberos.KerberosCredential;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.Host;
+import org.apache.ambari.server.state.HostState;
+import org.apache.ambari.server.state.MaintenanceState;
+import org.apache.ambari.server.state.SecurityState;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;

[2/2] ambari git commit: AMBARI-8700. Create orchestrator to manage enabling and disabling Kerberos on a cluster

2014-12-17 Thread jspeidel

AMBARI-8700. Create orchestrator to manage enabling and disabling Kerberos on a 
cluster


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ef341466
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ef341466
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ef341466

Branch: refs/heads/trunk
Commit: ef341466e49d8f7f2cc144bb63416fb25a77f6d3
Parents: 191232e
Author: Robert Levas rle...@hortonworks.com
Authored: Wed Dec 17 15:07:38 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed Dec 17 15:08:43 2014 -0500

--
 .../AmbariManagementControllerImpl.java |   24 +-
 .../server/controller/ClusterRequest.java   |   31 +-
 .../server/controller/KerberosHelper.java   | 1117 ++
 .../internal/ClusterResourceProvider.java   |  114 ++
 .../kerberos/FinalizeKerberosServerAction.java  |   94 ++
 .../kerberos/KerberosServerAction.java  |  137 ++-
 .../AmbariManagementControllerImplTest.java |   17 +
 .../AmbariManagementControllerTest.java |3 +
 .../server/controller/KerberosHelperTest.java   |  350 ++
 .../internal/ComponentResourceProviderTest.java |2 +
 .../internal/HostResourceProviderTest.java  |5 +
 .../kerberos/KerberosServerActionTest.java  |2 -
 12 files changed, 1828 insertions(+), 68 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/ef341466/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 435c828..f58fa9d 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -207,6 +207,11 @@ public class AmbariManagementControllerImpl implements 
AmbariManagementControlle
 
   private MaintenanceStateHelper maintenanceStateHelper;
 
+  /**
+   * The KerberosHelper to help setup for enabling for disabling Kerberos
+   */
+  private KerberosHelper kerberosHelper;
+
   final private String masterHostname;
   final private Integer masterPort;
   final private String masterProtocol;
@@ -246,7 +251,7 @@ public class AmbariManagementControllerImpl implements 
AmbariManagementControlle
 LOG.info(Initializing the AmbariManagementControllerImpl);
 masterHostname =  InetAddress.getLocalHost().getCanonicalHostName();
 maintenanceStateHelper = 
injector.getInstance(MaintenanceStateHelper.class);
-
+kerberosHelper = injector.getInstance(KerberosHelper.class);
 if(configs != null)
 {
   if (configs.getApiSSLAuthentication()) {
@@ -1316,7 +1321,22 @@ public class AmbariManagementControllerImpl implements 
AmbariManagementControlle
   saveClusterUpdate(request, clusterResponse);
 }
 
-return null;
+RequestStageContainer requestStageContainer = null;
+MapString, Service services = cluster.getServices();
+if ((services != null)  services.containsKey(KERBEROS)) {
+  // Handle either adding or removing Kerberos from the cluster. This may 
generate multiple stages
+  // or not depending the current state of the cluster.  The main 
configuration used to determine
+  // whether Kerberos is to be added or removed is 
cluster-config/security_enabled.
+  requestStageContainer = kerberosHelper.toggleKerberos(cluster,
+  request.getKerberosDescriptor(), null);
+}
+
+if (requestStageContainer != null) {
+  requestStageContainer.persist();
+  return requestStageContainer.getRequestStatusResponse();
+} else {
+  return null;
+}
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/ambari/blob/ef341466/ambari-server/src/main/java/org/apache/ambari/server/controller/ClusterRequest.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/ClusterRequest.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/ClusterRequest.java
index 5dd0be0..8d2cce1 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/ClusterRequest.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/ClusterRequest.java
@@ -18,6 +18,8 @@
 
 package org.apache.ambari.server.controller;
 
+import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
+
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -42,6 +44,11 @@ public class ClusterRequest {
   private

ambari git commit: AMBARI-8756. Set session attributes prior to updating cluster

2014-12-17 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 2b09328fa - b950cea20


AMBARI-8756. Set session attributes prior to updating cluster


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b950cea2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b950cea2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b950cea2

Branch: refs/heads/trunk
Commit: b950cea20a04a895c5fcfd5dcc9d7b828f7a8437
Parents: 2b09328
Author: Robert Levas rle...@hortonworks.com
Authored: Wed Dec 17 19:57:20 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed Dec 17 19:59:14 2014 -0500

--
 .../AmbariManagementControllerImpl.java | 19 -
 .../AmbariManagementControllerImplTest.java | 12 ++-
 .../AmbariManagementControllerTest.java | 89 +++-
 .../internal/JMXHostProviderTest.java   | 16 ++--
 4 files changed, 81 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/b950cea2/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index f58fa9d..f9f2959 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -1146,9 +1146,24 @@ public class AmbariManagementControllerImpl implements 
AmbariManagementControlle
 // configuration updates (create multiple configuration resources)...
 for (ClusterRequest request : requests) {
   // TODO : Is there ever a real world case where we could have multiple 
non-null responses?
-  response = updateCluster(request);
+
+  // ***
   // set any session attributes for this cluster request
-  clusters.addSessionAttributes(request.getClusterName(), 
request.getSessionAttributes());
+  Long requestedClusterId = request.getClusterId();
+  if (requestedClusterId == null) {
+throw new AmbariException(The cluster ID may not be null);
+  }
+
+  Cluster cluster = clusters.getClusterById(request.getClusterId());
+  if (cluster == null) {
+throw new AmbariException(The cluster may not be null);
+  }
+
+  cluster.addSessionAttributes(request.getSessionAttributes());
+  //
+  // ***
+
+  response = updateCluster(request);
 }
 return response;
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/b950cea2/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
index 5208a2d..e713d7f 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
@@ -513,9 +513,13 @@ public class AmbariManagementControllerImplTest {
 expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null);
 
expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class));
 expect(clusterRequest.getClusterName()).andReturn(clusterNew).times(4);
-expect(clusterRequest.getClusterId()).andReturn(1L).times(4);
-expect(clusters.getClusterById(1L)).andReturn(cluster);
+expect(clusterRequest.getClusterId()).andReturn(1L).times(6);
+expect(clusters.getClusterById(1L)).andReturn(cluster).times(2);
 expect(cluster.getClusterName()).andReturn(clusterOld).times(2);
+
+cluster.addSessionAttributes(anyObject(Map.class));
+expectLastCall().once();
+
 cluster.setClusterName(clusterNew);
 expectLastCall();
 
@@ -553,8 +557,8 @@ public class AmbariManagementControllerImplTest {
 expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null);
 
expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class));
 expect(clusterRequest.getClusterName()).andReturn(clusterNew).times(4);
-expect(clusterRequest.getClusterId()).andReturn(1L).times(4);
-expect(clusters.getClusterById(1L)).andReturn(cluster);
+

ambari git commit: AMBARI-8725. Inject Clusters object into KerberosServerAction

2014-12-16 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 828dc79fa - 254280c3c


AMBARI-8725. Inject Clusters object into KerberosServerAction


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/254280c3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/254280c3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/254280c3

Branch: refs/heads/trunk
Commit: 254280c3cf6f0f78bc178ad10b8c7ec563e656b3
Parents: 828dc79
Author: Robert Levas rle...@hortonworks.com
Authored: Tue Dec 16 12:42:39 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Tue Dec 16 12:45:16 2014 -0500

--
 .../kerberos/KerberosServerAction.java  | 68 +++---
 .../kerberos/KerberosServerActionTest.java  | 96 
 2 files changed, 92 insertions(+), 72 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/254280c3/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java
index 832602f..a3a1b5b 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java
@@ -18,10 +18,13 @@
 
 package org.apache.ambari.server.serveraction.kerberos;
 
+import com.google.inject.Inject;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.agent.CommandReport;
 import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -49,23 +52,10 @@ public abstract class KerberosServerAction extends 
AbstractServerAction {
   public static final String DATA_DIRECTORY = data_directory;
 
   /**
-   * A (command parameter) property name used to hold the KDC administrator's 
principal value.
-   * TODO (rlevas): For security purposes, this data should be moved to an 
encrypted storage facility.
+   * A (command parameter) property name used to hold encrypted data 
representing the KDC
+   * administrator credentials
*/
-  public static final String ADMINISTRATOR_PRINCIPAL = admin_principal;
-
-  /**
-   * A (command parameter) property name used to hold the KDC administrator's 
password value.
-   * TODO (rlevas): For security purposes, this data should be moved to an 
encrypted storage facility.
-   */
-  public static final String ADMINISTRATOR_PASSWORD = admin_password;
-
-  /**
-   * A (command parameter) property name used to hold the KDC administrator's 
(base64-encoded) keytab
-   * value.
-   * TODO (rlevas): For security purposes, this data should be moved to an 
encrypted storage facility.
-   */
-  public static final String ADMINISTRATOR_KEYTAB = admin_keytab;
+  public static final String ADMINISTRATOR_CREDENTIAL = 
kerberos_admin_credential;
 
   /**
* A (command parameter) property name used to hold the default Kerberos 
realm value.
@@ -86,6 +76,12 @@ public abstract class KerberosServerAction extends 
AbstractServerAction {
   private static final Logger LOG = 
LoggerFactory.getLogger(KerberosServerAction.class);
 
   /**
+   * The Cluster that this ServerAction implementation is executing on
+   */
+  @Inject
+  private Clusters clusters = null;
+
+  /**
* Given a (command parameter) Map and a property name, attempts to safely 
retrieve the requested
* data.
*
@@ -135,25 +131,6 @@ public abstract class KerberosServerAction extends 
AbstractServerAction {
   }
 
   /**
-   * Given a (command parameter) Map, attempts to safely retrieve the data 
needed to create a
-   * {@link org.apache.ambari.server.serveraction.kerberos.KerberosCredential} 
representing a KDC
-   * administrator.
-   * p/
-   * TODO (rlevas): For security purposes, this data should be moved to an 
encrypted storage facility.
-   *
-   * @param commandParameters a Map containing the dictionary of data to 
interrogate
-   * @return a KerberosCredential or null if commandParameters is null
-   */
-  protected static KerberosCredential getAdministratorCredential(MapString, 
String commandParameters) {
-return (commandParameters == null)
-? null
-: new KerberosCredential(
-commandParameters.get(ADMINISTRATOR_PRINCIPAL),
-commandParameters.get(ADMINISTRATOR_PASSWORD),
-commandParameters.get(ADMINISTRATOR_KEYTAB));

ambari git commit: AMBARI-8660. Provide encrypt and decrypt functionality in KerberosCredential

2014-12-15 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk bde8a5bb1 - 93fc4221d


AMBARI-8660. Provide encrypt and decrypt functionality in KerberosCredential


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/93fc4221
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/93fc4221
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/93fc4221

Branch: refs/heads/trunk
Commit: 93fc4221dd3dda65115dc366798d840766377b9f
Parents: bde8a5b
Author: Robert Levas rle...@hortonworks.com
Authored: Mon Dec 15 10:50:38 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Dec 15 10:52:13 2014 -0500

--
 .../kerberos/KerberosCredential.java| 181 +++
 .../kerberos/KerberosCredentialTest.java| 103 +++
 2 files changed, 284 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/93fc4221/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosCredential.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosCredential.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosCredential.java
index adb2edc..19997e7 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosCredential.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosCredential.java
@@ -17,16 +17,183 @@
  */
 package org.apache.ambari.server.serveraction.kerberos;
 
+import com.google.gson.Gson;
+import com.google.gson.JsonSyntaxException;
+import org.apache.ambari.server.AmbariException;
+import org.apache.commons.codec.binary.Base64;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.SecretKeySpec;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Map;
+
 /**
  * KerberosCredential encapsulates data needed to authenticate an identity to 
a KDC.
+ * p/
+ * This class has the ability to encrypt and decrypt itself using the AES 
encryption algorithm.
  */
 public class KerberosCredential {
 
+  /**
+   * A property name used to hold the KDC administrator's principal value.
+   */
+  public static final String KEY_NAME_PRINCIPAL = principal;
+  /**
+   * A property name used to hold the KDC administrator's password value.
+   */
+  public static final String KEY_NAME_PASSWORD = password;
+  /**
+   * A property name used to hold the KDC administrator's (base64-encoded) 
keytab
+   * value.
+   */
+  public static final String KEY_NAME_KEYTAB = keytab;
+
+  /**
+   * This principal value
+   */
   private String principal = null;
+
+  /**
+   * The plaintext password value
+   */
   private String password = null;
+
+  /**
+   * A base64-encoded keytab
+   */
   private String keytab = null;
 
   /**
+   * Given a Map of attributes, attempts to safely retrieve the data needed to 
create a
+   * KerberosCredential representing a KDC administrator.
+   * p/
+   * It is expected that the following properties exist in the Map:
+   * ul
+   * liprincipal/li
+   * lipassword (optional)/li
+   * likeytab (optional)/li
+   * /ul
+   * p/
+   * Each of these properties may be prefixed with some prefix value to 
generate a relevant key value.
+   * If prefix was kerberos_admin/, then the key representing the principal 
would be computed
+   * to be kerberos_admin/principal.
+   *
+   * @param mapa Map of attributes containing the values needed to create 
a new KerberosCredential
+   * @param prefix a String containing the prefix to used along with the base 
key name (principal, etc...)
+   *   to create the relevant key name ([prefix]base_key. etc...)
+   * @return a KerberosCredential or null if commandParameters is null
+   */
+  public static KerberosCredential fromMap(MapString, Object map, String 
prefix) {
+KerberosCredential credential = null;
+
+if (map != null) {
+  Object attribute;
+  String principal;
+  String password;
+  String keytab;
+
+  if (prefix == null) {
+prefix = ;
+  }
+
+  attribute = map.get(prefix + KEY_NAME_PRINCIPAL);
+  principal = (attribute == null) ? null : attribute.toString();
+
+  attribute = map.get(prefix + KEY_NAME_PASSWORD);
+  password = (attribute == null) ? null : attribute.toString();
+
+  attribute = map.get(prefix + KEY_NAME_KEYTAB);
+  keytab = (attribute == null) ? null : attribute.toString();
+
+  if (((principal != null)  !principal.isEmpty()) ||
+  ((password !=

ambari git commit: AMBARI-8247. Expose stack and service kerberos descriptors via REST API * Note, this patch doesn't expose new resources for these descriptors but returns them as new properties in t

2014-12-12 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk bf499456c - b62846d25


AMBARI-8247.  Expose stack and service kerberos descriptors via REST API
* Note, this patch doesn't expose new resources for these descriptors
  but returns them as new properties in the existing stack and service
  resources.  This is temporary and these properties will be removed
  shortly, before the 2.0 release, and replaced with proper sub-resources


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b62846d2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b62846d2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b62846d2

Branch: refs/heads/trunk
Commit: b62846d25f4a3d65ee3e01a9d1b03697548a96bc
Parents: bf49945
Author: Robert Levas rle...@hortonworks.com
Authored: Fri Dec 12 22:25:05 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Fri Dec 12 22:25:05 2014 -0500

--
 .../server/api/services/AmbariMetaInfo.java |   5 +
 .../server/controller/StackServiceResponse.java |  29 
 .../server/controller/StackVersionResponse.java |  64 +++-
 .../internal/StackServiceResourceProvider.java  |  34 +
 .../internal/StackVersionResourceProvider.java  |  63 
 .../ambari/server/stack/ServiceDirectory.java   |  19 +++
 .../ambari/server/stack/ServiceModule.java  |   4 +
 .../ambari/server/stack/StackDirectory.java |  20 +++
 .../apache/ambari/server/stack/StackModule.java |   1 +
 .../apache/ambari/server/state/ServiceInfo.java |  19 ++-
 .../apache/ambari/server/state/StackInfo.java   |  44 +-
 .../src/main/resources/properties.json  |   2 +
 .../server/api/services/AmbariMetaInfoTest.java |  58 
 .../ambari/server/stack/ServiceModuleTest.java  |  38 +
 .../HDP/2.0.8/services/HDFS/kerberos.json   | 147 +++
 .../stacks/HDP/2.1.1/services/HDFS/metainfo.xml | 146 ++
 16 files changed, 689 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
index 60d4f9b..2dd3351 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
@@ -91,6 +91,11 @@ public class AmbariMetaInfo {
   public static final String SERVICE_ALERT_FILE_NAME = alerts.json;
 
   /**
+   * The filename name for a Kerberos descriptor file at either the stack or 
service level
+   */
+  public static final String KERBEROS_DESCRIPTOR_FILE_NAME = kerberos.json;
+
+  /**
* This string is used in placeholder in places that are common for
* all operating systems or in situations where os type is not important.
*/

http://git-wip-us.apache.org/repos/asf/ambari/blob/b62846d2/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
index 2794c18..0eecb3f 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
@@ -21,6 +21,7 @@ package org.apache.ambari.server.controller;
 import org.apache.ambari.server.state.CustomCommandDefinition;
 import org.apache.ambari.server.state.ServiceInfo;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -42,6 +43,13 @@ public class StackServiceResponse {
   private ListString requiredServices;
 
   /**
+   * A File pointing to the service-level Kerberos descriptor file
+   *
+   * This may be null if a relevant file is not available.
+   */
+  private File kerberosDescriptorFile;
+
+  /**
* Constructor.
*
* @param service
@@ -67,6 +75,8 @@ public class StackServiceResponse {
 customCommands.add(command.getName());
   }
 }
+
+kerberosDescriptorFile = service.getKerberosDescriptorFile();
   }
 
   public String getStackName() {
@@ -138,6 +148,25 @@ public class StackServiceResponse {
   }
 
   /**
+   * Gets a File pointing to the service-level Kerberos descriptor
+   *
+   * @return a File pointing to the service-level Kerberos descriptor, or null 
if no relevant file is
+

ambari git commit: AMBARI-8657. Add Kerberos Configuration Metadata File Builder and Reader

2014-12-12 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk b62846d25 - 3c08f6f7d


AMBARI-8657. Add Kerberos Configuration Metadata File Builder and Reader


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3c08f6f7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3c08f6f7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3c08f6f7

Branch: refs/heads/trunk
Commit: 3c08f6f7d724dd566ebfb684fcf6e8acb598d27e
Parents: b62846d
Author: Robert Levas rle...@hortonworks.com
Authored: Fri Dec 12 23:24:01 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Fri Dec 12 23:24:01 2014 -0500

--
 .../AbstractKerberosDataFileBuilder.java| 124 
 .../AbstractKerberosDataFileReader.java | 116 
 .../kerberos/CreateKeytabFilesServerAction.java |  11 +-
 .../kerberos/KerberosActionDataFile.java|   5 +-
 .../kerberos/KerberosActionDataFileBuilder.java |  91 ++
 .../kerberos/KerberosActionDataFileReader.java  |  75 +
 .../kerberos/KerberosConfigDataFile.java|  31 ++
 .../kerberos/KerberosConfigDataFileBuilder.java |  65 +
 .../kerberos/KerberosConfigDataFileReader.java  |  44 +++
 .../kerberos/KerberosServerAction.java  |   4 +-
 .../kerberos/KerberosActionDataFileTest.java| 286 ++-
 .../kerberos/KerberosConfigDataFileTest.java| 146 ++
 12 files changed, 705 insertions(+), 293 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/3c08f6f7/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractKerberosDataFileBuilder.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractKerberosDataFileBuilder.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractKerberosDataFileBuilder.java
new file mode 100644
index 000..cc2e2ff
--- /dev/null
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractKerberosDataFileBuilder.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.kerberos;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+/**
+ * AbstractKerberosDataFileBuilder provides a generic facility to write a data 
file using some
+ * underlying record-based file writer.
+ * p/
+ * This class encapsulates a {@link org.apache.commons.csv.CSVPrinter} to 
create a CSV-formatted file.
+ */
+public abstract class AbstractKerberosDataFileBuilder {
+
+  private File file;
+  private CSVPrinter csvPrinter;
+
+  /**
+   * Creates a new KerberosConfigDataFileBuilder
+   * p/
+   * The file is opened upon creation, so there is no need to manually open it 
unless manually
+   * closed before using.
+   *
+   * @param file a File declaring where to write the data
+   * @throws java.io.IOException
+   */
+  public AbstractKerberosDataFileBuilder(File file) throws IOException {
+this.file = file;
+open();
+  }
+
+
+  /**
+   * Opens the data file for writing.
+   * p/
+   * This may be called multiple times and the appropriate action will occur 
depending on if the
+   * file has been previously opened or closed.
+   *
+   * @throws java.io.IOException
+   */
+  public void open() throws IOException {
+if (isClosed()) {
+  if (file == null) {
+throw new IOException(Missing file path);
+  } else {
+csvPrinter = new CSVPrinter(new FileWriter(file, true), 
CSVFormat.DEFAULT);
+
+// If the file is empty, write the header; else don't write the header.
+if (file.length() == 0) {
+  // Write the header
+  Iterable? headerRecord = getHeaderRecord();
+  csvPrinter.printRecord(headerRecord);
+}
+  }
+}
+  }
+
+  /**
+   * Tests this KerberosConfigDataFileBuilder to see

ambari git commit: AMBARI-8600. Fix Stack Inheritence where explicitly inherited service has deleted components

2014-12-11 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 24bb3 - 4772593c5


AMBARI-8600. Fix Stack Inheritence where explicitly inherited service has 
deleted components


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4772593c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4772593c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4772593c

Branch: refs/heads/trunk
Commit: 4772593c58ff84809215d55f65867c392ef67693
Parents: 24bb344
Author: Jayush Luniya jlun...@hortonworks.com
Authored: Thu Dec 11 12:23:30 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Dec 11 12:26:05 2014 -0500

--
 .../ambari/server/stack/ServiceModule.java  |  6 +-
 .../ambari/server/stack/StackManager.java   | 13 ++-
 .../apache/ambari/server/stack/StackModule.java |  4 +-
 .../AmbariManagementControllerTest.java |  2 +-
 .../ambari/server/stack/StackManagerTest.java   |  9 +-
 .../stacks/HDP/2.0.6.1/hooks/dummy-script.py| 20 +
 .../resources/stacks/HDP/2.0.6.1/metainfo.xml   | 24 +
 .../stacks/HDP/2.0.6.1/repos/repoinfo.xml   | 61 +
 .../stacks/HDP/2.0.6.1/role_command_order.json  | 92 
 .../services/FLUME/configuration/flume-conf.xml | 31 +++
 .../services/FLUME/configuration/flume-env.xml  | 40 +
 .../FLUME/configuration/flume-log4j.xml | 31 +++
 .../HDP/2.0.6.1/services/FLUME/metainfo.xml | 69 +++
 .../HDP/2.0.6.1/services/SQOOP/metainfo.xml | 34 
 .../HDP/2.0.6.1/services/YARN/metainfo.xml  | 27 ++
 15 files changed, 454 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/4772593c/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java 
b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
index d025946..ae7dd28 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
@@ -295,8 +295,10 @@ public class ServiceModule extends 
BaseModuleServiceModule, ServiceInfo {
 
 for (ConfigurationModule module : mergedModules) {
   configurationModules.put(module.getId(), module);
-  
serviceInfo.getProperties().addAll(module.getModuleInfo().getProperties());
-  serviceInfo.setTypeAttributes(module.getConfigType(), 
module.getModuleInfo().getAttributes());
+  if(!module.isDeleted()) {
+
serviceInfo.getProperties().addAll(module.getModuleInfo().getProperties());
+serviceInfo.setTypeAttributes(module.getConfigType(), 
module.getModuleInfo().getAttributes());
+  }
 }
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4772593c/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java 
b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
index 62a1987..0179abb 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
@@ -149,12 +149,23 @@ public class StackManager {
   private void fullyResolveStacks(
   MapString, StackModule stackModules, MapString, ServiceModule 
commonServiceModules)
   throws AmbariException {
+// Resolve all stacks without finalizing the stacks.
 for (StackModule stack : stackModules.values()) {
   if (stack.getModuleState() == ModuleState.INIT) {
 stack.resolve(null, stackModules, commonServiceModules);
   }
 }
-// execute all of the repo tasks in a single thread executor
+// Finalize the common services and stacks to remove sub-modules marked 
for deletion.
+// Finalizing the stacks AFTER all stacks are resolved ensures that the 
sub-modules marked for deletion are
+// inherited into the child module when explicit parent is defined and 
thereby ensuring all modules from parent module
+// are inlined into the child module even if the module is marked for 
deletion.
+for(ServiceModule commonService : commonServiceModules.values()) {
+  commonService.finalizeModule();
+}
+for (StackModule stack : stackModules.values()) {
+  stack.finalizeModule();
+}
+// Execute all of the repo tasks in a single thread executor
 stackContext.executeRepoTasks();
   }

[1/3] ambari git commit: AMBARI-7202: Add functionality for Ambari Common Services

2014-12-10 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk f6248ae5f - 14c869961


http://git-wip-us.apache.org/repos/asf/ambari/blob/14c86996/ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/HDFS/configuration/global.xml
--
diff --git 
a/ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/HDFS/configuration/global.xml
 
b/ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/HDFS/configuration/global.xml
new file mode 100644
index 000..9f4eae5
--- /dev/null
+++ 
b/ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/HDFS/configuration/global.xml
@@ -0,0 +1,145 @@
+?xml version=1.0?
+?xml-stylesheet type=text/xsl href=configuration.xsl?
+!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+--
+
+configuration
+  property
+namenamenode_host/name
+value/value
+descriptionNameNode Host./description
+  /property
+  property
+namedfs_name_dir/name
+value/hadoop/hdfs/namenode/value
+descriptionNameNode Directories./description
+  /property
+  property
+namesnamenode_host/name
+value/value
+descriptionSecondary NameNode./description
+  /property
+  property
+namefs_checkpoint_dir/name
+value/hadoop/hdfs/namesecondary/value
+descriptionSecondary NameNode checkpoint dir./description
+  /property
+  property
+namedatanode_hosts/name
+value/value
+descriptionList of Datanode Hosts./description
+  /property
+  property
+namedfs_data_dir/name
+value/hadoop/hdfs/data/value
+descriptionData directories for Data Nodes./description
+  /property
+  property
+namehdfs_log_dir_prefix/name
+value/var/log/hadoop/value
+descriptionHadoop Log Dir Prefix/description
+  /property
+  property
+namehadoop_pid_dir_prefix/name
+value/var/run/hadoop/value
+descriptionHadoop PID Dir Prefix/description
+  /property
+  property
+namedfs_webhdfs_enabled/name
+valuetrue/value
+descriptionWebHDFS enabled/description
+  /property
+  property
+namehadoop_heapsize/name
+value1024/value
+descriptionHadoop maximum Java heap size/description
+  /property
+  property
+namenamenode_heapsize/name
+value1024/value
+descriptionNameNode Java heap size/description
+  /property
+  property
+namenamenode_opt_newsize/name
+value200/value
+descriptionNameNode new generation size/description
+  /property
+  property
+namenamenode_opt_maxnewsize/name
+value640/value
+descriptionNameNode maximum new generation size/description
+  /property
+  property
+namenamenode_opt_permsize/name
+value128/value
+descriptionNameNode permanent generation size/description
+  /property
+  property
+namenamenode_opt_maxpermsize/name
+value256/value
+descriptionNameNode maximum permanent generation size/description
+  /property
+  property
+namedatanode_du_reserved/name
+value1/value
+descriptionReserved space for HDFS/description
+  /property
+  property
+namedtnode_heapsize/name
+value1024/value
+descriptionDataNode maximum Java heap size/description
+  /property
+  property
+namedfs_datanode_failed_volume_tolerated/name
+value0/value
+descriptionDataNode volumes failure toleration/description
+  /property
+  property
+namefs_checkpoint_period/name
+value21600/value
+descriptionHDFS Maximum Checkpoint Delay/description
+  /property
+  property
+namefs_checkpoint_size/name
+value0.5/value
+descriptionFS Checkpoint Size./description
+  /property
+  property
+namesecurity_enabled/name
+valuefalse/value
+descriptionHadoop Security/description
+  /property
+  property
+namekerberos_domain/name
+valueEXAMPLE.COM/value
+descriptionKerberos realm./description
+  /property
+  property
+namekerberos_domain/name
+valueEXAMPLE.COM/value
+descriptionKerberos realm./description
+  /property
+  property
+namekeytab_path/name
+value/etc/security/keytabs/value
+descriptionKeyTab Directory./description
+  /property
+  
+/configuration

[3/3] ambari git commit: AMBARI-7202: Add functionality for Ambari Common Services

2014-12-10 Thread jspeidel

AMBARI-7202: Add functionality for Ambari Common Services


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/14c86996
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/14c86996
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/14c86996

Branch: refs/heads/trunk
Commit: 14c869961a4bc5cb9ffa041d8ef79ade33bb
Parents: f6248ae
Author: Jayush Luniya jlun...@hortonworks.com
Authored: Wed Dec 10 21:24:53 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed Dec 10 21:26:43 2014 -0500

--
 ambari-server/conf/unix/ambari.properties   |   1 +
 ambari-server/conf/windows/ambari.properties|   1 +
 .../server/api/services/AmbariMetaInfo.java |  10 +-
 .../server/configuration/Configuration.java |   9 +
 .../apache/ambari/server/stack/BaseModule.java  |  28 +-
 .../ambari/server/stack/ComponentModule.java|   2 +-
 .../server/stack/ConfigurationModule.java   |   2 +-
 .../apache/ambari/server/stack/ModuleState.java |  29 ++
 .../ambari/server/stack/ServiceModule.java  |  85 +++-
 .../server/stack/StackDefinitionModule.java |  17 +-
 .../ambari/server/stack/StackManager.java   | 176 ++--
 .../apache/ambari/server/stack/StackModule.java | 193 ++---
 .../server/api/services/AmbariMetaInfoTest.java |   2 +-
 .../services/KerberosServiceMetaInfoTest.java   |   2 +-
 .../AmbariManagementControllerTest.java |   2 +-
 .../server/stack/ComponentModuleTest.java   |   2 +-
 .../ambari/server/stack/ServiceModuleTest.java  |   2 +-
 .../stack/StackManagerCommonServicesTest.java   | 167 
 .../server/stack/StackManagerMiscTest.java  |  91 +
 .../ambari/server/stack/StackManagerTest.java   |  80 +---
 .../HBASE/1.0/configuration/hbase-site.xml  | 137 +++
 .../common-services/HBASE/1.0/metainfo.xml  | 121 ++
 .../HDFS/1.0/configuration/hbase-site.xml   | 137 +++
 .../HDFS/1.0/configuration/hdfs-site.xml| 396 ++
 .../common-services/HDFS/1.0/metainfo.xml   | 133 ++
 .../common-services/HIVE/1.0/metainfo.xml   | 127 ++
 .../MAPREDUCE/1.0/configuration/hbase-site.xml  | 137 +++
 .../MAPREDUCE/1.0/configuration/hdfs-site.xml   | 396 ++
 .../MAPREDUCE/1.0/configuration/mapred-site.xml | 400 +++
 .../common-services/MAPREDUCE/1.0/metainfo.xml  |  89 +
 .../PIG/1.0/configuration/pig.properties|  52 +++
 .../common-services/PIG/1.0/metainfo.xml|  61 +++
 .../common-services/ZOOKEEPER/1.0/metainfo.xml  |  72 
 .../HDP/0.1/metainfo.xml|  22 +
 .../HDP/0.1/repos/repoinfo.xml  |  57 +++
 .../HDP/0.1/services/HDFS/metainfo.xml  |  46 +++
 .../HDP/0.1/services/MAPREDUCE/metainfo.xml |  23 ++
 .../HDP/0.1/services/PIG/metainfo.xml   |  26 ++
 .../HDP/0.2/metainfo.xml|  22 +
 .../HDP/0.2/repos/repoinfo.xml  |  57 +++
 .../HDP/0.2/services/HBASE/metainfo.xml |  26 ++
 .../0.2/services/HDFS/configuration/global.xml  | 145 +++
 .../services/HDFS/configuration/hadoop-env.xml  | 223 +++
 .../services/HDFS/configuration/hbase-site.xml  | 137 +++
 .../services/HDFS/configuration/hdfs-log4j.xml  | 199 +
 .../services/HDFS/configuration/hdfs-site.xml   | 396 ++
 .../HDP/0.2/services/HDFS/metainfo.xml  |  30 ++
 .../HDP/0.2/services/HIVE/metainfo.xml  |  26 ++
 .../HDP/0.2/services/MAPREDUCE/metainfo.xml |  23 ++
 .../HDP/0.2/services/ZOOKEEPER/metainfo.xml |  26 ++
 50 files changed, 4452 insertions(+), 191 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/14c86996/ambari-server/conf/unix/ambari.properties
--
diff --git a/ambari-server/conf/unix/ambari.properties 
b/ambari-server/conf/unix/ambari.properties
index 9ef645b..fd1818b 100644
--- a/ambari-server/conf/unix/ambari.properties
+++ b/ambari-server/conf/unix/ambari.properties
@@ -27,6 +27,7 @@ 
jce_policy1.6.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-6.zi
 
jdk1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-7u67-linux-x64.tar.gz
 
jce_policy1.7.url=http://public-repo-1.hortonworks.com/ARTIFACTS/UnlimitedJCEPolicyJDK7.zip
 metadata.path=/var/lib/ambari-server/resources/stacks
+common.services.path=
 server.version.file=/var/lib/ambari-server/resources/version
 webapp.dir=/usr/lib/ambari-server/web
 bootstrap.dir=/var/run/ambari-server/bootstrap

http://git-wip-us.apache.org/repos/asf/ambari/blob/14c86996/ambari-server/conf/windows/ambari.properties
--
diff --git a/ambari-server/conf/windows/ambari.properties

[2/3] ambari git commit: AMBARI-7202: Add functionality for Ambari Common Services

2014-12-10 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/14c86996/ambari-server/src/test/resources/common-services/HDFS/1.0/configuration/hbase-site.xml
--
diff --git 
a/ambari-server/src/test/resources/common-services/HDFS/1.0/configuration/hbase-site.xml
 
b/ambari-server/src/test/resources/common-services/HDFS/1.0/configuration/hbase-site.xml
new file mode 100644
index 000..5024e85
--- /dev/null
+++ 
b/ambari-server/src/test/resources/common-services/HDFS/1.0/configuration/hbase-site.xml
@@ -0,0 +1,137 @@
+?xml version=1.0?
+?xml-stylesheet type=text/xsl href=configuration.xsl?
+!--
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+--
+configuration
+  property
+namehbase.regionserver.msginterval/name
+value1000/value
+descriptionInterval between messages from the RegionServer to HMaster
+in milliseconds.  Default is 15. Set this value low if you want unit
+tests to be responsive.
+/description
+  /property
+  property
+namehbase.client.pause/name
+value5000/value
+descriptionGeneral client pause value.  Used mostly as value to wait
+before running a retry of a failed get, region lookup, etc./description
+  /property
+  property
+namehbase.master.meta.thread.rescanfrequency/name
+value1/value
+descriptionHow long the HMaster sleeps (in milliseconds) between scans of
+the root and meta tables.
+/description
+  /property
+  property
+namehbase.server.thread.wakefrequency/name
+value1000/value
+descriptionTime to sleep in between searches for work (in milliseconds).
+Used as sleep interval by service threads such as META scanner and log 
roller.
+/description
+  /property
+  property
+namehbase.regionserver.handler.count/name
+value5/value
+descriptionCount of RPC Server instances spun up on RegionServers
+Same property is used by the HMaster for count of master handlers.
+Default is 10.
+/description
+  /property
+  property
+namehbase.master.lease.period/name
+value6000/value
+descriptionLength of time the master will wait before timing out a region
+server lease. Since region servers report in every second (see above), this
+value has been reduced so that the master will notice a dead region server
+sooner. The default is 30 seconds.
+/description
+  /property
+  property
+namehbase.master.info.port/name
+value-1/value
+descriptionThe port for the hbase master web UI
+Set to -1 if you do not want the info server to run.
+/description
+  /property
+  property
+namehbase.regionserver.info.port/name
+value-1/value
+descriptionThe port for the hbase regionserver web UI
+Set to -1 if you do not want the info server to run.
+/description
+  /property
+  property
+namehbase.regionserver.info.port.auto/name
+valuetrue/value
+descriptionInfo server auto port bind. Enables automatic port
+search if hbase.regionserver.info.port is already in use.
+Enabled for testing to run multiple tests on one machine.
+/description
+  /property
+  property
+namehbase.master.lease.thread.wakefrequency/name
+value3000/value
+descriptionThe interval between checks for expired region server leases.
+This value has been reduced due to the other reduced values above so that
+the master will notice a dead region server sooner. The default is 15 
seconds.
+/description
+  /property
+  property
+namehbase.regionserver.optionalcacheflushinterval/name
+value1/value
+description
+Amount of time to wait since the last time a region was flushed before
+invoking an optional cache flush. Default 60,000.
+/description
+  /property
+  property
+namehbase.regionserver.safemode/name
+valuefalse/value
+description
+Turn on/off safe mode in region server. Always on for production, always 
off
+for tests.
+/description
+  /property
+  property
+namehbase.hregion.max.filesize/name
+value67108864/value
+description
+Maximum desired file size for an HRegion.  If filesize exceeds
+value +

ambari git commit: AMBARI-8578. Update Apache Directory Server Library from 1.5.5 to 2.0.0-M19

2014-12-08 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 3ba1889f4 - 74f9ddcf8


AMBARI-8578. Update Apache Directory Server Library from 1.5.5 to 2.0.0-M19


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/74f9ddcf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/74f9ddcf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/74f9ddcf

Branch: refs/heads/trunk
Commit: 74f9ddcf8b0d9a8c0650de7352015745e8e2af28
Parents: 3ba1889
Author: Robert Levas rle...@hortonworks.com
Authored: Mon Dec 8 14:06:00 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Dec 8 14:07:27 2014 -0500

--
 ambari-project/pom.xml  | 35 +-
 ambari-server/pom.xml   | 27 +++
 ...mbariLdapAuthenticationProviderBaseTest.java | 48 +++
 ...uthenticationProviderForDNWithSpaceTest.java | 47 ---
 .../AmbariLdapAuthenticationProviderTest.java   | 49 +---
 ambari-server/src/test/resources/users.ldif |  2 +-
 .../test/resources/users_for_dn_with_space.ldif |  2 +-
 7 files changed, 154 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/74f9ddcf/ambari-project/pom.xml
--
diff --git a/ambari-project/pom.xml b/ambari-project/pom.xml
index edba1dc..930080e 100644
--- a/ambari-project/pom.xml
+++ b/ambari-project/pom.xml
@@ -186,13 +186,44 @@
   /dependency
   dependency
 groupIdorg.apache.directory.server/groupId
+artifactIdapacheds-server-annotations/artifactId
+version2.0.0-M19/version
+  /dependency
+  dependency
+groupIdorg.apache.directory.server/groupId
+artifactIdapacheds-core-integ/artifactId
+version2.0.0-M19/version
+  /dependency
+  dependency
+groupIdorg.apache.directory.server/groupId
+artifactIdapacheds-server-integ/artifactId
+version2.0.0-M19/version
+  /dependency
+  dependency
+groupIdorg.apache.directory.server/groupId
+artifactIdapacheds-jdbm/artifactId
+version2.0.0-M5/version
+  /dependency
+  dependency
+groupIdorg.apache.directory.server/groupId
+artifactIdapacheds-kerberos-codec/artifactId
+version2.0.0-M19/version
+  /dependency
+  dependency
+groupIdorg.apache.directory.server/groupId
 artifactIdapacheds-core/artifactId
-version1.5.5/version
+version2.0.0-M19/version
   /dependency
   dependency
 groupIdorg.apache.directory.server/groupId
 artifactIdapacheds-protocol-ldap/artifactId
-version1.5.5/version
+version2.0.0-M19/version
+exclusions
+  exclusion
+groupIdorg.apache.directory.jdbm/groupId
+artifactIdapacheds-jdbm1/artifactId
+  /exclusion
+/exclusions
   /dependency
   dependency
 groupIdorg.apache.directory.shared/groupId

http://git-wip-us.apache.org/repos/asf/ambari/blob/74f9ddcf/ambari-server/pom.xml
--
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index 4076d73..b11538d 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -1433,15 +1433,42 @@
 /dependency
 dependency
   groupIdorg.apache.directory.server/groupId
+  artifactIdapacheds-server-annotations/artifactId
+  scopetest/scope
+/dependency
+dependency
+  groupIdorg.apache.directory.server/groupId
+  artifactIdapacheds-core-integ/artifactId
+  scopetest/scope
+/dependency
+dependency
+  groupIdorg.apache.directory.server/groupId
+  artifactIdapacheds-server-integ/artifactId
+  scopetest/scope
+/dependency
+dependency
+  groupIdorg.apache.directory.server/groupId
+  artifactIdapacheds-jdbm/artifactId
+  scopetest/scope
+/dependency
+dependency
+  groupIdorg.apache.directory.server/groupId
+  artifactIdapacheds-kerberos-codec/artifactId
+/dependency
+dependency
+  groupIdorg.apache.directory.server/groupId
   artifactIdapacheds-core/artifactId
+  scopetest/scope
 /dependency
 dependency
   groupIdorg.apache.directory.server/groupId
   artifactIdapacheds-protocol-ldap/artifactId
+  scopetest/scope
 /dependency
 dependency
   groupIdorg.apache.directory.shared/groupId
   artifactIdshared-ldap/artifactId
+  scopetest/scope
 /dependency
 dependency
   groupIdorg.slf4j/groupId

[2/3] ambari git commit: AMBARI-8542. Provide a way to parse and handle Kerberos descriptors

2014-12-08 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/e7b8383b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
new file mode 100644
index 000..59e6104
--- /dev/null
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
@@ -0,0 +1,445 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.kerberos;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * KerberosKeytabDescriptor is an implementation of an 
AbstractKerberosDescriptor that
+ * encapsulates data related to a Kerberos keytab file.  This class is 
typically associated with a
+ * KerberosPrincipalDescriptor via a KerberosIdentityDescriptor.
+ * p/
+ * A KerberosKeytabDescriptor has the following properties:
+ * ul
+ * lifile/li
+ * liowner {name, access}/li
+ * ligroup {name, access}/li
+ * liconfiguration/li
+ * /ul
+ * p/
+ * The following JSON Schema will yield a valid KerberosPrincipalDescriptor
+ * pre
+ *   {
+ *  $schema: http://json-schema.org/draft-04/schema#;,
+ *  title: KerberosKeytabDescriptor,
+ *  description: Describes a Kerberos keytab file and associated 
details,
+ *  type: object,
+ *  properties: {
+ *file: {
+ *  description: The absolute path for the keytab file,
+ *  type: string
+ *},
+ *owner: {
+ *  description: Details about the file's user ownership,
+ *  type: object,
+ *  properties: {
+ *name: {
+ *  description: The local username that should be set as the 
owner of this file,
+ *  type: string
+ *},
+ *access: {
+ *  description: The relevant access permissions that should be 
set for the owner of
+ *  this file. Expected values are 'rw', 'r', ''
+ *  type: string
+ *}
+ *  }
+ *}
+ *group: {
+ *  description: Details about the file's group ownership,
+ *  type: object,
+ *  properties: {
+ *name: {
+ *  description: The local group name that should be set as the 
group owner of this file,
+ *  type: string
+ *},
+ *access: {
+ *  description: The relevant access permissions that should be 
set for the group
+ *  owner of this file. Expected values are 'rw', 
'r', ''
+ *  type: string
+ *}
+ *  }
+ *}
+ *configuration: {
+ *  description: The configuration type and property name 
indicating the property to be
+ *  updated with the generated absolute path to the 
keytab file
+ *  - format: config-type/property.name,
+ *  type: string
+ *}
+ *  }
+ *   }
+ * /pre
+ * p/
+ * In this implementation,
+ * {@link 
org.apache.ambari.server.state.kerberos.AbstractKerberosDescriptor#name} will 
hold the
+ * KerberosKeytabDescriptor#file value
+ */
+public class KerberosKeytabDescriptor extends AbstractKerberosDescriptor {
+
+  /**
+   * A String declaring the local username that should be set as the owner of 
the keytab file
+   */
+  private String ownerName = null;
+
+  /**
+   * A String declaring the access permissions that should be set on the 
keytab file related to the
+   * owner.
+   * p/
+   * Expected values are:
+   * ul
+   * lirw - read/write/li
+   * lir - read-only/li
+   * li - no access/li
+   * /ul
+   */
+  private String ownerAccess = null;
+
+  /**
+   * A String declaring the local groip name that should be set as the group 
owner of the keytab file
+   */
+  private String groupName = null;
+
+  /**
+   * A String declaring the access permissions that should be set on the 
keytab file related to the
+   * group.
+   * p/
+   * Expected values are:
+

[3/3] ambari git commit: AMBARI-8542. Provide a way to parse and handle Kerberos descriptors

2014-12-08 Thread jspeidel

AMBARI-8542. Provide a way to parse and handle Kerberos descriptors


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e7b8383b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e7b8383b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e7b8383b

Branch: refs/heads/trunk
Commit: e7b8383b5af0a39b2530dbfc24563eefd7f781c7
Parents: 1a18dde
Author: Robert Levas rle...@hortonworks.com
Authored: Mon Dec 8 17:44:37 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Dec 8 17:45:52 2014 -0500

--
 .../kerberos/AbstractKerberosDescriptor.java| 331 ++
 .../AbstractKerberosDescriptorContainer.java| 626 +++
 .../kerberos/KerberosComponentDescriptor.java   | 115 
 .../KerberosConfigurationDescriptor.java| 205 ++
 .../state/kerberos/KerberosDescriptor.java  | 351 +++
 .../state/kerberos/KerberosDescriptorType.java  |  54 ++
 .../kerberos/KerberosIdentityDescriptor.java| 278 
 .../kerberos/KerberosKeytabDescriptor.java  | 445 +
 .../kerberos/KerberosPrincipalDescriptor.java   | 212 +++
 .../kerberos/KerberosServiceDescriptor.java | 318 ++
 .../KerberosComponentDescriptorTest.java| 206 ++
 .../KerberosConfigurationDescriptorTest.java| 242 +++
 .../state/kerberos/KerberosDescriptorTest.java  | 328 ++
 .../KerberosIdentityDescriptorTest.java | 154 +
 .../kerberos/KerberosKeytabDescriptorTest.java  | 146 +
 .../KerberosPrincipalDescriptorTest.java| 109 
 .../kerberos/KerberosServiceDescriptorTest.java | 238 +++
 17 files changed, 4358 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/e7b8383b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
new file mode 100644
index 000..b18e412
--- /dev/null
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
@@ -0,0 +1,331 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.state.kerberos;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.apache.ambari.server.AmbariException;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * AbstractKerberosDescriptor is the base class for all Kerberos*Descriptor 
and associated classes.
+ * p/
+ * It provides storage and management for the parent and name values on behalf 
of implementing classes.
+ * It also provides utility and helper methods.
+ */
+public abstract class AbstractKerberosDescriptor {
+
+  /**
+   * a regular expression Pattern used to find variable placeholders in 
strings
+   */
+  private static final Pattern PATTERN_VARIABLE = 
Pattern.compile(\\$\\{(?:(.+?)/)?(.+?)\\});
+
+  /**
+   * An AbstractKerberosDescriptor serving as the parent (or container) for 
this
+   * AbstractKerberosDescriptor.
+   * p/
+   * This value may be null in the event no parent has been identified.
+   */
+  private AbstractKerberosDescriptor parent = null;
+
+  /**
+   * A String declaring the name of this AbstractKerberosDescriptor.
+   * p/
+   * This value may be null in the event a name (or identifier) is not 
relevant.
+   */
+  private String name = null;
+
+  /**
+   * Performs variable replacement on the supplied String value using values 
from the replacementsMap.
+   * p/
+   * The value is a String containing one or more variables in the form of 
${variable_name}, such
+   * that

[1/3] ambari git commit: AMBARI-8542. Provide a way to parse and handle Kerberos descriptors

2014-12-08 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 1a18ddeed - e7b8383b5


http://git-wip-us.apache.org/repos/asf/ambari/blob/e7b8383b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptorTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptorTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptorTest.java
new file mode 100644
index 000..d905522
--- /dev/null
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptorTest.java
@@ -0,0 +1,238 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.kerberos;
+
+import junit.framework.Assert;
+import org.apache.ambari.server.AmbariException;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class KerberosServiceDescriptorTest {
+  public static final String JSON_VALUE =
+  { +
+\name\: \SERVICE_NAME\, +
+\identities\: [ +
+  KerberosIdentityDescriptorTest.JSON_VALUE +
+  ], +
+\components\: [ +
+  KerberosComponentDescriptorTest.JSON_VALUE +
+  ], +
+\configurations\: [ +
+  { +
+\service-site\: { +
+  \service.property1\: \value1\, +
+  \service.property2\: \value2\ +
+} +
+  } +
+] +
+  };
+
+  public static final MapString, Object MAP_VALUE =
+  new HashMapString, Object() {
+{
+  put(name, A_DIFFERENT_SERVICE_NAME);
+  put(KerberosDescriptorType.IDENTITY.getDescriptorPluralName(), new 
ArrayListObject() {{
+add(KerberosIdentityDescriptorTest.MAP_VALUE);
+  }});
+  put(KerberosDescriptorType.COMPONENT.getDescriptorPluralName(), new 
ArrayListObject() {{
+add(KerberosComponentDescriptorTest.MAP_VALUE);
+  }});
+  put(KerberosDescriptorType.CONFIGURATION.getDescriptorPluralName(), 
new ArrayListMapString, Object() {{
+add(new HashMapString, Object() {
+  {
+put(service-site, new HashMapString, String() {
+  {
+put(service.property1, red);
+put(service.property, green);
+  }
+});
+  }
+});
+  }});
+}
+  };
+
+  public static void validateFromJSON(KerberosServiceDescriptor 
serviceDescriptor) {
+Assert.assertNotNull(serviceDescriptor);
+Assert.assertTrue(serviceDescriptor.isContainer());
+
+Assert.assertEquals(SERVICE_NAME, serviceDescriptor.getName());
+
+MapString, KerberosComponentDescriptor componentDescriptors = 
serviceDescriptor.getComponents();
+Assert.assertNotNull(componentDescriptors);
+Assert.assertEquals(1, componentDescriptors.size());
+
+for (KerberosComponentDescriptor componentDescriptor : 
componentDescriptors.values()) {
+  KerberosComponentDescriptorTest.validateFromJSON(componentDescriptor);
+}
+
+ListKerberosIdentityDescriptor identityDescriptors = 
serviceDescriptor.getIdentities();
+Assert.assertNotNull(identityDescriptors);
+Assert.assertEquals(1, identityDescriptors.size());
+
+for (KerberosIdentityDescriptor identityDescriptor : identityDescriptors) {
+  KerberosIdentityDescriptorTest.validateFromJSON(identityDescriptor);
+}
+
+MapString, KerberosConfigurationDescriptor configurations = 
serviceDescriptor.getConfigurations();
+
+Assert.assertNotNull(configurations);
+Assert.assertEquals(1, configurations.size());
+
+KerberosConfigurationDescriptor configuration = 
configurations.get(service-site);
+
+Assert.assertNotNull(configuration);
+
+MapString, String properties = configuration.getProperties();
+
+Assert.assertEquals(service-site, configuration.getType());
+Assert.assertNotNull(properties);
+Assert.assertEquals(2, properties.size());
+

[1/2] ambari git commit: AMBARI-8458. Add support for add hosts specifying host name, blueprint name and host group name AMBARI-8437. Fix regression that prvented cluster creation via blueprints

2014-11-26 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 501785fa5 - 058dc168e


http://git-wip-us.apache.org/repos/asf/ambari/blob/058dc168/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java
index c53eb78..d868320 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java
@@ -38,6 +38,7 @@ import 
org.apache.ambari.server.controller.ConfigurationRequest;
 import org.apache.ambari.server.controller.HostRequest;
 import org.apache.ambari.server.controller.HostResponse;
 import org.apache.ambari.server.controller.MaintenanceStateHelper;
+import org.apache.ambari.server.controller.RequestStatusResponse;
 import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
 import org.apache.ambari.server.controller.spi.NoSuchResourceException;
 import org.apache.ambari.server.controller.spi.Predicate;
@@ -48,6 +49,7 @@ import 
org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException;
 import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.apache.ambari.server.orm.entities.BlueprintEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
@@ -55,19 +57,19 @@ import org.apache.ambari.server.state.DesiredConfig;
 import org.apache.ambari.server.state.Host;
 import org.apache.ambari.server.state.MaintenanceState;
 import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.configgroup.ConfigGroup;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.inject.Inject;
 import com.google.inject.assistedinject.Assisted;
 import com.google.inject.assistedinject.AssistedInject;
-import com.google.inject.persist.Transactional;
 
 
 /**
  * Resource provider for host resources.
  */
-public class HostResourceProvider extends AbstractControllerResourceProvider {
+public class HostResourceProvider extends BaseBlueprintProcessor {
 
   // - Property ID constants -
 
@@ -114,6 +116,13 @@ public class HostResourceProvider extends 
AbstractControllerResourceProvider {
   protected static final String HOST_DESIRED_CONFIGS_PROPERTY_ID = 
   PropertyHelper.getPropertyId(Hosts, desired_configs);
 
+  protected static final String BLUEPRINT_PROPERTY_ID =
+  PropertyHelper.getPropertyId(null, blueprint);
+  protected static final String HOSTGROUP_PROPERTY_ID =
+  PropertyHelper.getPropertyId(null, host_group);
+  protected static final String HOST_NAME_NO_CATEGORY_PROPERTY_ID =
+  PropertyHelper.getPropertyId(null, host_name);
+
   private static SetString pkPropertyIds =
   new HashSetString(Arrays.asList(new String[]{
   HOST_NAME_PROPERTY_ID}));
@@ -140,31 +149,31 @@ public class HostResourceProvider extends 
AbstractControllerResourceProvider {
   // - ResourceProvider 
 
   @Override
-  public RequestStatus createResources(Request request)
+  public RequestStatus createResources(final Request request)
   throws SystemException,
   UnsupportedPropertyException,
   ResourceAlreadyExistsException,
   NoSuchParentResourceException {
 
-final SetHostRequest requests = new HashSetHostRequest();
-for (MapString, Object propertyMap : request.getProperties()) {
-  requests.add(getRequest(propertyMap));
+RequestStatusResponse createResponse = null;
+if (isHostGroupRequest(request)) {
+  createResponse = addHostsUsingHostgroup(request);
+} else {
+  createResources(new CommandVoid() {
+@Override
+public Void invoke() throws AmbariException {
+  createHosts(request);
+  return null;
+}
+  });
 }
-createResources(new CommandVoid() {
-  @Override
-  public Void invoke() throws AmbariException {
-createHosts(requests);
-return null;
-  }
-});
 
 notifyCreate(Resource.Type.Host, request);
 
-return getRequestStatus(null);
+return getRequestStatus(createResponse);
   }
 
   @Override
-  @Transactional
   public SetResource getResources(Request request, Predicate predicate)
   throws SystemException, UnsupportedPropertyException, 
NoSuchResourceException, NoSuchParentResourceException {
 
@@ -256,7 +265,7 @@ public class

[2/2] ambari git commit: AMBARI-8458. Add support for add hosts specifying host name, blueprint name and host group name AMBARI-8437. Fix regression that prvented cluster creation via blueprints

2014-11-26 Thread jspeidel

AMBARI-8458. Add support for add hosts specifying host name, blueprint name 
and host group name
AMBARI-8437. Fix regression that prvented cluster creation via blueprints


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/058dc168
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/058dc168
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/058dc168

Branch: refs/heads/trunk
Commit: 058dc168e6173f4206837a1db87d066b372b07b1
Parents: 501785f
Author: John Speidel jspei...@hortonworks.com
Authored: Wed Nov 26 13:35:45 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed Nov 26 17:56:02 2014 -0500

--
 .../controller/AmbariManagementController.java  |  18 -
 .../AmbariManagementControllerImpl.java | 336 +---
 .../ambari/server/controller/HostRequest.java   |  31 +-
 .../internal/AbstractResourceProvider.java  |   2 +-
 .../internal/BaseBlueprintProcessor.java|  81 +++
 .../internal/ClusterControllerImpl.java |  13 +-
 .../internal/ClusterResourceProvider.java   |  46 +-
 .../internal/ConfigGroupResourceProvider.java   |  31 +-
 .../internal/HostComponentResourceProvider.java | 544 ++-
 .../internal/HostResourceProvider.java  | 324 ---
 .../org/apache/ambari/server/state/Cluster.java |   7 +-
 .../server/state/cluster/ClusterImpl.java   |   2 +-
 .../AmbariManagementControllerTest.java | 145 ++---
 .../internal/ClusterResourceProviderTest.java   |   9 +-
 .../HostComponentResourceProviderTest.java  | 220 +++-
 .../internal/HostResourceProviderTest.java  |  18 +-
 16 files changed, 1239 insertions(+), 588 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/058dc168/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
index 5af88a8..a1ece2c 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
@@ -235,24 +235,6 @@ public interface AmbariManagementController {
   throws AmbariException;
 
   /**
-   * Update the host component identified by the given request object with the
-   * values carried by the given request object.
-   *
-   *
-   *
-   * @param requests   the request object which defines which host 
component to
-   *   update and the values to set
-   * @param requestProperties  the request properties
-   * @param runSmokeTest   indicates whether or not to run a smoke test
-   *
-   * @return a track action response
-   *
-   * @throws AmbariException thrown if the resource cannot be updated
-   */
-  public RequestStatusResponse updateHostComponents(
-  SetServiceComponentHostRequest requests, MapString, String 
requestProperties, boolean runSmokeTest) throws AmbariException;
-
-  /**
* Updates the users specified.
*
* @param requests the users to modify

http://git-wip-us.apache.org/repos/asf/ambari/blob/058dc168/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 7d08a7b..0e65a1d 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -44,7 +44,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumMap;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
@@ -120,11 +119,8 @@ import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.StackInfo;
 import org.apache.ambari.server.state.State;
 import org.apache.ambari.server.state.configgroup.ConfigGroupFactory;
-import org.apache.ambari.server.state.fsm.InvalidStateTransitionException;
 import org.apache.ambari.server.state.scheduler.RequestExecutionFactory;
-import 
org.apache.ambari.server.state.svccomphost.ServiceComponentHostDisableEvent;
 import 
org.apache.ambari.server.state.svccomphost.ServiceComponentHostInstallEvent;

ambari git commit: AMBARI-7985. Add server side command functionality. Fixes issue introduced in original submission of this patch.

2014-11-19 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk cb8987357 - a52b04976


AMBARI-7985.  Add server side command functionality.
Fixes issue introduced in original submission of this patch.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a52b0497
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a52b0497
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a52b0497

Branch: refs/heads/trunk
Commit: a52b04976f09b34e37467238811baab5fbb9102b
Parents: cb89873
Author: Robert Levas rle...@hortonworks.com
Authored: Wed Nov 19 16:39:24 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed Nov 19 16:40:45 2014 -0500

--
 .../main/java/org/apache/ambari/server/actionmanager/Stage.java| 2 +-
 .../apache/ambari/server/actionmanager/TestActionScheduler.java| 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/a52b0497/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java 
b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java
index 01f2085..974acd4 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java
@@ -331,7 +331,7 @@ public class Stage {
   cmdParams.putAll(commandParams);
 }
 if (timeout != null) {
-  cmdParams.put(ExecutionCommand.KeyNames.COMMAND_TIMEOUT, 
NumberFormat.getIntegerInstance().format(timeout));
+  cmdParams.put(ExecutionCommand.KeyNames.COMMAND_TIMEOUT, 
Long.toString(timeout));
 }
 cmd.setCommandParams(cmdParams);
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52b0497/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
index c5c5bde..0640c34 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
@@ -496,7 +496,7 @@ public class TestActionScheduler {
 
 ListStage stages = new ArrayListStage();
 MapString, String payload = new HashMapString, String();
-final Stage s = getStageWithServerAction(1, 977, payload, test, 300);
+final Stage s = getStageWithServerAction(1, 977, payload, test, 1200);
 stages.add(s);
 
 ActionDBAccessor db = mock(ActionDBAccessor.class);

[2/2] ambari git commit: AMBARI-7985. Add server side command functionality. Allow tasks to be executed on the Ambari Server host.

2014-11-17 Thread jspeidel

AMBARI-7985.  Add server side command functionality.
Allow tasks to be executed on the Ambari Server host.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3d397dc0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3d397dc0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3d397dc0

Branch: refs/heads/trunk
Commit: 3d397dc04aab8d81c00aae2a8e5afa099fc57567
Parents: cc076cf
Author: Robert Levas rle...@hortonworks.com
Authored: Mon Nov 17 12:22:15 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Nov 17 12:25:43 2014 -0500

--
 .../server/actionmanager/ActionDBAccessor.java  |   5 +
 .../actionmanager/ActionDBAccessorImpl.java |   7 +-
 .../server/actionmanager/ActionManager.java |   5 +-
 .../server/actionmanager/ActionScheduler.java   |  97 +---
 .../ambari/server/actionmanager/Stage.java  |  39 +-
 .../server/controller/ControllerModule.java |   3 -
 .../server/orm/dao/HostRoleCommandDAO.java  |  11 +
 .../serveraction/AbstractServerAction.java  | 138 +
 .../server/serveraction/ServerAction.java   |  72 ++-
 .../serveraction/ServerActionExecutor.java  | 525 +++
 .../serveraction/ServerActionManager.java   |  32 --
 .../serveraction/ServerActionManagerImpl.java   |  74 ---
 .../server/state/ServiceComponentHostEvent.java |   2 +
 .../state/ServiceComponentHostEventType.java|   6 +-
 .../ServiceComponentHostServerActionEvent.java  |  76 +++
 .../ServiceComponentHostUpgradeEvent.java   |   3 +-
 .../actionmanager/TestActionDBAccessorImpl.java |  80 ++-
 .../server/actionmanager/TestActionManager.java |   6 +-
 .../actionmanager/TestActionScheduler.java  | 395 +-
 .../server/agent/TestHeartbeatHandler.java  |   6 +-
 .../AmbariManagementControllerTest.java |  25 +-
 .../server/serveraction/MockServerAction.java   |  92 
 .../serveraction/ServerActionExecutorTest.java  | 248 +
 23 files changed, 1581 insertions(+), 366 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/3d397dc0/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java
index 1f99b4a..6ff365b 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessor.java
@@ -149,6 +149,11 @@ public interface ActionDBAccessor {
   public CollectionHostRoleCommand getTasks(CollectionLong taskIds);
 
   /**
+   * Get a List of host role commands where the role and status are as 
specified
+   */
+  public ListHostRoleCommand getTasksByHostRoleAndStatus(String hostname, 
String role, HostRoleStatus status);
+
+  /**
* Get all stages that contain tasks with specified host role statuses
*/
   public ListStage getStagesByHostRoleStatus(SetHostRoleStatus statuses);

http://git-wip-us.apache.org/repos/asf/ambari/blob/3d397dc0/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
index 5e879cc..e8be3cc 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
@@ -212,7 +212,7 @@ public class ActionDBAccessorImpl implements 
ActionDBAccessor {
 if (clusterEntity != null) {
   clusterId = clusterEntity.getClusterId();
 }
-
+
 requestEntity.setClusterId(clusterId);
 requestDAO.create(requestEntity);
 
@@ -550,6 +550,11 @@ public class ActionDBAccessorImpl implements 
ActionDBAccessor {
   }
 
   @Override
+  public ListHostRoleCommand getTasksByHostRoleAndStatus(String hostname, 
String role, HostRoleStatus status) {
+return 
getTasks(hostRoleCommandDAO.findTaskIdsByHostRoleAndStatus(hostname, role, 
status));
+  }
+
+  @Override
   public ListStage getStagesByHostRoleStatus(SetHostRoleStatus statuses) {
 ListStage stages = new ArrayListStage();
 for (StageEntity stageEntity : stageDAO.findByCommandStatuses(statuses)) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/3d397dc0/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionManager.java

[1/2] ambari git commit: AMBARI-7985. Add server side command functionality. Allow tasks to be executed on the Ambari Server host.

2014-11-17 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk cc076cf34 - 3d397dc04


http://git-wip-us.apache.org/repos/asf/ambari/blob/3d397dc0/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
index 7224924..c5c5bde 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
@@ -25,14 +25,7 @@ import static org.mockito.Matchers.anyLong;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.*;
 import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -52,9 +45,7 @@ import org.apache.ambari.server.agent.CommandReport;
 import org.apache.ambari.server.agent.ExecutionCommand;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.HostsMap;
-import org.apache.ambari.server.serveraction.ServerAction;
-import org.apache.ambari.server.serveraction.ServerActionManager;
-import org.apache.ambari.server.serveraction.ServerActionManagerImpl;
+import org.apache.ambari.server.serveraction.MockServerAction;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Host;
@@ -65,6 +56,7 @@ import org.apache.ambari.server.state.ServiceComponentHost;
 import org.apache.ambari.server.state.ServiceComponentHostEvent;
 import 
org.apache.ambari.server.state.svccomphost.ServiceComponentHostInstallEvent;
 import 
org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpFailedEvent;
+import 
org.apache.ambari.server.state.svccomphost.ServiceComponentHostServerActionEvent;
 import 
org.apache.ambari.server.state.svccomphost.ServiceComponentHostUpgradeEvent;
 import org.apache.ambari.server.utils.StageUtils;
 import org.easymock.Capture;
@@ -86,6 +78,8 @@ public class TestActionScheduler {
   private static final String CLUSTER_HOST_INFO_UPDATED = 
{all_hosts=[c6401.ambari.apache.org,
   +  c6402.ambari.apache.org], slave_hosts=[c6401.ambari.apache.org,
   +  c6402.ambari.apache.org]};
+
+  private final String serverHostname = StageUtils.getHostName();
   private final String hostname = ahost.ambari.apache.org;
   private final int MAX_CYCLE_ITERATIONS = 100;
 
@@ -96,7 +90,7 @@ public class TestActionScheduler {
*/
   @Test
   public void testActionSchedule() throws Exception {
-
+
 Type type = new TypeTokenMapString, SetString() {}.getType();
 MapString, ListString clusterHostInfo = 
StageUtils.getGson().fromJson(CLUSTER_HOST_INFO, type);
 
@@ -116,7 +110,7 @@ public class TestActionScheduler {
 when(serviceObj.getServiceComponent(anyString())).thenReturn(scomp);
 when(scomp.getServiceComponentHost(anyString())).thenReturn(sch);
 when(serviceObj.getCluster()).thenReturn(oneClusterMock);
-
+
 Host host = mock(Host.class);
 HashMapString, ServiceComponentHost hosts =
 new HashMapString, ServiceComponentHost();
@@ -141,7 +135,7 @@ public class TestActionScheduler {
 //Keep large number of attempts so that the task is not expired finally
 //Small action timeout to test rescheduling
 ActionScheduler scheduler = new ActionScheduler(100, 5, db, aq, fsm,
-1, new HostsMap((String) null), null, unitOfWork, conf);
+1, new HostsMap((String) null), unitOfWork, conf);
 scheduler.setTaskTimeoutAdjustment(false);
 
 ListAgentCommand ac = waitForQueueSize(hostname, aq, 1, scheduler);
@@ -167,7 +161,7 @@ public class TestActionScheduler {
   int expectedQueueSize, ActionScheduler scheduler) {
 int cycleCount = 0;
 while (cycleCount++ = MAX_CYCLE_ITERATIONS) {
-  ListAgentCommand ac = aq.dequeueAll(hostname);  
+  ListAgentCommand ac = aq.dequeueAll(hostname);
   if (ac != null) {
 if (ac.size() == expectedQueueSize) {
   return ac;
@@ -239,7 +233,7 @@ public class TestActionScheduler {
 
 //Small action timeout to test rescheduling
 ActionScheduler scheduler = new ActionScheduler(100, 0, db, aq, fsm, 3,
-new HostsMap((String) null), null, unitOfWork, conf);
+new HostsMap((String) null), unitOfWork, conf);
 scheduler.setTaskTimeoutAdjustment(false);
 // Start the thread
 
@@ -305,12 +299,10 @@ public class TestActionScheduler {
   }

[04/11] ambari git commit: AMBARI-7175. Add explicit stack service inheritance

2014-11-10 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/2fc7adec/ambari-server/src/test/resources/stacks/OTHER/1.0/services/HDFS/metrics.json
--
diff --git 
a/ambari-server/src/test/resources/stacks/OTHER/1.0/services/HDFS/metrics.json 
b/ambari-server/src/test/resources/stacks/OTHER/1.0/services/HDFS/metrics.json
new file mode 100644
index 000..2938552
--- /dev/null
+++ 
b/ambari-server/src/test/resources/stacks/OTHER/1.0/services/HDFS/metrics.json
@@ -0,0 +1,7840 @@
+{
+  NAMENODE: {
+Component: [
+  {
+type: ganglia,
+metrics: {
+  metrics/dfs/FSNamesystem/TotalLoad: {
+metric: dfs.FSNamesystem.TotalLoad,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/BlockCapacity: {
+metric: dfs.FSNamesystem.BlockCapacity,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/CapacityTotal: {
+metric: dfs.FSNamesystem.CapacityTotal,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/CapacityUsed: {
+metric: dfs.FSNamesystem.CapacityUsed,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/CapacityRemaining: {
+metric: dfs.FSNamesystem.CapacityRemaining,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/CapacityNonDFSUsed: {
+metric: dfs.FSNamesystem.CapacityUsedNonDFS,
+pointInTime: false,
+temporal: true
+  },   
+  metrics/dfs/namenode/GetListingOps: {
+metric: dfs.namenode.GetListingOps,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/namenode/FilesAppended: {
+metric: dfs.namenode.FilesAppended,
+pointInTime: true,
+temporal: true
+  },
+  metrics/rpcdetailed/getProtocolVersion_num_ops: {
+metric: rpcdetailed.rpcdetailed.getProtocolVersion_num_ops,
+pointInTime: false,
+temporal: true
+  },
+  metrics/rpcdetailed/fsync_avg_time: {
+metric: rpcdetailed.rpcdetailed.FsyncAvgTime,
+pointInTime: false,
+temporal: true
+  },
+  metrics/ugi/loginSuccess_avg_time: {
+metric: ugi.UgiMetrics.LoginSuccessAvgTime,
+pointInTime: false,
+temporal: true
+  },
+  metrics/load/load_one: {
+metric: load_one,
+pointInTime: true,
+temporal: true
+  },
+  metrics/rpcdetailed/renewLease_num_ops: {
+metric: rpcdetailed.rpcdetailed.RenewLeaseNumOps,
+pointInTime: true,
+temporal: true
+  },
+  metrics/rpcdetailed/getFileInfo_avg_time: {
+metric: rpcdetailed.rpcdetailed.GetFileInfoAvgTime,
+pointInTime: false,
+temporal: true
+  },
+  metrics/jvm/memNonHeapUsedM: {
+metric: jvm.JvmMetrics.MemNonHeapUsedM,
+pointInTime: false,
+temporal: true
+  },
+  metrics/rpcdetailed/complete_avg_time: {
+metric: rpcdetailed.rpcdetailed.CompleteAvgTime,
+pointInTime: false,
+temporal: true
+  },
+  metrics/rpcdetailed/setPermission_num_ops: {
+metric: rpcdetailed.rpcdetailed.SetPermissionNumOps,
+pointInTime: true,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/CapacityTotalGB: {
+metric: dfs.FSNamesystem.CapacityTotalGB,
+pointInTime: false,
+temporal: true
+  },
+  metrics/rpcdetailed/setOwner_num_ops: {
+metric: rpcdetailed.rpcdetailed.SetOwnerNumOps,
+pointInTime: true,
+temporal: true
+  },
+  metrics/rpcdetailed/getBlockLocations_num_ops: {
+metric: rpcdetailed.rpcdetailed.GetBlockLocationsNumOps,
+pointInTime: false,
+temporal: true
+  },
+  metrics/process/proc_run: {
+metric: proc_run,
+pointInTime: true,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/CapacityUsedGB: {
+metric: dfs.FSNamesystem.CapacityUsedGB,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/namenode/AddBlockOps: {
+metric: dfs.namenode.AddBlockOps,
+pointInTime: false,
+temporal: true
+  },
+  metrics/memory/swap_total: {
+metric: swap_total,
+pointInTime: true,
+temporal: true
+  },
+  metrics/dfs/namenode/FilesDeleted: {

[11/11] ambari git commit: AMBARI-7175. Add explicit stack service inheritance

2014-11-10 Thread jspeidel

AMBARI-7175.  Add explicit stack service inheritance


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2fc7adec
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2fc7adec
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2fc7adec

Branch: refs/heads/trunk
Commit: 2fc7adecea03ec70141f41e98f1aa2d089c3b364
Parents: b71407f
Author: John Speidel jspei...@hortonworks.com
Authored: Wed Nov 5 23:12:11 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Nov 10 16:31:43 2014 -0500

--
 .../ambari/server/ObjectNotFoundException.java  |   18 +-
 .../server/ParentObjectNotFoundException.java   |2 +-
 .../ambari/server/agent/HeartBeatHandler.java   |2 +-
 .../ambari/server/agent/HeartbeatMonitor.java   |8 +-
 .../query/render/ClusterBlueprintRenderer.java  |   20 +-
 .../server/api/services/AmbariMetaInfo.java |  629 +-
 .../server/api/util/StackExtensionHelper.java   |  975 ---
 .../controller/AmbariActionExecutionHelper.java |9 +-
 .../AmbariCustomCommandExecutionHelper.java |   14 +-
 .../AmbariManagementControllerImpl.java |   49 +-
 .../server/controller/StackServiceResponse.java |   10 +-
 .../internal/AbstractResourceProvider.java  |   12 +-
 .../internal/BlueprintResourceProvider.java |1 -
 .../internal/ClientConfigResourceProvider.java  |4 +-
 .../internal/ClusterResourceProvider.java   |   21 +-
 .../internal/ComponentResourceProvider.java |   29 +-
 .../internal/ServiceResourceProvider.java   |   36 +-
 .../server/controller/internal/Stack.java   |   29 -
 .../server/metadata/RoleCommandOrder.java   |2 +-
 .../server/orm/entities/BlueprintEntity.java|   17 +-
 .../apache/ambari/server/stack/BaseModule.java  |   93 +
 .../ambari/server/stack/ComponentModule.java|  150 +
 .../server/stack/ConfigurationDirectory.java|  129 +
 .../ambari/server/stack/ConfigurationInfo.java  |  164 +
 .../server/stack/ConfigurationModule.java   |  170 +
 .../server/stack/ModuleFileUnmarshaller.java|   77 +
 .../ambari/server/stack/ServiceDirectory.java   |  181 +
 .../ambari/server/stack/ServiceModule.java  |  289 +
 .../ambari/server/stack/StackContext.java   |  196 +
 .../server/stack/StackDefinitionDirectory.java  |  106 +
 .../server/stack/StackDefinitionModule.java |   69 +
 .../ambari/server/stack/StackDirectory.java |  365 +
 .../ambari/server/stack/StackManager.java   |  175 +
 .../apache/ambari/server/stack/StackModule.java |  498 ++
 .../ambari/server/state/AutoDeployInfo.java |   20 +
 .../ambari/server/state/ComponentInfo.java  |   48 +
 .../ambari/server/state/ConfigHelper.java   |   54 +-
 .../ambari/server/state/DependencyInfo.java |   26 +
 .../server/state/ServiceComponentImpl.java  |   27 +-
 .../apache/ambari/server/state/ServiceImpl.java |4 +-
 .../apache/ambari/server/state/ServiceInfo.java |  145 +-
 .../ambari/server/state/ServiceOsSpecific.java  |   60 +
 .../org/apache/ambari/server/state/Stack.java   |   66 -
 .../apache/ambari/server/state/StackInfo.java   |  112 +-
 .../server/state/cluster/ClusterImpl.java   |8 +-
 .../render/ClusterBlueprintRendererTest.java|   10 +-
 .../server/api/services/AmbariMetaInfoTest.java |  405 +-
 .../api/util/StackExtensionHelperTest.java  |  792 --
 .../AmbariManagementControllerImplTest.java |6 +-
 .../AmbariManagementControllerTest.java |   31 +-
 .../internal/BlueprintResourceProviderTest.java |   33 +-
 .../ClientConfigResourceProviderTest.java   |6 +-
 .../internal/ClusterResourceProviderTest.java   |8 -
 .../internal/ComponentResourceProviderTest.java |   75 +-
 .../internal/HostResourceProviderTest.java  |8 +-
 .../internal/ServiceResourceProviderTest.java   |   32 +-
 .../orm/entities/BlueprintEntityTest.java   |   61 +-
 .../server/stack/ComponentModuleTest.java   |  409 +
 .../ambari/server/stack/ServiceModuleTest.java  |  983 +++
 .../ambari/server/stack/StackManagerTest.java   |  594 ++
 .../ambari/server/state/PropertyInfoTest.java   |   85 +-
 .../bad-stacks/HDP/0.1/repos/repoinfo.xml   |   37 +-
 .../stacks/HDP/2.0.7/services/HIVE/metainfo.xml |2 -
 .../stacks/HDP/2.0.7/services/YARN/metainfo.xml |4 +
 .../services/HBASE/configuration/hbase-site.xml |  356 +
 .../HDP/2.0.8/services/HBASE/metainfo.xml   |3 +
 .../stacks/HDP/2.1.1/services/PIG/metainfo.xml  |8 +-
 .../resources/stacks/OTHER/1.0/metainfo.xml |   23 +
 .../resources/stacks/OTHER/1.0/repos/hdp.json   |   10 +
 .../stacks/OTHER/1.0/repos/repoinfo.xml |   62 +
 .../stacks/OTHER/1.0/role_command_order.json|  104 +
 .../services/HDFS/configuration/hdfs-site.xml   |   45 +
 .../stacks/OTHER/1.0/services/HDFS/metainfo.xml |  146 +
 .../stacks/OTHER/1.0/services/HDFS/metrics.json | 7840

[02/11] ambari git commit: AMBARI-7175. Add explicit stack service inheritance

2014-11-10 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/2fc7adec/ambari-server/src/test/resources/stacks_with_cycle/OTHER/1.0/services/HDFS/metrics.json
--
diff --git 
a/ambari-server/src/test/resources/stacks_with_cycle/OTHER/1.0/services/HDFS/metrics.json
 
b/ambari-server/src/test/resources/stacks_with_cycle/OTHER/1.0/services/HDFS/metrics.json
new file mode 100644
index 000..2938552
--- /dev/null
+++ 
b/ambari-server/src/test/resources/stacks_with_cycle/OTHER/1.0/services/HDFS/metrics.json
@@ -0,0 +1,7840 @@
+{
+  NAMENODE: {
+Component: [
+  {
+type: ganglia,
+metrics: {
+  metrics/dfs/FSNamesystem/TotalLoad: {
+metric: dfs.FSNamesystem.TotalLoad,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/BlockCapacity: {
+metric: dfs.FSNamesystem.BlockCapacity,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/CapacityTotal: {
+metric: dfs.FSNamesystem.CapacityTotal,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/CapacityUsed: {
+metric: dfs.FSNamesystem.CapacityUsed,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/CapacityRemaining: {
+metric: dfs.FSNamesystem.CapacityRemaining,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/CapacityNonDFSUsed: {
+metric: dfs.FSNamesystem.CapacityUsedNonDFS,
+pointInTime: false,
+temporal: true
+  },   
+  metrics/dfs/namenode/GetListingOps: {
+metric: dfs.namenode.GetListingOps,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/namenode/FilesAppended: {
+metric: dfs.namenode.FilesAppended,
+pointInTime: true,
+temporal: true
+  },
+  metrics/rpcdetailed/getProtocolVersion_num_ops: {
+metric: rpcdetailed.rpcdetailed.getProtocolVersion_num_ops,
+pointInTime: false,
+temporal: true
+  },
+  metrics/rpcdetailed/fsync_avg_time: {
+metric: rpcdetailed.rpcdetailed.FsyncAvgTime,
+pointInTime: false,
+temporal: true
+  },
+  metrics/ugi/loginSuccess_avg_time: {
+metric: ugi.UgiMetrics.LoginSuccessAvgTime,
+pointInTime: false,
+temporal: true
+  },
+  metrics/load/load_one: {
+metric: load_one,
+pointInTime: true,
+temporal: true
+  },
+  metrics/rpcdetailed/renewLease_num_ops: {
+metric: rpcdetailed.rpcdetailed.RenewLeaseNumOps,
+pointInTime: true,
+temporal: true
+  },
+  metrics/rpcdetailed/getFileInfo_avg_time: {
+metric: rpcdetailed.rpcdetailed.GetFileInfoAvgTime,
+pointInTime: false,
+temporal: true
+  },
+  metrics/jvm/memNonHeapUsedM: {
+metric: jvm.JvmMetrics.MemNonHeapUsedM,
+pointInTime: false,
+temporal: true
+  },
+  metrics/rpcdetailed/complete_avg_time: {
+metric: rpcdetailed.rpcdetailed.CompleteAvgTime,
+pointInTime: false,
+temporal: true
+  },
+  metrics/rpcdetailed/setPermission_num_ops: {
+metric: rpcdetailed.rpcdetailed.SetPermissionNumOps,
+pointInTime: true,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/CapacityTotalGB: {
+metric: dfs.FSNamesystem.CapacityTotalGB,
+pointInTime: false,
+temporal: true
+  },
+  metrics/rpcdetailed/setOwner_num_ops: {
+metric: rpcdetailed.rpcdetailed.SetOwnerNumOps,
+pointInTime: true,
+temporal: true
+  },
+  metrics/rpcdetailed/getBlockLocations_num_ops: {
+metric: rpcdetailed.rpcdetailed.GetBlockLocationsNumOps,
+pointInTime: false,
+temporal: true
+  },
+  metrics/process/proc_run: {
+metric: proc_run,
+pointInTime: true,
+temporal: true
+  },
+  metrics/dfs/FSNamesystem/CapacityUsedGB: {
+metric: dfs.FSNamesystem.CapacityUsedGB,
+pointInTime: false,
+temporal: true
+  },
+  metrics/dfs/namenode/AddBlockOps: {
+metric: dfs.namenode.AddBlockOps,
+pointInTime: false,
+temporal: true
+  },
+  metrics/memory/swap_total: {
+metric: swap_total,
+pointInTime: true,
+temporal: true
+  },
+

[06/11] ambari git commit: AMBARI-7175. Add explicit stack service inheritance

2014-11-10 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/2fc7adec/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest.java
index d8c0e29..b965554 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest.java
@@ -22,12 +22,15 @@ import com.google.gson.Gson;
 
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.state.PropertyInfo;
+import org.apache.ambari.server.state.ServiceInfo;
 import org.junit.Test;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -83,8 +86,11 @@ public class BlueprintEntityTest {
   @Test
   public void testValidateConfigurations_clusterConfig() throws Exception {
 AmbariMetaInfo metaInfo = createMock(AmbariMetaInfo.class);
+ServiceInfo service = new ServiceInfo();
+service.setName(service1);
+
+ListPropertyInfo serviceProperties = new ArrayListPropertyInfo();
 
-MapString, PropertyInfo requiredProps = new HashMapString, 
PropertyInfo();
 PropertyInfo prop = new PropertyInfo();
 prop.setFilename(core-site.xml);
 prop.setName(super.secret.password);
@@ -93,7 +99,9 @@ public class BlueprintEntityTest {
 propertyTypes.add(PropertyInfo.PropertyType.PASSWORD);
 prop.setPropertyTypes(propertyTypes);
 prop.setValue(null);
-requiredProps.put(super.secret.password, prop);
+serviceProperties.add(prop);
+service.getProperties().addAll(serviceProperties);
+service.getProperties().addAll(serviceProperties);
 
 BlueprintEntity entity = new BlueprintEntity();
 entity.setStackName(stackName);
@@ -130,7 +138,7 @@ public class BlueprintEntityTest {
 entity.setHostGroups(hostGroupEntities);
 
 expect(metaInfo.getComponentToService(stackName, version, 
component1)).andReturn(service1);
-expect(metaInfo.getRequiredProperties(stackName, version, 
service1)).andReturn(requiredProps);
+expect(metaInfo.getService(stackName, version, 
service1)).andReturn(service);
 
 replay(metaInfo);
 
@@ -145,17 +153,20 @@ public class BlueprintEntityTest {
   @Test
   public void testValidateConfigurations_hostGroupConfig() throws Exception {
 AmbariMetaInfo metaInfo = createMock(AmbariMetaInfo.class);
-
-MapString, PropertyInfo requiredProps = new HashMapString, 
PropertyInfo();
-PropertyInfo prop = new PropertyInfo();
-prop.setFilename(core-site.xml);
-prop.setName(super.secret.password);
-prop.setRequireInput(true);
+ServiceInfo service = new ServiceInfo();
+service.setName(service1);
+
+ListPropertyInfo serviceProperties = new ArrayListPropertyInfo();
+PropertyInfo prop1 = new PropertyInfo();
+prop1.setFilename(core-site.xml);
+prop1.setName(super.secret.password);
+prop1.setRequireInput(true);
 SetPropertyInfo.PropertyType propertyTypes = new 
HashSetPropertyInfo.PropertyType();
 propertyTypes.add(PropertyInfo.PropertyType.PASSWORD);
-prop.setPropertyTypes(propertyTypes);
-prop.setValue(null);
-requiredProps.put(super.secret.password, prop);
+prop1.setPropertyTypes(propertyTypes);
+prop1.setValue(null);
+serviceProperties.add(prop1);
+service.getProperties().addAll(serviceProperties);
 
 BlueprintEntity entity = new BlueprintEntity();
 entity.setStackName(stackName);
@@ -193,7 +204,7 @@ public class BlueprintEntityTest {
 entity.setHostGroups(hostGroupEntities);
 
 expect(metaInfo.getComponentToService(stackName, version, 
component1)).andReturn(service1);
-expect(metaInfo.getRequiredProperties(stackName, version, 
service1)).andReturn(requiredProps);
+expect(metaInfo.getService(stackName, version, 
service1)).andReturn(service);
 
 replay(metaInfo);
 
@@ -208,16 +219,20 @@ public class BlueprintEntityTest {
   @Test
   public void testValidateConfigurations_negative() throws Exception {
 AmbariMetaInfo metaInfo = createMock(AmbariMetaInfo.class);
+ServiceInfo service = new ServiceInfo();
+service.setName(service1);
 
-MapString, PropertyInfo requiredProps = new HashMapString, 
PropertyInfo();
-PropertyInfo prop = new PropertyInfo();
-prop.setFilename(core-site.xml);
-prop.setName(super.secret.password);
-prop.setRequireInput(true);
+ListPropertyInfo serviceProperties = new ArrayListPropertyInfo();
+
+PropertyInfo prop1 = new PropertyInfo();
+prop1.setFilename(core-site.xml);
+prop1.setName(super.secret.password);
+

[07/11] ambari git commit: AMBARI-7175. Add explicit stack service inheritance

2014-11-10 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/2fc7adec/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java
deleted file mode 100644
index f05aab5..000
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java
+++ /dev/null
@@ -1,792 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * License); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.api.util;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-
-import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.metadata.ActionMetadata;
-import org.apache.ambari.server.state.*;
-import org.apache.ambari.server.state.stack.ConfigurationXml;
-import org.junit.Test;
-import org.xml.sax.SAXException;
-
-import javax.xml.bind.JAXBException;
-import javax.xml.namespace.QName;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.xpath.XPathExpressionException;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.*;
-
-import static org.easymock.EasyMock.*;
-import static org.junit.Assert.*;
-
-public class StackExtensionHelperTest {
-
-  private final String stackRootStr = ./src/test/resources/stacks/.
-  replaceAll(/, File.separator);
-
-  private Injector injector = Guice.createInjector(new MockModule());
-  
-  
-  public class MockModule extends AbstractModule {
-@Override
-protected void configure() {
-  bind(ActionMetadata.class);
-}
-  }
-
-  /**
-  * Checks than service metainfo is parsed correctly both for ver 1 services
-  * and for ver 2 services
-  */
-  @Test
-  public void testPopulateServicesForStack() throws Exception {
-File stackRoot = new File(stackRootStr);
-StackInfo stackInfo = new StackInfo();
-stackInfo.setName(HDP);
-stackInfo.setVersion(2.0.7);
-StackExtensionHelper helper = new StackExtensionHelper(injector, 
stackRoot);
-helper.populateServicesForStack(stackInfo);
-ListServiceInfo services =  stackInfo.getServices();
-assertEquals(8, services.size());
-for (ServiceInfo serviceInfo : services) {
-  if (serviceInfo.getName().equals(HIVE)) {
-// Check old-style service
-assertEquals(HIVE, serviceInfo.getName());
-assertEquals(Hive, serviceInfo.getDisplayName());
-assertEquals(2.0, serviceInfo.getSchemaVersion());
-assertTrue(serviceInfo.getComment().startsWith(Data warehouse 
system));
-assertEquals(0.11.0.2.0.5.0, serviceInfo.getVersion());
-// Check some component definitions
-ListComponentInfo components = serviceInfo.getComponents();
-assertEquals(HIVE_METASTORE, components.get(0).getName());
-assertEquals(Hive Metastore, components.get(0).getDisplayName());
-assertEquals(MASTER, components.get(0).getCategory());
-ListPropertyInfo properties = serviceInfo.getProperties();
-// Check some property
-assertEquals(37, properties.size());
-boolean found = false;
-for (PropertyInfo property : properties) {
-  if 
(property.getName().equals(javax.jdo.option.ConnectionDriverName)) {
-assertEquals(com.mysql.jdbc.Driver, property.getValue());
-assertEquals(hive-site.xml,
-property.getFilename());
-assertEquals(false, property.isDeleted());
-found = true;
-  }
-}
-assertTrue(Property not found in a list of properties, found);
-// Check config dependencies
-ListString configDependencies = serviceInfo.getConfigDependencies();
-assertEquals(2, configDependencies.size());
-assertEquals(hive-site, configDependencies.get(1));
-  } else if (serviceInfo.getName().equals(HBASE)) {
-assertEquals(HBASE, serviceInfo.getName());
-assertEquals(HDP/2.0.7/services/HBASE/package,
-

[08/11] ambari git commit: AMBARI-7175. Add explicit stack service inheritance

2014-11-10 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/2fc7adec/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java 
b/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java
index c9794d8..dcfd00f 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java
@@ -207,6 +207,10 @@ public class ComponentInfo {
 this.dependencies = dependencies;
   }
 
+  public void setAutoDeploy(AutoDeployInfo autoDeploy) {
+this.autoDeploy = autoDeploy;
+  }
+
   public AutoDeployInfo getAutoDeploy() {
 return autoDeploy;
   }
@@ -226,4 +230,48 @@ public class ComponentInfo {
   public void setClientsToUpdateConfigs(ListString clientsToUpdateConfigs) {
 this.clientsToUpdateConfigs = clientsToUpdateConfigs;
   }
+
+  @Override
+  public boolean equals(Object o) {
+if (this == o) return true;
+if (o == null || getClass() != o.getClass()) return false;
+
+ComponentInfo that = (ComponentInfo) o;
+
+if (deleted != that.deleted) return false;
+if (autoDeploy != null ? !autoDeploy.equals(that.autoDeploy) : 
that.autoDeploy != null) return false;
+if (cardinality != null ? !cardinality.equals(that.cardinality) : 
that.cardinality != null) return false;
+if (category != null ? !category.equals(that.category) : that.category != 
null) return false;
+if (clientConfigFiles != null ? 
!clientConfigFiles.equals(that.clientConfigFiles) : that.clientConfigFiles != 
null)
+  return false;
+if (commandScript != null ? !commandScript.equals(that.commandScript) : 
that.commandScript != null) return false;
+if (configDependencies != null ? 
!configDependencies.equals(that.configDependencies) : that.configDependencies 
!= null)
+  return false;
+if (customCommands != null ? !customCommands.equals(that.customCommands) : 
that.customCommands != null)
+  return false;
+if (dependencies != null ? !dependencies.equals(that.dependencies) : 
that.dependencies != null) return false;
+if (displayName != null ? !displayName.equals(that.displayName) : 
that.displayName != null) return false;
+if (name != null ? !name.equals(that.name) : that.name != null) return 
false;
+if (clientConfigFiles != null ? 
!clientConfigFiles.equals(that.clientConfigFiles) :
+that.clientConfigFiles != null) return false;
+
+return true;
+  }
+
+  @Override
+  public int hashCode() {
+int result = name != null ? name.hashCode() : 0;
+result = 31 * result + (displayName != null ? displayName.hashCode() : 0);
+result = 31 * result + (category != null ? category.hashCode() : 0);
+result = 31 * result + (deleted ? 1 : 0);
+result = 31 * result + (cardinality != null ? cardinality.hashCode() : 0);
+result = 31 * result + (commandScript != null ? commandScript.hashCode() : 
0);
+result = 31 * result + (clientConfigFiles != null ? 
clientConfigFiles.hashCode() : 0);
+result = 31 * result + (customCommands != null ? customCommands.hashCode() 
: 0);
+result = 31 * result + (dependencies != null ? dependencies.hashCode() : 
0);
+result = 31 * result + (autoDeploy != null ? autoDeploy.hashCode() : 0);
+result = 31 * result + (configDependencies != null ? 
configDependencies.hashCode() : 0);
+result = 31 * result + (clientConfigFiles != null ? 
clientConfigFiles.hashCode() : 0);
+return result;
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/2fc7adec/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java 
b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index e15a62a..d4cbd4e 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -417,13 +417,13 @@ public class ConfigHelper {
* @param propertyName
*/
   public SetString findConfigTypesByPropertyName(StackId stackId, String 
propertyName, String clusterName) throws AmbariException {
-StackInfo stack = ambariMetaInfo.getStackInfo(stackId.getStackName(),
+StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(),
 stackId.getStackVersion());
 
 SetString result = new HashSetString();
 
 for(Service service : 
clusters.getCluster(clusterName).getServices().values()) {
-  SetPropertyInfo stackProperties = 
ambariMetaInfo.getProperties(stack.getName(), stack.getVersion(), 
service.getName());
+  SetPropertyInfo stackProperties = 
ambariMetaInfo.getServiceProperties(stack.getName(),

[05/11] ambari git commit: AMBARI-7175. Add explicit stack service inheritance

2014-11-10 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/2fc7adec/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java
new file mode 100644
index 000..a6cbc6a
--- /dev/null
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java
@@ -0,0 +1,594 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.stack;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.metadata.ActionMetadata;
+import org.apache.ambari.server.orm.dao.MetainfoDAO;
+import org.apache.ambari.server.state.ClientConfigFileDefinition;
+import org.apache.ambari.server.state.CommandScriptDefinition;
+import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.PropertyInfo;
+import org.apache.ambari.server.state.ServiceInfo;
+import org.apache.ambari.server.state.ServiceOsSpecific;
+import org.apache.ambari.server.state.StackInfo;
+import org.apache.ambari.server.state.stack.OsFamily;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * StackManager unit tests.
+ */
+public class StackManagerTest {
+
+  private static StackManager stackManager;
+  private static MetainfoDAO dao;
+  private static ActionMetadata actionMetadata;
+  private static OsFamily osFamily;
+
+  @BeforeClass
+  public static void initStack() throws Exception{
+stackManager = createTestStackManager();
+  }
+
+  public static StackManager createTestStackManager() throws Exception {
+return createTestStackManager(./src/test/resources/stacks/);
+  }
+
+  public static StackManager createTestStackManager(String stackRoot) throws 
Exception {
+try {
+  //todo: dao , actionMetaData expectations
+  dao = createNiceMock(MetainfoDAO.class);
+  actionMetadata = createNiceMock(ActionMetadata.class);
+  Configuration config = createNiceMock(Configuration.class);
+  
expect(config.getSharedResourcesDirPath()).andReturn(./src/test/resources).anyTimes();
+  replay(config);
+  osFamily = new OsFamily(config);
+
+  replay(dao, actionMetadata);
+  return new StackManager(new File(stackRoot), new StackContext(dao, 
actionMetadata, osFamily));
+} catch (Exception e) {
+  e.printStackTrace();
+  throw e;
+}
+  }
+
+  @Test
+  public void testGetStacks_count() throws Exception {
+CollectionStackInfo stacks = stackManager.getStacks();
+assertEquals(16, stacks.size());
+  }
+
+  @Test
+  public void testGetStack_name__count() {
+CollectionStackInfo stacks = stackManager.getStacks(HDP);
+assertEquals(12, stacks.size());
+
+stacks = stackManager.getStacks(OTHER);
+assertEquals(2, stacks.size());
+  }
+
+  @Test
+  public void testGetStack_basic() {
+StackInfo stack = stackManager.getStack(HDP, 0.1);
+assertNotNull(stack);
+assertEquals(HDP, stack.getName());
+assertEquals(0.1, stack.getVersion());
+
+
+CollectionServiceInfo services = stack.getServices();
+assertEquals(3, services.size());
+
+MapString, ServiceInfo serviceMap = new HashMapString, ServiceInfo();
+for (ServiceInfo service : services) {
+  serviceMap.put(service.getName(), service);
+}
+ServiceInfo hdfsService = serviceMap.get(HDFS);
+assertNotNull(hdfsService);
+

[01/11] ambari git commit: AMBARI-7175. Add explicit stack service inheritance

2014-11-10 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk b71407f06 - 2fc7adece


http://git-wip-us.apache.org/repos/asf/ambari/blob/2fc7adec/ambari-server/src/test/resources/stacks_with_cycle/OTHER/1.0/services/HDFS/package/dummy-script.py
--
diff --git 
a/ambari-server/src/test/resources/stacks_with_cycle/OTHER/1.0/services/HDFS/package/dummy-script.py
 
b/ambari-server/src/test/resources/stacks_with_cycle/OTHER/1.0/services/HDFS/package/dummy-script.py
new file mode 100644
index 000..35de4bb
--- /dev/null
+++ 
b/ambari-server/src/test/resources/stacks_with_cycle/OTHER/1.0/services/HDFS/package/dummy-script.py
@@ -0,0 +1,20 @@
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+License); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an AS IS BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/2fc7adec/ambari-server/src/test/resources/stacks_with_cycle/OTHER/1.0/services/SQOOP2/metainfo.xml
--
diff --git 
a/ambari-server/src/test/resources/stacks_with_cycle/OTHER/1.0/services/SQOOP2/metainfo.xml
 
b/ambari-server/src/test/resources/stacks_with_cycle/OTHER/1.0/services/SQOOP2/metainfo.xml
new file mode 100644
index 000..33cf448
--- /dev/null
+++ 
b/ambari-server/src/test/resources/stacks_with_cycle/OTHER/1.0/services/SQOOP2/metainfo.xml
@@ -0,0 +1,30 @@
+?xml version=1.0?
+!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the License); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an AS IS BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+--
+metainfo
+  schemaVersion2.0/schemaVersion
+  services
+service
+  nameSQOOP2/name
+  displayNameSqoop/displayName
+  commentResults in a cycle since v2.0 extends this version/comment
+  versionExtended Version/version
+  extendsOTHER/2.0/SQOOP/extends
+/service
+  /services
+
+/metainfo

http://git-wip-us.apache.org/repos/asf/ambari/blob/2fc7adec/ambari-server/src/test/resources/stacks_with_cycle/OTHER/2.0/metainfo.xml
--
diff --git 
a/ambari-server/src/test/resources/stacks_with_cycle/OTHER/2.0/metainfo.xml 
b/ambari-server/src/test/resources/stacks_with_cycle/OTHER/2.0/metainfo.xml
new file mode 100644
index 000..716972f
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks_with_cycle/OTHER/2.0/metainfo.xml
@@ -0,0 +1,24 @@
+?xml version=1.0?
+!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the License); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an AS IS BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+--
+metainfo
+versions
+  activetrue/active
+/versions
+extends1.0/extends
+/metainfo
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/2fc7adec/ambari-server/src/test/resources/stacks_with_cycle/OTHER/2.0/repos/hdp.json
--
diff --git 
a/ambari-server/src/test/resources/stacks_with_cycle/OTHER/2.0/repos/hdp.json

[10/11] ambari git commit: AMBARI-7175. Add explicit stack service inheritance

2014-11-10 Thread jspeidel

http://git-wip-us.apache.org/repos/asf/ambari/blob/2fc7adec/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java
deleted file mode 100644
index e014627..000
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java
+++ /dev/null
@@ -1,975 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * License); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.server.api.util;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.Set;
-
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Unmarshaller;
-import javax.xml.namespace.QName;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathExpression;
-import javax.xml.xpath.XPathExpressionException;
-import javax.xml.xpath.XPathFactory;
-
-import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.metadata.ActionMetadata;
-import org.apache.ambari.server.state.ClientConfigFileDefinition;
-import org.apache.ambari.server.state.CommandScriptDefinition;
-import org.apache.ambari.server.state.ComponentInfo;
-import org.apache.ambari.server.state.ConfigHelper;
-import org.apache.ambari.server.state.CustomCommandDefinition;
-import org.apache.ambari.server.state.DependencyInfo;
-import org.apache.ambari.server.state.PropertyInfo;
-import org.apache.ambari.server.state.ServiceInfo;
-import org.apache.ambari.server.state.ServiceOsSpecific;
-import org.apache.ambari.server.state.StackInfo;
-import org.apache.ambari.server.state.stack.ConfigurationXml;
-import org.apache.ambari.server.state.stack.RepositoryXml;
-import org.apache.ambari.server.state.stack.ServiceMetainfoXml;
-import org.apache.ambari.server.state.stack.StackMetainfoXml;
-import org.apache.ambari.server.state.stack.UpgradePack;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.xml.sax.SAXException;
-
-import com.google.inject.Injector;
-
-/**
- * Helper methods for providing stack extension behavior -
- * Apache Jira: AMBARI-2819
- *
- * Stack extension processing is done in two steps. At first step, we parse
- * all information for every stack from stack files. At second step, we
- * go through parent and perform inheritance where needed. At both steps,
- * stacks are processed at random order, that's why extension implementation
- * for any new stack/service/component property should also consist of two
- * separate steps (otherwise child may happen to be processed before parent's
- * properties are populated).
- */
-public class StackExtensionHelper {
-  private ActionMetadata actionMetadata;
-
-  private File stackRoot;
-  private final static Logger LOG = 
LoggerFactory.getLogger(StackExtensionHelper.class);
-  private final MapString, StackInfo stackVersionMap = new HashMapString,
-StackInfo();
-  private MapString, ListStackInfo stackParentsMap = null;
-  public final static String HOOKS_FOLDER_NAME = hooks;
-  private static final String PACKAGE_FOLDER_NAME = package;
-
-  private static final MapClass?, JAXBContext _jaxbContexts =
-  new HashMapClass?, JAXBContext ();
-  static {
-try {
-  JAXBContext ctx = JAXBContext.newInstance(StackMetainfoXml.class,
-  RepositoryXml.class, ConfigurationXml.class, UpgradePack.class);
-  _jaxbContexts.put(StackMetainfoXml.class, ctx);
-  _jaxbContexts.put(RepositoryXml.class, ctx);
-  _jaxbContexts.put(ConfigurationXml.class, ctx);
-

ambari git commit: AMBARI-8218. Replace hostnames with host_group tokens for property oozie.service.ProxyUserService.proxyuser.knox.hosts when doing blueprint export

2014-11-07 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-1.7.0 d10503b44 - 024e24ed2


AMBARI-8218.  Replace hostnames with host_group tokens for property
oozie.service.ProxyUserService.proxyuser.knox.hosts when doing
blueprint export


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/024e24ed
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/024e24ed
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/024e24ed

Branch: refs/heads/branch-1.7.0
Commit: 024e24ed2cbbe80713aada02a34cf9dee824a03f
Parents: d10503b
Author: John Speidel jspei...@hortonworks.com
Authored: Fri Nov 7 18:09:04 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Fri Nov 7 19:32:44 2014 -0500

--
 .../internal/BlueprintConfigurationProcessor.java  | 13 ++---
 .../internal/BlueprintConfigurationProcessorTest.java  |  4 
 2 files changed, 6 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/024e24ed/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 9237bc9..5af81be 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -883,17 +883,12 @@ public class BlueprintConfigurationProcessor {
 MapString, PropertyUpdater stormSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater falconStartupPropertiesMap = new 
HashMapString, PropertyUpdater();
 MapString, PropertyUpdater kafkaBrokerMap = new HashMapString, 
PropertyUpdater();
-
-
-
 MapString, PropertyUpdater mapredEnvMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater hadoopEnvMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater hbaseEnvMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater hiveEnvMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater oozieEnvMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater oozieEnvOriginalValueMap = new 
HashMapString, PropertyUpdater();
-
-
 MapString, PropertyUpdater multiWebhcatSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater multiHbaseSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater multiStormSiteMap = new HashMapString, 
PropertyUpdater();
@@ -904,13 +899,7 @@ public class BlueprintConfigurationProcessor {
 MapString, PropertyUpdater multiSliderClientMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater multiYarnSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater multiOozieSiteMap = new HashMapString, 
PropertyUpdater();
-
-
-
-
-
 MapString, PropertyUpdater dbHiveSiteMap = new HashMapString, 
PropertyUpdater();
-
 MapString, PropertyUpdater nagiosEnvMap = new HashMapString 
,PropertyUpdater();
 
 
@@ -1006,6 +995,7 @@ public class BlueprintConfigurationProcessor {
 oozieSiteMap.put(oozie.service.HadoopAccessorService.kerberos.principal, 
new SingleHostTopologyUpdater(OOZIE_SERVER));
 oozieEnvMap.put(oozie_hostname, new 
SingleHostTopologyUpdater(OOZIE_SERVER));
 multiCoreSiteMap.put(hadoop.proxyuser.oozie.hosts, new 
MultipleHostTopologyUpdater(OOZIE_SERVER));
+
 // register updaters for Oozie properties that may point to an external DB
 oozieEnvOriginalValueMap.put(oozie_existing_mysql_host, new 
OriginalValuePropertyUpdater());
 oozieSiteOriginalValueMap.put(oozie.service.JPAService.jdbc.url, new 
OriginalValuePropertyUpdater());
@@ -1042,6 +1032,7 @@ public class BlueprintConfigurationProcessor {
 multiCoreSiteMap.put(hadoop.proxyuser.knox.hosts, new 
MultipleHostTopologyUpdater(KNOX_GATEWAY));
 multiWebhcatSiteMap.put(webhcat.proxyuser.knox.hosts, new 
MultipleHostTopologyUpdater(KNOX_GATEWAY));
 multiOozieSiteMap.put(hadoop.proxyuser.knox.hosts, new 
MultipleHostTopologyUpdater(KNOX_GATEWAY));
+
multiOozieSiteMap.put(oozie.service.ProxyUserService.proxyuser.knox.hosts, 
new MultipleHostTopologyUpdater(KNOX_GATEWAY));
 
 
 // Required due to AMBARI-4933.  These no longer seem to be required as 
the default values in the stack

ambari git commit: AMBARI-8218. Replace hostnames with host_group tokens for property oozie.service.ProxyUserService.proxyuser.knox.hosts when doing blueprint export

2014-11-07 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 9590cfbd1 - 45658d2aa


AMBARI-8218.  Replace hostnames with host_group tokens for property
oozie.service.ProxyUserService.proxyuser.knox.hosts when doing
blueprint export


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/45658d2a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/45658d2a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/45658d2a

Branch: refs/heads/trunk
Commit: 45658d2aa2935e05f4ae42ad9187d2482029e1bf
Parents: 9590cfb
Author: John Speidel jspei...@hortonworks.com
Authored: Fri Nov 7 18:09:04 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Fri Nov 7 19:34:21 2014 -0500

--
 .../internal/BlueprintConfigurationProcessor.java  | 13 ++---
 .../internal/BlueprintConfigurationProcessorTest.java  |  4 
 2 files changed, 6 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/45658d2a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 9237bc9..5af81be 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -883,17 +883,12 @@ public class BlueprintConfigurationProcessor {
 MapString, PropertyUpdater stormSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater falconStartupPropertiesMap = new 
HashMapString, PropertyUpdater();
 MapString, PropertyUpdater kafkaBrokerMap = new HashMapString, 
PropertyUpdater();
-
-
-
 MapString, PropertyUpdater mapredEnvMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater hadoopEnvMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater hbaseEnvMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater hiveEnvMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater oozieEnvMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater oozieEnvOriginalValueMap = new 
HashMapString, PropertyUpdater();
-
-
 MapString, PropertyUpdater multiWebhcatSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater multiHbaseSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater multiStormSiteMap = new HashMapString, 
PropertyUpdater();
@@ -904,13 +899,7 @@ public class BlueprintConfigurationProcessor {
 MapString, PropertyUpdater multiSliderClientMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater multiYarnSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater multiOozieSiteMap = new HashMapString, 
PropertyUpdater();
-
-
-
-
-
 MapString, PropertyUpdater dbHiveSiteMap = new HashMapString, 
PropertyUpdater();
-
 MapString, PropertyUpdater nagiosEnvMap = new HashMapString 
,PropertyUpdater();
 
 
@@ -1006,6 +995,7 @@ public class BlueprintConfigurationProcessor {
 oozieSiteMap.put(oozie.service.HadoopAccessorService.kerberos.principal, 
new SingleHostTopologyUpdater(OOZIE_SERVER));
 oozieEnvMap.put(oozie_hostname, new 
SingleHostTopologyUpdater(OOZIE_SERVER));
 multiCoreSiteMap.put(hadoop.proxyuser.oozie.hosts, new 
MultipleHostTopologyUpdater(OOZIE_SERVER));
+
 // register updaters for Oozie properties that may point to an external DB
 oozieEnvOriginalValueMap.put(oozie_existing_mysql_host, new 
OriginalValuePropertyUpdater());
 oozieSiteOriginalValueMap.put(oozie.service.JPAService.jdbc.url, new 
OriginalValuePropertyUpdater());
@@ -1042,6 +1032,7 @@ public class BlueprintConfigurationProcessor {
 multiCoreSiteMap.put(hadoop.proxyuser.knox.hosts, new 
MultipleHostTopologyUpdater(KNOX_GATEWAY));
 multiWebhcatSiteMap.put(webhcat.proxyuser.knox.hosts, new 
MultipleHostTopologyUpdater(KNOX_GATEWAY));
 multiOozieSiteMap.put(hadoop.proxyuser.knox.hosts, new 
MultipleHostTopologyUpdater(KNOX_GATEWAY));
+
multiOozieSiteMap.put(oozie.service.ProxyUserService.proxyuser.knox.hosts, 
new MultipleHostTopologyUpdater(KNOX_GATEWAY));
 
 
 // Required due to AMBARI-4933.  These no longer seem to be required as 
the default values in the stack

ambari git commit: AMBARI-8226. Export topology related properties in blueprint that have undefined hosts

2014-11-07 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-1.7.0 61efc45d1 - 464537311


AMBARI-8226.  Export topology related properties in blueprint that have 
undefined hosts


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/46453731
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/46453731
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/46453731

Branch: refs/heads/branch-1.7.0
Commit: 464537311d997ccaba419af0d69101dcccb2cf4d
Parents: 61efc45
Author: John Speidel jspei...@hortonworks.com
Authored: Fri Nov 7 20:35:40 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Fri Nov 7 21:11:46 2014 -0500

--
 .../BlueprintConfigurationProcessor.java| 19 -
 .../BlueprintConfigurationProcessorTest.java| 44 +++-
 2 files changed, 59 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/46453731/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 5af81be..f1a6110 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -342,8 +342,12 @@ public class BlueprintConfigurationProcessor {
   // except in the case of HA-related properties, that
   // can contain nameservice references instead of hostnames (Fix for 
Bug AMBARI-7458).
   // also will not remove properties that reference the special 
0.0.0.0 network
-  // address
-  if (! matchedHost  ! isNameServiceProperty(propertyName)  
!isSpecialNetworkAddress(propValue)) {
+  // address or properties with undefined hosts
+  if (! matchedHost 
+  ! isNameServiceProperty(propertyName) 
+  ! isSpecialNetworkAddress(propValue)  
+  ! isUndefinedAddress(propValue)) {
+
 typeProperties.remove(propertyName);
   }
 }
@@ -380,6 +384,17 @@ public class BlueprintConfigurationProcessor {
   }
 
   /**
+   * Determine if a property has an undefined host.
+   *
+   * @param propertyValue  property value
+   *
+   * @return true if the property value contains undefined
+   */
+  private static boolean isUndefinedAddress(String propertyValue) {
+return propertyValue.contains(undefined);
+  }
+
+  /**
* Update multi host topology configuration properties for blueprint export.
*
* @param hostGroups  cluster host groups

http://git-wip-us.apache.org/repos/asf/ambari/blob/46453731/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 6c00927..48b9077 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -2042,8 +2042,6 @@ public class BlueprintConfigurationProcessorTest {
 MapString, String kafkaBrokerProperties =
   new HashMapString, String();
 
-
-
 configProperties.put(core-site, coreSiteProperties);
 configProperties.put(hbase-site, hbaseSiteProperties);
 configProperties.put(webhcat-site, webHCatSiteProperties);
@@ -2232,6 +2230,48 @@ public class BlueprintConfigurationProcessorTest {
 
   }
 
+  @Test
+  public void testPropertyWithUndefinedHostisExported() throws Exception {
+final String expectedHostName = c6401.apache.ambari.org;
+final String expectedHostGroupName = host_group_1;
+
+EasyMockSupport mockSupport = new EasyMockSupport();
+
+HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+
expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName,
 serverTwo)).atLeastOnce();
+
expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+
+mockSupport.replayAll();
+
+MapString, MapString, String configProperties = new HashMapString, 
MapString, String();
+
+MapString, String properties = new

ambari git commit: AMBARI-8226. Export topology related properties in blueprint that have undefined hosts

2014-11-07 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk ae2cd3d7a - b6326be3a


AMBARI-8226.  Export topology related properties in blueprint that have 
undefined hosts


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b6326be3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b6326be3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b6326be3

Branch: refs/heads/trunk
Commit: b6326be3ae26f5b3ca9f24d2b861b2bb8bad3f18
Parents: ae2cd3d
Author: John Speidel jspei...@hortonworks.com
Authored: Fri Nov 7 20:35:40 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Fri Nov 7 21:12:37 2014 -0500

--
 .../BlueprintConfigurationProcessor.java| 19 -
 .../BlueprintConfigurationProcessorTest.java| 44 +++-
 2 files changed, 59 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/b6326be3/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 5af81be..f1a6110 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -342,8 +342,12 @@ public class BlueprintConfigurationProcessor {
   // except in the case of HA-related properties, that
   // can contain nameservice references instead of hostnames (Fix for 
Bug AMBARI-7458).
   // also will not remove properties that reference the special 
0.0.0.0 network
-  // address
-  if (! matchedHost  ! isNameServiceProperty(propertyName)  
!isSpecialNetworkAddress(propValue)) {
+  // address or properties with undefined hosts
+  if (! matchedHost 
+  ! isNameServiceProperty(propertyName) 
+  ! isSpecialNetworkAddress(propValue)  
+  ! isUndefinedAddress(propValue)) {
+
 typeProperties.remove(propertyName);
   }
 }
@@ -380,6 +384,17 @@ public class BlueprintConfigurationProcessor {
   }
 
   /**
+   * Determine if a property has an undefined host.
+   *
+   * @param propertyValue  property value
+   *
+   * @return true if the property value contains undefined
+   */
+  private static boolean isUndefinedAddress(String propertyValue) {
+return propertyValue.contains(undefined);
+  }
+
+  /**
* Update multi host topology configuration properties for blueprint export.
*
* @param hostGroups  cluster host groups

http://git-wip-us.apache.org/repos/asf/ambari/blob/b6326be3/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 6c00927..48b9077 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -2042,8 +2042,6 @@ public class BlueprintConfigurationProcessorTest {
 MapString, String kafkaBrokerProperties =
   new HashMapString, String();
 
-
-
 configProperties.put(core-site, coreSiteProperties);
 configProperties.put(hbase-site, hbaseSiteProperties);
 configProperties.put(webhcat-site, webHCatSiteProperties);
@@ -2232,6 +2230,48 @@ public class BlueprintConfigurationProcessorTest {
 
   }
 
+  @Test
+  public void testPropertyWithUndefinedHostisExported() throws Exception {
+final String expectedHostName = c6401.apache.ambari.org;
+final String expectedHostGroupName = host_group_1;
+
+EasyMockSupport mockSupport = new EasyMockSupport();
+
+HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+
expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName,
 serverTwo)).atLeastOnce();
+
expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+
+mockSupport.replayAll();
+
+MapString, MapString, String configProperties = new HashMapString, 
MapString, String();
+
+MapString, String properties = new HashMapString,

git commit: AMBARI-8009. Fixes issue where multiple config versions are present after blueprint cluster install

2014-11-02 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-1.7.0 e52e4f855 - f8bb8fffd


AMBARI-8009.  Fixes issue where multiple config versions are present after 
blueprint cluster install


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f8bb8fff
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f8bb8fff
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f8bb8fff

Branch: refs/heads/branch-1.7.0
Commit: f8bb8fffdd29cfd9840028c2dd6fd027e075fe95
Parents: e52e4f8
Author: Robert Nettleton rnettle...@hortonworks.com
Authored: Sun Nov 2 21:50:15 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Sun Nov 2 21:50:15 2014 -0500

--
 .../server/controller/StackServiceResponse.java |   8 +
 .../internal/ClusterResourceProvider.java   | 215 +++---
 .../server/controller/internal/Stack.java   |  28 ++
 .../internal/BlueprintResourceProviderTest.java |   4 +
 .../internal/ClusterResourceProviderTest.java   | 295 +++
 5 files changed, 323 insertions(+), 227 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/f8bb8fff/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
index 9c986b1..a8e3451 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 public class StackServiceResponse {
 
@@ -39,6 +40,8 @@ public class StackServiceResponse {
   private ListString customCommands;
 
   private MapString, MapString, MapString, String configTypes;
+  private SetString excludedConfigTypes;
+
   private ListString requiredServices;
 
   /**
@@ -54,6 +57,7 @@ public class StackServiceResponse {
 comments = service.getComment();
 serviceVersion = service.getVersion();
 configTypes = service.getConfigTypes();
+excludedConfigTypes = service.getExcludedConfigTypes();
 requiredServices = service.getRequiredServices();
 serviceCheckSupported = null != service.getCommandScript();
 
@@ -128,6 +132,10 @@ public class StackServiceResponse {
   public MapString, MapString, MapString, String getConfigTypes() {
 return configTypes;
   }
+
+  public SetString getExcludedConfigTypes() {
+return excludedConfigTypes;
+  }
   
   public ListString getRequiredServices() {
 return requiredServices;

http://git-wip-us.apache.org/repos/asf/ambari/blob/f8bb8fff/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
index 52e966b..09c3853 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -434,7 +435,7 @@ public class ClusterResourceProvider extends 
BaseBlueprintProcessor {
 
 String clusterName = (String) properties.get(CLUSTER_NAME_PROPERTY_ID);
 createClusterResource(buildClusterResourceProperties(stack, clusterName));
-setConfigurationsOnCluster(clusterName);
+setConfigurationsOnCluster(clusterName, stack, blueprintHostGroups);
 
 SetString services = getServicesToDeploy(stack, blueprintHostGroups);
 
@@ -685,55 +686,134 @@ public class ClusterResourceProvider extends 
BaseBlueprintProcessor {
* Set all configurations on the cluster resource.
*
* @param clusterName  cluster name
+   * @param stack Stack definition object used for this cluster
+   * @param blueprintHostGroups host groups defined in the Blueprint for this 
cluster
*
* @throws SystemException an unexpected exception occurred
*/
-  private void setConfigurationsOnCluster(String clusterName) throws 
SystemException {
-for (Map.EntryString, MapString, String entry :

git commit: AMBARI-8009. Fixes issue where multiple configuration versions were present when deploying cluster via a blueprint

2014-11-02 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk d744db13a - b583fa9f5


AMBARI-8009.  Fixes issue where multiple configuration versions were present 
when deploying cluster via a blueprint


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b583fa9f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b583fa9f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b583fa9f

Branch: refs/heads/trunk
Commit: b583fa9f53c29a3a3640396fc8a2e58cc1a18fe8
Parents: d744db1
Author: Robert Nettleton rnettle...@hortonworks.com
Authored: Sun Nov 2 22:29:59 2014 -0500
Committer: John Speidel jspei...@hortonworks.com
Committed: Sun Nov 2 22:29:59 2014 -0500

--
 .../server/controller/StackServiceResponse.java |   8 +
 .../internal/ClusterResourceProvider.java   | 215 +++---
 .../server/controller/internal/Stack.java   |  28 ++
 .../internal/BlueprintResourceProviderTest.java |   4 +
 .../internal/ClusterResourceProviderTest.java   | 295 +++
 5 files changed, 323 insertions(+), 227 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/b583fa9f/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
index 9c986b1..a8e3451 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 public class StackServiceResponse {
 
@@ -39,6 +40,8 @@ public class StackServiceResponse {
   private ListString customCommands;
 
   private MapString, MapString, MapString, String configTypes;
+  private SetString excludedConfigTypes;
+
   private ListString requiredServices;
 
   /**
@@ -54,6 +57,7 @@ public class StackServiceResponse {
 comments = service.getComment();
 serviceVersion = service.getVersion();
 configTypes = service.getConfigTypes();
+excludedConfigTypes = service.getExcludedConfigTypes();
 requiredServices = service.getRequiredServices();
 serviceCheckSupported = null != service.getCommandScript();
 
@@ -128,6 +132,10 @@ public class StackServiceResponse {
   public MapString, MapString, MapString, String getConfigTypes() {
 return configTypes;
   }
+
+  public SetString getExcludedConfigTypes() {
+return excludedConfigTypes;
+  }
   
   public ListString getRequiredServices() {
 return requiredServices;

http://git-wip-us.apache.org/repos/asf/ambari/blob/b583fa9f/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
index 8dd06ec..96b4c8c 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -434,7 +435,7 @@ public class ClusterResourceProvider extends 
BaseBlueprintProcessor {
 
 String clusterName = (String) properties.get(CLUSTER_NAME_PROPERTY_ID);
 createClusterResource(buildClusterResourceProperties(stack, clusterName));
-setConfigurationsOnCluster(clusterName);
+setConfigurationsOnCluster(clusterName, stack, blueprintHostGroups);
 
 SetString services = getServicesToDeploy(stack, blueprintHostGroups);
 
@@ -685,55 +686,134 @@ public class ClusterResourceProvider extends 
BaseBlueprintProcessor {
* Set all configurations on the cluster resource.
*
* @param clusterName  cluster name
+   * @param stack Stack definition object used for this cluster
+   * @param blueprintHostGroups host groups defined in the Blueprint for this 
cluster
*
* @throws SystemException an unexpected exception occurred
*/
-  private void setConfigurationsOnCluster(String clusterName) throws 
SystemException {
-for (Map.EntryString, MapString, String entry :

git commit: AMBARI-8021. Adds Blueprint export handling for Knox and Zookeeper properties in HDP 2.2 stack

2014-10-30 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-1.7.0 ce051947e - 8fa3ea45a


AMBARI-8021.  Adds Blueprint export handling for Knox and Zookeeper properties 
in HDP 2.2 stack


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8fa3ea45
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8fa3ea45
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8fa3ea45

Branch: refs/heads/branch-1.7.0
Commit: 8fa3ea45a23c3b4b455252853fb7c24d95121f45
Parents: ce05194
Author: Robert Nettleton rnettle...@hortonworks.com
Authored: Thu Oct 30 11:52:11 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Oct 30 12:33:55 2014 -0400

--
 .../BlueprintConfigurationProcessor.java|  25 
 .../BlueprintConfigurationProcessorTest.java| 137 +++
 2 files changed, 162 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/8fa3ea45/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index be379c9..9237bc9 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -882,6 +882,8 @@ public class BlueprintConfigurationProcessor {
 MapString, PropertyUpdater oozieSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater stormSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater falconStartupPropertiesMap = new 
HashMapString, PropertyUpdater();
+MapString, PropertyUpdater kafkaBrokerMap = new HashMapString, 
PropertyUpdater();
+
 
 
 MapString, PropertyUpdater mapredEnvMap = new HashMapString, 
PropertyUpdater();
@@ -898,6 +900,13 @@ public class BlueprintConfigurationProcessor {
 MapString, PropertyUpdater multiCoreSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater multiHdfsSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater multiHiveSiteMap = new HashMapString, 
PropertyUpdater();
+MapString, PropertyUpdater multiKafkaBrokerMap = new HashMapString, 
PropertyUpdater();
+MapString, PropertyUpdater multiSliderClientMap = new HashMapString, 
PropertyUpdater();
+MapString, PropertyUpdater multiYarnSiteMap = new HashMapString, 
PropertyUpdater();
+MapString, PropertyUpdater multiOozieSiteMap = new HashMapString, 
PropertyUpdater();
+
+
+
 
 
 MapString, PropertyUpdater dbHiveSiteMap = new HashMapString, 
PropertyUpdater();
@@ -917,6 +926,7 @@ public class BlueprintConfigurationProcessor {
 singleHostTopologyUpdaters.put(nagios-env, nagiosEnvMap);
 singleHostTopologyUpdaters.put(hive-env, hiveEnvMap);
 singleHostTopologyUpdaters.put(oozie-env, oozieEnvMap);
+singleHostTopologyUpdaters.put(kafka-broker, kafkaBrokerMap);
 
 mPropertyUpdaters.put(hadoop-env, hadoopEnvMap);
 mPropertyUpdaters.put(hbase-env, hbaseEnvMap);
@@ -928,6 +938,10 @@ public class BlueprintConfigurationProcessor {
 multiHostTopologyUpdaters.put(core-site, multiCoreSiteMap);
 multiHostTopologyUpdaters.put(hdfs-site, multiHdfsSiteMap);
 multiHostTopologyUpdaters.put(hive-site, multiHiveSiteMap);
+multiHostTopologyUpdaters.put(kafka-broker, multiKafkaBrokerMap);
+multiHostTopologyUpdaters.put(slider-client, multiSliderClientMap);
+multiHostTopologyUpdaters.put(yarn-site, multiYarnSiteMap);
+multiHostTopologyUpdaters.put(oozie-site, multiOozieSiteMap);
 
 dbHostTopologyUpdaters.put(hive-site, dbHiveSiteMap);
 
@@ -1000,6 +1014,9 @@ public class BlueprintConfigurationProcessor {
 multiHbaseSiteMap.put(hbase.zookeeper.quorum, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
 multiWebhcatSiteMap.put(templeton.zookeeper.hosts, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
 multiCoreSiteMap.put(ha.zookeeper.quorum, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
+multiYarnSiteMap.put(hadoop.registry.zk.quorum, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
+multiSliderClientMap.put(slider.zookeeper.quorum, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
+multiKafkaBrokerMap.put(zookeeper.connect, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
 
 // STORM
 stormSiteMap.put(nimbus.host, new SingleHostTopologyUpdater(NIMBUS));
@@ -1018,6 +1035,14 @@ public class BlueprintConfigurationProcessor {

git commit: AMBARI-8021. Adds Blueprint export handling for Knox and Zookeeper properties in HDP 2.2 stack

2014-10-30 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk d8e8a0915 - ce69d0077


AMBARI-8021.  Adds Blueprint export handling for Knox and Zookeeper properties 
in HDP 2.2 stack


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ce69d007
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ce69d007
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ce69d007

Branch: refs/heads/trunk
Commit: ce69d0077c9f45f9ad0824a40352771604b3b17a
Parents: d8e8a09
Author: Robert Nettleton rnettle...@hortonworks.com
Authored: Thu Oct 30 11:52:11 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Oct 30 13:39:08 2014 -0400

--
 .../BlueprintConfigurationProcessor.java|  25 
 .../BlueprintConfigurationProcessorTest.java| 137 +++
 2 files changed, 162 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/ce69d007/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index be379c9..9237bc9 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -882,6 +882,8 @@ public class BlueprintConfigurationProcessor {
 MapString, PropertyUpdater oozieSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater stormSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater falconStartupPropertiesMap = new 
HashMapString, PropertyUpdater();
+MapString, PropertyUpdater kafkaBrokerMap = new HashMapString, 
PropertyUpdater();
+
 
 
 MapString, PropertyUpdater mapredEnvMap = new HashMapString, 
PropertyUpdater();
@@ -898,6 +900,13 @@ public class BlueprintConfigurationProcessor {
 MapString, PropertyUpdater multiCoreSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater multiHdfsSiteMap = new HashMapString, 
PropertyUpdater();
 MapString, PropertyUpdater multiHiveSiteMap = new HashMapString, 
PropertyUpdater();
+MapString, PropertyUpdater multiKafkaBrokerMap = new HashMapString, 
PropertyUpdater();
+MapString, PropertyUpdater multiSliderClientMap = new HashMapString, 
PropertyUpdater();
+MapString, PropertyUpdater multiYarnSiteMap = new HashMapString, 
PropertyUpdater();
+MapString, PropertyUpdater multiOozieSiteMap = new HashMapString, 
PropertyUpdater();
+
+
+
 
 
 MapString, PropertyUpdater dbHiveSiteMap = new HashMapString, 
PropertyUpdater();
@@ -917,6 +926,7 @@ public class BlueprintConfigurationProcessor {
 singleHostTopologyUpdaters.put(nagios-env, nagiosEnvMap);
 singleHostTopologyUpdaters.put(hive-env, hiveEnvMap);
 singleHostTopologyUpdaters.put(oozie-env, oozieEnvMap);
+singleHostTopologyUpdaters.put(kafka-broker, kafkaBrokerMap);
 
 mPropertyUpdaters.put(hadoop-env, hadoopEnvMap);
 mPropertyUpdaters.put(hbase-env, hbaseEnvMap);
@@ -928,6 +938,10 @@ public class BlueprintConfigurationProcessor {
 multiHostTopologyUpdaters.put(core-site, multiCoreSiteMap);
 multiHostTopologyUpdaters.put(hdfs-site, multiHdfsSiteMap);
 multiHostTopologyUpdaters.put(hive-site, multiHiveSiteMap);
+multiHostTopologyUpdaters.put(kafka-broker, multiKafkaBrokerMap);
+multiHostTopologyUpdaters.put(slider-client, multiSliderClientMap);
+multiHostTopologyUpdaters.put(yarn-site, multiYarnSiteMap);
+multiHostTopologyUpdaters.put(oozie-site, multiOozieSiteMap);
 
 dbHostTopologyUpdaters.put(hive-site, dbHiveSiteMap);
 
@@ -1000,6 +1014,9 @@ public class BlueprintConfigurationProcessor {
 multiHbaseSiteMap.put(hbase.zookeeper.quorum, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
 multiWebhcatSiteMap.put(templeton.zookeeper.hosts, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
 multiCoreSiteMap.put(ha.zookeeper.quorum, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
+multiYarnSiteMap.put(hadoop.registry.zk.quorum, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
+multiSliderClientMap.put(slider.zookeeper.quorum, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
+multiKafkaBrokerMap.put(zookeeper.connect, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
 
 // STORM
 stormSiteMap.put(nimbus.host, new SingleHostTopologyUpdater(NIMBUS));
@@ -1018,6 +1035,14 @@ public class BlueprintConfigurationProcessor {
 // NAGIOS

git commit: AMBARI-7988. Properly export topology related properties with a value of 0.0.0.0 to blueprints

2014-10-27 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-1.7.0 c115cf830 - 14b916cdb


AMBARI-7988.  Properly export topology related properties with a value of 
0.0.0.0 to blueprints


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/14b916cd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/14b916cd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/14b916cd

Branch: refs/heads/branch-1.7.0
Commit: 14b916cdb723745b49ab993ca9101fd1d2272443
Parents: c115cf8
Author: John Speidel jspei...@hortonworks.com
Authored: Mon Oct 27 20:34:20 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Oct 27 20:37:06 2014 -0400

--
 .../BlueprintConfigurationProcessor.java| 19 +-
 .../BlueprintConfigurationProcessorTest.java| 66 
 2 files changed, 84 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/14b916cd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 4183b9d..be379c9 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -341,7 +341,9 @@ public class BlueprintConfigurationProcessor {
   // remove properties that do not contain hostnames,
   // except in the case of HA-related properties, that
   // can contain nameservice references instead of hostnames (Fix for 
Bug AMBARI-7458).
-  if (! matchedHost  ! isNameServiceProperty(propertyName)) {
+  // also will not remove properties that reference the special 
0.0.0.0 network
+  // address
+  if (! matchedHost  ! isNameServiceProperty(propertyName)  
!isSpecialNetworkAddress(propValue)) {
 typeProperties.remove(propertyName);
   }
 }
@@ -363,6 +365,21 @@ public class BlueprintConfigurationProcessor {
   }
 
   /**
+   * Queries a property value to determine if the value contains
+   *   a host address with all zeros (0.0.0.0).  This is a special
+   *   address that signifies that the service is available on
+   *   all network interfaces on a given machine.
+   *
+   * @param propertyValue the property value to inspect
+   *
+   * @return true if the 0.0.0.0 address is included in this string
+   * false if the 0.0.0.0 address is not included in this string
+   */
+  private static boolean isSpecialNetworkAddress(String propertyValue) {
+return propertyValue.contains(0.0.0.0);
+  }
+
+  /**
* Update multi host topology configuration properties for blueprint export.
*
* @param hostGroups  cluster host groups

http://git-wip-us.apache.org/repos/asf/ambari/blob/14b916cd/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 04c75a1..cf06064 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -1696,6 +1696,72 @@ public class BlueprintConfigurationProcessorTest {
   }
 
   @Test
+  public void testYarnConfigExportedWithDefaultZeroHostAddress() throws 
Exception {
+final String expectedHostName = c6401.apache.ambari.org;
+final String expectedPortNum = 808080;
+final String expectedHostGroupName = host_group_1;
+
+EasyMockSupport mockSupport = new EasyMockSupport();
+
+HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+
expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName,
 serverTwo)).atLeastOnce();
+
expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+
+mockSupport.replayAll();
+
+MapString, MapString, String configProperties =
+  new HashMapString, MapString, String();
+
+MapString, String yarnSiteProperties =
+  new HashMapString, String();
+
+configProperties.put(yarn-site, yarnSiteProperties);
+
+

git commit: AMBARI-7988. Ensure that topology related properties with the value of 0.0.0.0 are exported to blueprint

2014-10-27 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk eebb085e8 - f1720123b


AMBARI-7988.  Ensure that topology related properties with the value of 0.0.0.0 
are exported to blueprint


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f1720123
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f1720123
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f1720123

Branch: refs/heads/trunk
Commit: f1720123b881b2377fef73bc4f3f7b922c50c64d
Parents: eebb085
Author: Robert Nettleton rnettle...@hortonworks.com
Authored: Mon Oct 27 19:55:00 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Oct 27 20:49:17 2014 -0400

--
 .../BlueprintConfigurationProcessor.java| 19 +-
 .../BlueprintConfigurationProcessorTest.java| 66 
 2 files changed, 84 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/f1720123/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 4183b9d..be379c9 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -341,7 +341,9 @@ public class BlueprintConfigurationProcessor {
   // remove properties that do not contain hostnames,
   // except in the case of HA-related properties, that
   // can contain nameservice references instead of hostnames (Fix for 
Bug AMBARI-7458).
-  if (! matchedHost  ! isNameServiceProperty(propertyName)) {
+  // also will not remove properties that reference the special 
0.0.0.0 network
+  // address
+  if (! matchedHost  ! isNameServiceProperty(propertyName)  
!isSpecialNetworkAddress(propValue)) {
 typeProperties.remove(propertyName);
   }
 }
@@ -363,6 +365,21 @@ public class BlueprintConfigurationProcessor {
   }
 
   /**
+   * Queries a property value to determine if the value contains
+   *   a host address with all zeros (0.0.0.0).  This is a special
+   *   address that signifies that the service is available on
+   *   all network interfaces on a given machine.
+   *
+   * @param propertyValue the property value to inspect
+   *
+   * @return true if the 0.0.0.0 address is included in this string
+   * false if the 0.0.0.0 address is not included in this string
+   */
+  private static boolean isSpecialNetworkAddress(String propertyValue) {
+return propertyValue.contains(0.0.0.0);
+  }
+
+  /**
* Update multi host topology configuration properties for blueprint export.
*
* @param hostGroups  cluster host groups

http://git-wip-us.apache.org/repos/asf/ambari/blob/f1720123/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 04c75a1..cf06064 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -1696,6 +1696,72 @@ public class BlueprintConfigurationProcessorTest {
   }
 
   @Test
+  public void testYarnConfigExportedWithDefaultZeroHostAddress() throws 
Exception {
+final String expectedHostName = c6401.apache.ambari.org;
+final String expectedPortNum = 808080;
+final String expectedHostGroupName = host_group_1;
+
+EasyMockSupport mockSupport = new EasyMockSupport();
+
+HostGroup mockHostGroupOne = mockSupport.createMock(HostGroup.class);
+
+
expect(mockHostGroupOne.getHostInfo()).andReturn(Arrays.asList(expectedHostName,
 serverTwo)).atLeastOnce();
+
expect(mockHostGroupOne.getName()).andReturn(expectedHostGroupName).atLeastOnce();
+
+mockSupport.replayAll();
+
+MapString, MapString, String configProperties =
+  new HashMapString, MapString, String();
+
+MapString, String yarnSiteProperties =
+  new HashMapString, String();
+
+configProperties.put(yarn-site, yarnSiteProperties);
+
+

git commit: AMBARI-7738. Fix blueprint processor to properly handle config topology update for components with a valid cardinality of 0

2014-10-23 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 27a9d78c9 - c19b78966


AMBARI-7738.  Fix blueprint processor to properly handle config topology update 
for components with a valid cardinality of 0


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c19b7896
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c19b7896
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c19b7896

Branch: refs/heads/trunk
Commit: c19b7896667bbeedf83525b7b7c7e62e31a96b57
Parents: 27a9d78
Author: Robert Nettleton rnettle...@hortonworks.com
Authored: Thu Oct 23 14:44:55 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Oct 23 14:45:51 2014 -0400

--
 .../internal/BaseBlueprintProcessor.java| 487 ---
 .../BlueprintConfigurationProcessor.java|  89 ++--
 .../server/controller/internal/Cardinality.java |  86 
 .../internal/ClusterResourceProvider.java   |   4 +-
 .../server/controller/internal/Stack.java   | 450 +
 .../internal/BaseBlueprintProcessorTest.java|  24 +-
 .../BlueprintConfigurationProcessorTest.java| 211 +++-
 .../internal/ClusterResourceProviderTest.java   |  24 +-
 8 files changed, 812 insertions(+), 563 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/c19b7896/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
index abd22a4..9be73cc 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
@@ -24,13 +24,6 @@ import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.StackAccessException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.StackConfigurationRequest;
-import org.apache.ambari.server.controller.StackConfigurationResponse;
-import org.apache.ambari.server.controller.StackLevelConfigurationRequest;
-import org.apache.ambari.server.controller.StackServiceComponentRequest;
-import org.apache.ambari.server.controller.StackServiceComponentResponse;
-import org.apache.ambari.server.controller.StackServiceRequest;
-import org.apache.ambari.server.controller.StackServiceResponse;
 import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.orm.dao.BlueprintDAO;
@@ -42,10 +35,8 @@ import org.apache.ambari.server.orm.entities.HostGroupEntity;
 import org.apache.ambari.server.state.AutoDeployInfo;
 import org.apache.ambari.server.state.ConfigHelper;
 import org.apache.ambari.server.state.DependencyInfo;
-import org.apache.ambari.server.state.PropertyInfo;
 
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -455,418 +446,6 @@ public abstract class BaseBlueprintProcessor extends 
AbstractControllerResourceP
   // - Inner Classes -
 
   /**
-   * Encapsulates stack information.
-   */
-  protected static class Stack {
-/**
- * Stack name
- */
-private String name;
-
-/**
- * Stack version
- */
-private String version;
-
-/**
- * Map of service name to components
- */
-private MapString, CollectionString serviceComponents =
-new HashMapString, CollectionString();
-
-/**
- * Map of component to service
- */
-private MapString, String componentService = new HashMapString, 
String();
-
-/**
- * Map of component to dependencies
- */
-private MapString, CollectionDependencyInfo dependencies =
-new HashMapString, CollectionDependencyInfo();
-
-/**
- * Map of dependency to conditional service
- */
-private MapDependencyInfo, String dependencyConditionalServiceMap =
-new HashMapDependencyInfo, String();
-
-/**
- * Map of database component name to configuration property which 
indicates whether
- * the database in to be managed or if it is an external non-managed 
instance.
- * If the value of the config property starts with 'New', the database is 
determined
- * to be managed, otherwise it is non-managed.
- */
-private MapString, String dbDependencyInfo = new

git commit: AMBARI-7738. Fix blueprint processor to properly handle config topology update for components with a valid cardinality of 0

2014-10-23 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-1.7.0 1cc2d64e5 - a736a0cfa


AMBARI-7738.  Fix blueprint processor to properly handle config topology update 
for components with a valid cardinality of 0


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a736a0cf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a736a0cf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a736a0cf

Branch: refs/heads/branch-1.7.0
Commit: a736a0cfa5c6add7c2223560ac8bd14d42003c88
Parents: 1cc2d64
Author: Robert Nettleton rnettle...@hortonworks.com
Authored: Thu Oct 23 14:44:55 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Oct 23 14:47:00 2014 -0400

--
 .../internal/BaseBlueprintProcessor.java| 487 ---
 .../BlueprintConfigurationProcessor.java|  89 ++--
 .../server/controller/internal/Cardinality.java |  86 
 .../internal/ClusterResourceProvider.java   |   4 +-
 .../server/controller/internal/Stack.java   | 450 +
 .../internal/BaseBlueprintProcessorTest.java|  24 +-
 .../BlueprintConfigurationProcessorTest.java| 211 +++-
 .../internal/ClusterResourceProviderTest.java   |  24 +-
 8 files changed, 812 insertions(+), 563 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/a736a0cf/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
index abd22a4..9be73cc 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java
@@ -24,13 +24,6 @@ import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.StackAccessException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.StackConfigurationRequest;
-import org.apache.ambari.server.controller.StackConfigurationResponse;
-import org.apache.ambari.server.controller.StackLevelConfigurationRequest;
-import org.apache.ambari.server.controller.StackServiceComponentRequest;
-import org.apache.ambari.server.controller.StackServiceComponentResponse;
-import org.apache.ambari.server.controller.StackServiceRequest;
-import org.apache.ambari.server.controller.StackServiceResponse;
 import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.orm.dao.BlueprintDAO;
@@ -42,10 +35,8 @@ import org.apache.ambari.server.orm.entities.HostGroupEntity;
 import org.apache.ambari.server.state.AutoDeployInfo;
 import org.apache.ambari.server.state.ConfigHelper;
 import org.apache.ambari.server.state.DependencyInfo;
-import org.apache.ambari.server.state.PropertyInfo;
 
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -455,418 +446,6 @@ public abstract class BaseBlueprintProcessor extends 
AbstractControllerResourceP
   // - Inner Classes -
 
   /**
-   * Encapsulates stack information.
-   */
-  protected static class Stack {
-/**
- * Stack name
- */
-private String name;
-
-/**
- * Stack version
- */
-private String version;
-
-/**
- * Map of service name to components
- */
-private MapString, CollectionString serviceComponents =
-new HashMapString, CollectionString();
-
-/**
- * Map of component to service
- */
-private MapString, String componentService = new HashMapString, 
String();
-
-/**
- * Map of component to dependencies
- */
-private MapString, CollectionDependencyInfo dependencies =
-new HashMapString, CollectionDependencyInfo();
-
-/**
- * Map of dependency to conditional service
- */
-private MapDependencyInfo, String dependencyConditionalServiceMap =
-new HashMapDependencyInfo, String();
-
-/**
- * Map of database component name to configuration property which 
indicates whether
- * the database in to be managed or if it is an external non-managed 
instance.
- * If the value of the config property starts with 'New', the database is 
determined
- * to be managed, otherwise it is non-managed.
- */
-private MapString, String

git commit: AMBARI-7870. Change ShellCommandUtil.Results class from package private to public

2014-10-23 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 4c56e4e66 - 897b67a55


AMBARI-7870.  Change ShellCommandUtil.Results class from package private to 
public


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/897b67a5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/897b67a5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/897b67a5

Branch: refs/heads/trunk
Commit: 897b67a5549b1fa93b339370c789764c489ed471
Parents: 4c56e4e
Author: Robert Levas rle...@hortonworks.com
Authored: Thu Oct 23 16:21:03 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Thu Oct 23 17:22:57 2014 -0400

--
 .../apache/ambari/server/utils/ShellCommandUtil.java |  8 
 .../ambari/server/utils/TestShellCommandUtil.java| 15 ++-
 2 files changed, 18 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/897b67a5/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java
index ee83aa0..947b336 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/utils/ShellCommandUtil.java
@@ -200,7 +200,7 @@ public class ShellCommandUtil {
 return sb.toString();
   }
 
-  static class Result {
+  public static class Result {
 
 Result(int exitCode, String stdout, String stderr) {
   this.exitCode = exitCode;
@@ -212,15 +212,15 @@ public class ShellCommandUtil {
 private final String stdout;
 private final String stderr;
 
-int getExitCode() {
+public int getExitCode() {
   return exitCode;
 }
 
-String getStdout() {
+public String getStdout() {
   return stdout;
 }
 
-String getStderr() {
+public String getStderr() {
   return stderr;
 }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/897b67a5/ambari-server/src/test/java/org/apache/ambari/server/utils/TestShellCommandUtil.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/utils/TestShellCommandUtil.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/utils/TestShellCommandUtil.java
index 27a033c..fc27eda 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/utils/TestShellCommandUtil.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/utils/TestShellCommandUtil.java
@@ -37,6 +37,10 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.security.ProtectionDomain;
+import java.util.Arrays;
 import java.util.Properties;
 import java.util.Random;
 
@@ -142,5 +146,14 @@ public class TestShellCommandUtil extends TestCase {
 
assertFalse(ShellCommandUtil.hideOpenSslPassword(command_pass).contains(1234));
 
assertFalse(ShellCommandUtil.hideOpenSslPassword(command_key).contains(1234));
   }
-  
+
+  public void testResultsClassIsPublic() throws Exception {
+Class resultClass = ShellCommandUtil.Result.class;
+
+assertEquals(Modifier.PUBLIC, resultClass.getModifiers()  
Modifier.PUBLIC);
+
+for(Method method : resultClass.getMethods()) {
+  assertEquals(method.getName(), Modifier.PUBLIC, (method.getModifiers()  
Modifier.PUBLIC));
+}
+  }
 }

git commit: AMBARI-7869. Added export handling for new Hive/Zookeeper config property in HDP 2.2

2014-10-22 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk a64a37055 - 2e3658194


AMBARI-7869.  Added export handling for new Hive/Zookeeper config property in 
HDP 2.2


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2e365819
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2e365819
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2e365819

Branch: refs/heads/trunk
Commit: 2e3658194b7c5d7a41588931aff35754ac261ac4
Parents: a64a370
Author: Robert Nettleton rnettle...@hortonworks.com
Authored: Wed Oct 22 13:19:41 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed Oct 22 13:19:41 2014 -0400

--
 .../controller/internal/BlueprintConfigurationProcessor.java| 1 +
 .../internal/BlueprintConfigurationProcessorTest.java   | 5 +
 2 files changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/2e365819/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 0b58c8d..10cc016 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -940,6 +940,7 @@ public class BlueprintConfigurationProcessor {
 multiWebhcatSiteMap.put(templeton.kerberos.principal, new 
MultipleHostTopologyUpdater(WEBHCAT_SERVER));
 hiveEnvMap.put(hive_hostname, new 
SingleHostTopologyUpdater(HIVE_SERVER));
 multiHiveSiteMap.put(hive.zookeeper.quorum, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
+
multiHiveSiteMap.put(hive.cluster.delegation.token.store.zookeeper.connectString,
 new MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
 
 // OOZIE_SERVER
 oozieSiteMap.put(oozie.base.url, new 
SingleHostTopologyUpdater(OOZIE_SERVER));

http://git-wip-us.apache.org/repos/asf/ambari/blob/2e365819/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index a57bacb..260d043 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -1641,6 +1641,7 @@ public class BlueprintConfigurationProcessorTest {
 hiveSiteProperties.put(hive.metastore.uris, expectedHostName + : + 
expectedPortNum);
 hiveSiteProperties.put(javax.jdo.option.ConnectionURL, expectedHostName 
+ : + expectedPortNum);
 hiveSiteProperties.put(hive.zookeeper.quorum, expectedHostName + : + 
expectedPortNum + , + expectedHostNameTwo + : + expectedPortNum);
+
hiveSiteProperties.put(hive.cluster.delegation.token.store.zookeeper.connectString,
 expectedHostName + : + expectedPortNum + , + expectedHostNameTwo + : + 
expectedPortNum);
 hiveEnvProperties.put(hive_hostname, expectedHostName);
 
 
@@ -1683,6 +1684,10 @@ public class BlueprintConfigurationProcessorTest {
   createExportedAddress(expectedPortNum, expectedHostGroupName) + , + 
createExportedAddress(expectedPortNum, expectedHostGroupNameTwo),
   hiveSiteProperties.get(hive.zookeeper.quorum));
 
+assertEquals(hive zookeeper connectString property not properly exported,
+  createExportedAddress(expectedPortNum, expectedHostGroupName) + , + 
createExportedAddress(expectedPortNum, expectedHostGroupNameTwo),
+  
hiveSiteProperties.get(hive.cluster.delegation.token.store.zookeeper.connectString));
+
 mockSupport.verifyAll();
   }

git commit: AMBARI-7869. Added export handling for new Hive/Zookeeper config property in HDP 2.2

2014-10-22 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-1.7.0 de49f5e28 - 14302687e


AMBARI-7869.  Added export handling for new Hive/Zookeeper config property in 
HDP 2.2


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/14302687
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/14302687
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/14302687

Branch: refs/heads/branch-1.7.0
Commit: 14302687e811a4a17dd261d7ff95a68c4afb6e07
Parents: de49f5e
Author: Robert Nettleton rnettle...@hortonworks.com
Authored: Wed Oct 22 13:19:41 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed Oct 22 13:21:24 2014 -0400

--
 .../controller/internal/BlueprintConfigurationProcessor.java| 1 +
 .../internal/BlueprintConfigurationProcessorTest.java   | 5 +
 2 files changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/14302687/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 0b58c8d..10cc016 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -940,6 +940,7 @@ public class BlueprintConfigurationProcessor {
 multiWebhcatSiteMap.put(templeton.kerberos.principal, new 
MultipleHostTopologyUpdater(WEBHCAT_SERVER));
 hiveEnvMap.put(hive_hostname, new 
SingleHostTopologyUpdater(HIVE_SERVER));
 multiHiveSiteMap.put(hive.zookeeper.quorum, new 
MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
+
multiHiveSiteMap.put(hive.cluster.delegation.token.store.zookeeper.connectString,
 new MultipleHostTopologyUpdater(ZOOKEEPER_SERVER));
 
 // OOZIE_SERVER
 oozieSiteMap.put(oozie.base.url, new 
SingleHostTopologyUpdater(OOZIE_SERVER));

http://git-wip-us.apache.org/repos/asf/ambari/blob/14302687/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index a57bacb..260d043 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -1641,6 +1641,7 @@ public class BlueprintConfigurationProcessorTest {
 hiveSiteProperties.put(hive.metastore.uris, expectedHostName + : + 
expectedPortNum);
 hiveSiteProperties.put(javax.jdo.option.ConnectionURL, expectedHostName 
+ : + expectedPortNum);
 hiveSiteProperties.put(hive.zookeeper.quorum, expectedHostName + : + 
expectedPortNum + , + expectedHostNameTwo + : + expectedPortNum);
+
hiveSiteProperties.put(hive.cluster.delegation.token.store.zookeeper.connectString,
 expectedHostName + : + expectedPortNum + , + expectedHostNameTwo + : + 
expectedPortNum);
 hiveEnvProperties.put(hive_hostname, expectedHostName);
 
 
@@ -1683,6 +1684,10 @@ public class BlueprintConfigurationProcessorTest {
   createExportedAddress(expectedPortNum, expectedHostGroupName) + , + 
createExportedAddress(expectedPortNum, expectedHostGroupNameTwo),
   hiveSiteProperties.get(hive.zookeeper.quorum));
 
+assertEquals(hive zookeeper connectString property not properly exported,
+  createExportedAddress(expectedPortNum, expectedHostGroupName) + , + 
createExportedAddress(expectedPortNum, expectedHostGroupNameTwo),
+  
hiveSiteProperties.get(hive.cluster.delegation.token.store.zookeeper.connectString));
+
 mockSupport.verifyAll();
   }

git commit: AMBARI-7835. Blueprint export now properly handles configurations with no properties.

2014-10-21 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 55fe12731 - 64a73207e


AMBARI-7835.  Blueprint export now properly handles configurations with no 
properties.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/64a73207
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/64a73207
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/64a73207

Branch: refs/heads/trunk
Commit: 64a73207ec4a4131af420a84e374b94f4ee7c220
Parents: 55fe127
Author: Robert Nettleton rnettle...@hortonworks.com
Authored: Tue Oct 21 10:44:19 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Tue Oct 21 10:44:19 2014 -0400

--
 .../query/render/ClusterBlueprintRenderer.java  | 15 -
 .../render/ClusterBlueprintRendererTest.java| 65 ++--
 2 files changed, 75 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/64a73207/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
index 1afae39..847735d 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
@@ -37,6 +37,8 @@ import 
org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.apache.ambari.server.state.DesiredConfig;
 import org.apache.ambari.server.state.HostConfig;
 import org.apache.ambari.server.state.PropertyInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.net.InetAddress;
 import java.net.UnknownHostException;
@@ -66,6 +68,8 @@ public class ClusterBlueprintRenderer extends BaseRenderer 
implements Renderer {
*/
   private MapString, CollectionString propertiesToStrip = new 
HashMapString, CollectionString();
 
+  private final static Logger LOG = 
LoggerFactory.getLogger(ClusterBlueprintRenderer.class);
+
 
   // - Renderer --
 
@@ -538,7 +542,16 @@ public class ClusterBlueprintRenderer extends BaseRenderer 
implements Renderer {
 
   // property map type is currently String, Object
   properties = (Map) 
configNode.getObject().getPropertiesMap().get(properties);
-  stripRequiredProperties(properties);
+
+  if (properties != null) {
+stripRequiredProperties(properties);
+  } else {
+LOG.warn(Empty configuration found for configuration type =  + type +
+   during Blueprint export.  This may occur after an upgrade of 
Ambari, when +
+  attempting to export a Blueprint from a cluster started by an older 
version of  +
+  Ambari.);
+  }
+
 }
 
 /**

http://git-wip-us.apache.org/repos/asf/ambari/blob/64a73207/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
index 04d86ad..6e58267 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
@@ -31,7 +31,6 @@ import 
org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.internal.ResourceImpl;
 import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.state.PropertyInfo;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import java.net.InetAddress;
@@ -99,7 +98,6 @@ public class ClusterBlueprintRendererTest {
 
assertTrue(propertyTree.getChild(Host/HostComponent).getObject().contains(HostRoles/component_name));
   }
 
-  @Ignore
   @Test
   public void testFinalizeResult() throws Exception{
 
@@ -124,7 +122,6 @@ public class ClusterBlueprintRendererTest {
 Resource blueprintResource = blueprintNode.getObject();
 MapString, MapString, Object properties = 
blueprintResource.getPropertiesMap();
 
-assertEquals(blueprint-testCluster, 
properties.get(Blueprints).get(blueprint_name));
 assertEquals(HDP, properties.get(Blueprints).get(stack_name));
 assertEquals(1.3.3, properties.get(Blueprints).get(stack_version));
 
@@ -176,11 +173,71 @@ public class

git commit: AMBARI-7835. Blueprint export now properly handles configurations with no properties.

2014-10-21 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-1.7.0 fe60f057e - c1e024521


AMBARI-7835.  Blueprint export now properly handles configurations with no 
properties.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c1e02452
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c1e02452
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c1e02452

Branch: refs/heads/branch-1.7.0
Commit: c1e02452164f4e1faef76943d5fe115a0eac0131
Parents: fe60f05
Author: Robert Nettleton rnettle...@hortonworks.com
Authored: Tue Oct 21 10:44:19 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Tue Oct 21 11:16:41 2014 -0400

--
 .../query/render/ClusterBlueprintRenderer.java  | 15 -
 .../render/ClusterBlueprintRendererTest.java| 65 ++--
 2 files changed, 75 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/c1e02452/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
index 1afae39..847735d 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
@@ -37,6 +37,8 @@ import 
org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.apache.ambari.server.state.DesiredConfig;
 import org.apache.ambari.server.state.HostConfig;
 import org.apache.ambari.server.state.PropertyInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.net.InetAddress;
 import java.net.UnknownHostException;
@@ -66,6 +68,8 @@ public class ClusterBlueprintRenderer extends BaseRenderer 
implements Renderer {
*/
   private MapString, CollectionString propertiesToStrip = new 
HashMapString, CollectionString();
 
+  private final static Logger LOG = 
LoggerFactory.getLogger(ClusterBlueprintRenderer.class);
+
 
   // - Renderer --
 
@@ -538,7 +542,16 @@ public class ClusterBlueprintRenderer extends BaseRenderer 
implements Renderer {
 
   // property map type is currently String, Object
   properties = (Map) 
configNode.getObject().getPropertiesMap().get(properties);
-  stripRequiredProperties(properties);
+
+  if (properties != null) {
+stripRequiredProperties(properties);
+  } else {
+LOG.warn(Empty configuration found for configuration type =  + type +
+   during Blueprint export.  This may occur after an upgrade of 
Ambari, when +
+  attempting to export a Blueprint from a cluster started by an older 
version of  +
+  Ambari.);
+  }
+
 }
 
 /**

http://git-wip-us.apache.org/repos/asf/ambari/blob/c1e02452/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
index 04d86ad..6e58267 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
@@ -31,7 +31,6 @@ import 
org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.internal.ResourceImpl;
 import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.state.PropertyInfo;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import java.net.InetAddress;
@@ -99,7 +98,6 @@ public class ClusterBlueprintRendererTest {
 
assertTrue(propertyTree.getChild(Host/HostComponent).getObject().contains(HostRoles/component_name));
   }
 
-  @Ignore
   @Test
   public void testFinalizeResult() throws Exception{
 
@@ -124,7 +122,6 @@ public class ClusterBlueprintRendererTest {
 Resource blueprintResource = blueprintNode.getObject();
 MapString, MapString, Object properties = 
blueprintResource.getPropertiesMap();
 
-assertEquals(blueprint-testCluster, 
properties.get(Blueprints).get(blueprint_name));
 assertEquals(HDP, properties.get(Blueprints).get(stack_name));
 assertEquals(1.3.3, properties.get(Blueprints).get(stack_version));
 
@@ -176,11 +173,71 @@

git commit: AMBARI-7666. Dashboard doesn't display after installing from BP.

2014-10-07 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 86798709c - d2e82f4cc


AMBARI-7666.  Dashboard doesn't display after installing from BP.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d2e82f4c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d2e82f4c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d2e82f4c

Branch: refs/heads/trunk
Commit: d2e82f4ccfe0e999bfb4cbb1f537c1e3d7c4457f
Parents: 8679870
Author: John Speidel jspei...@hortonworks.com
Authored: Mon Oct 6 20:00:56 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Tue Oct 7 13:29:30 2014 -0400

--
 .../internal/ClusterResourceProvider.java   | 21 +++---
 .../internal/ClusterResourceProviderTest.java   | 73 
 2 files changed, 57 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/d2e82f4c/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
index f40979c..090eb92 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
@@ -445,7 +445,7 @@ public class ClusterResourceProvider extends 
BaseBlueprintProcessor {
 
 registerConfigGroups(clusterName, blueprintHostGroups, stack);
 
-persistInstallStateForUI();
+persistInstallStateForUI(clusterName);
 
 RequestStatusResponse request = ((ServiceResourceProvider) 
getResourceProvider(Resource.Type.Service)).
 installAndStart(clusterName);
@@ -819,15 +819,20 @@ public class ClusterResourceProvider extends 
BaseBlueprintProcessor {
* Persist cluster state for the ambari UI.  Setting this state informs that 
UI that a cluster has been
* installed and started and that the monitoring screen for the cluster 
should be displayed to the user.
*
-   * @throws SystemException if an unexpected exception occurs
+   * @param clusterName  name of cluster
+   *
+   * @throws SystemException if unable to update the cluster with the UI 
installed flag
*/
-  private void persistInstallStateForUI() throws SystemException {
-PersistKeyValueService persistService = new PersistKeyValueService();
+  private void persistInstallStateForUI(String clusterName) throws 
SystemException {
+MapString, Object clusterProperties = new HashMapString, Object();
+clusterProperties.put(CLUSTER_PROVISIONING_STATE_PROPERTY_ID, INSTALLED);
+clusterProperties.put(CLUSTER_NAME_PROPERTY_ID, clusterName);
+
 try {
-  persistService.update({\CLUSTER_CURRENT_STATUS\: 
\{\\\clusterState\\\:\\\CLUSTER_STARTED_5\\\}\});
-} catch (Exception e) {
-  throw new SystemException(Unable to finalize state of cluster for UI.  
 +
-  Cluster creation will not be affected but the cluster may be 
inaccessible by Ambari UI. );
+  getManagementController().updateClusters(
+  Collections.singleton(getRequest(clusterProperties)), null);
+} catch (AmbariException e) {
+  throw new SystemException(Unable to finalize state of cluster for UI.);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/d2e82f4c/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
index d5f575a..c1185ae 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
@@ -34,7 +34,6 @@ import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -46,8 +45,6 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.api.services.PersistKeyValueImpl;
-import org.apache.ambari.server.api.services.PersistKeyValueService;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import

git commit: AMBARI-7666. Dashboard doesn't display after installing from BP.

2014-10-06 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/branch-1.7.0 f108b52e5 - 6675decbc


AMBARI-7666.  Dashboard doesn't display after installing from BP.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6675decb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6675decb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6675decb

Branch: refs/heads/branch-1.7.0
Commit: 6675decbc567066d4104dd5abc5ef0fcbe76425c
Parents: f108b52
Author: John Speidel jspei...@hortonworks.com
Authored: Mon Oct 6 20:00:56 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Mon Oct 6 20:54:30 2014 -0400

--
 .../internal/ClusterResourceProvider.java   | 18 +++--
 .../internal/ClusterResourceProviderTest.java   | 73 
 2 files changed, 56 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/6675decb/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
index f40979c..f9dd263 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
@@ -445,7 +445,7 @@ public class ClusterResourceProvider extends 
BaseBlueprintProcessor {
 
 registerConfigGroups(clusterName, blueprintHostGroups, stack);
 
-persistInstallStateForUI();
+persistInstallStateForUI(clusterName);
 
 RequestStatusResponse request = ((ServiceResourceProvider) 
getResourceProvider(Resource.Type.Service)).
 installAndStart(clusterName);
@@ -819,13 +819,19 @@ public class ClusterResourceProvider extends 
BaseBlueprintProcessor {
* Persist cluster state for the ambari UI.  Setting this state informs that 
UI that a cluster has been
* installed and started and that the monitoring screen for the cluster 
should be displayed to the user.
*
-   * @throws SystemException if an unexpected exception occurs
+   * @param clusterName  name of cluster
+   *
+   * @throws SystemException if unable to update the cluster with the UI 
installed flag
*/
-  private void persistInstallStateForUI() throws SystemException {
-PersistKeyValueService persistService = new PersistKeyValueService();
+  private void persistInstallStateForUI(String clusterName) throws 
SystemException {
+MapString, Object clusterProperties = new HashMapString, Object();
+clusterProperties.put(CLUSTER_PROVISIONING_STATE_PROPERTY_ID, INSTALLED);
+clusterProperties.put(CLUSTER_NAME_PROPERTY_ID, clusterName);
+
 try {
-  persistService.update({\CLUSTER_CURRENT_STATUS\: 
\{\\\clusterState\\\:\\\CLUSTER_STARTED_5\\\}\});
-} catch (Exception e) {
+  getManagementController().updateClusters(
+  Collections.singleton(getRequest(clusterProperties)), null);
+} catch (AmbariException e) {
   throw new SystemException(Unable to finalize state of cluster for UI.  
 +
   Cluster creation will not be affected but the cluster may be 
inaccessible by Ambari UI. );
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/6675decb/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
--
diff --git 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
index d5f575a..c1185ae 100644
--- 
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
+++ 
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
@@ -34,7 +34,6 @@ import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -46,8 +45,6 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.api.services.PersistKeyValueImpl;
-import org.apache.ambari.server.api.services.PersistKeyValueService;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.ClusterRequest;
 import

git commit: Removes Oozie password override from HDP 2.1 stack

2014-09-26 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk bd635ecea - 353daeb94


Removes Oozie password override from HDP 2.1 stack

When an Ambari Blueprint is exported from a running cluster
  based on the HDP 2.1 stack, and including Oozie, an Oozie
  password will be included in the exported Blueprint.

This is incorrect behavior, since no password data should
  be available in the exported Blueprint. The Blueprint
  processing code relies on the HDP stack definitions in
  order to determine which fields are passwords.

This patch addresses this problem by removing the Oozie
  password field form the HDP 2.1 stack definition.  The HDP
  2.0.6 stack (which is a parent of the 2.1 stack) already
  includes the correct metadata for this password field, such
  that the Blueprint processor will keep this field out of
  an exported Blueprint as expected.  Since there is no need
  to override this property in the 2.1 stack, the correct
  metadata can be inherited from the 2.0.6 stack.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/353daeb9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/353daeb9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/353daeb9

Branch: refs/heads/trunk
Commit: 353daeb940f47ef92ad01daa9086104539809795
Parents: bd635ec
Author: Bob Nettleton rnettle...@hortonworks.com
Authored: Mon Sep 22 16:55:26 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Fri Sep 26 11:08:16 2014 -0400

--
 .../HDP/2.1/services/OOZIE/configuration/oozie-site.xml  | 11 ---
 1 file changed, 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/353daeb9/ambari-server/src/main/resources/stacks/HDP/2.1/services/OOZIE/configuration/oozie-site.xml
--
diff --git 
a/ambari-server/src/main/resources/stacks/HDP/2.1/services/OOZIE/configuration/oozie-site.xml
 
b/ambari-server/src/main/resources/stacks/HDP/2.1/services/OOZIE/configuration/oozie-site.xml
index a7efb9c..3b6fd0b 100644
--- 
a/ambari-server/src/main/resources/stacks/HDP/2.1/services/OOZIE/configuration/oozie-site.xml
+++ 
b/ambari-server/src/main/resources/stacks/HDP/2.1/services/OOZIE/configuration/oozie-site.xml
@@ -222,17 +222,6 @@
   /property
 
   property
-nameoozie.service.JPAService.jdbc.password/name
-value /value
-description
-  DB user password.
-
-  IMPORTANT: if password is emtpy leave a 1 space string, the service 
trims the value,
-  if empty Configuration assumes it is NULL.
-/description
-  /property
-
-  property
 nameoozie.service.JPAService.pool.max.active.conn/name
 value10/value
 description

git commit: Remove Oozie external DB properties from Blueprint Export

2014-09-26 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk 2e3082af1 - 20f112acd


Remove Oozie external DB properties from Blueprint Export

This patch resolves bug AMBARI-7507.

Some Oozie properties related to using an external MySQL
  Database (oozie_existing_mysql_host, oozie.service.JPAService.jdbc.url)
  are currently appearing in an exported Blueprint with
  hostname information included.  This is incorrect, since
  the hostname information should be completely removed or
  masked in an exported Blueprint.  Having a reference to a
  given hostname in this scenario is not very useful, since the
  Blueprint may be applied to a completely different set of
  machines.

These Oozie properties generally will only be set to
  either a default, in-memory DB, or to an external
  MYSQL server instance.  This means that the normal
  PropertyUpdater processing won't really apply to
  these two properties.

This patch fixes this problem by creating a new
  PropertyUpdater type, which is only responsible
  for tracking these two property types, and marking
  them as properties that will not be included
  in an exported Blueprint.

This patch also adds a processing method to
  handle any properties that use this new
  PropertyUpdater, and this processing method
  will remove these registered properties
  during the export process.

This patch also adds a unit test to verify this change.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/20f112ac
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/20f112ac
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/20f112ac

Branch: refs/heads/trunk
Commit: 20f112acd619f58aaad85a5fa65ff82c2e4cc498
Parents: 2e3082a
Author: Bob Nettleton rnettle...@hortonworks.com
Authored: Thu Sep 25 21:37:42 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Fri Sep 26 11:42:07 2014 -0400

--
 .../BlueprintConfigurationProcessor.java| 60 
 .../BlueprintConfigurationProcessorTest.java|  9 +++
 2 files changed, 69 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/20f112ac/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 3e1bcf0..d33ae65 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -61,6 +61,15 @@ public class BlueprintConfigurationProcessor {
   new HashMapString, MapString, PropertyUpdater();
 
   /**
+   * Updaters that preserve the original property value, functions
+   *   as a placeholder for DB-related properties that need to be
+   *   removed from export, but do not require an update during
+   *   cluster creation
+   */
+  private static MapString, MapString, PropertyUpdater 
removePropertyUpdaters =
+new HashMapString, MapString, PropertyUpdater();
+
+  /**
* Collection of all updaters
*/
   private static CollectionMapString, MapString, PropertyUpdater 
allUpdaters =
@@ -181,10 +190,37 @@ public class BlueprintConfigurationProcessor {
 
 doMultiHostExportUpdate(hostGroups, multiHostTopologyUpdaters);
 
+doRemovePropertyExport(removePropertyUpdaters);
+
 return properties;
   }
 
   /**
+   * Performs export update for the set of properties that do not
+   * require update during cluster setup, but should be removed
+   * during a Blueprint export.
+   *
+   * In the case of a service referring to an external DB, any
+   * properties that contain external host information should
+   * be removed from the configuration that will be available in
+   * the exported Blueprint.
+   *
+   * @param updaters set of updaters for properties that should
+   * always be removed during a Blueprint export
+   */
+  private void doRemovePropertyExport(MapString, MapString, 
PropertyUpdater updaters) {
+for (Map.EntryString, MapString, PropertyUpdater entry : 
updaters.entrySet()) {
+  String type = entry.getKey();
+  for (String propertyName : entry.getValue().keySet()) {
+MapString, String typeProperties = properties.get(type);
+if ( (typeProperties != null)  
(typeProperties.containsKey(propertyName)) ) {
+  typeProperties.remove(propertyName);
+}
+  }
+}
+  }
+
+  /**
* Perform export update processing for HA configuration for NameNodes.  The 
HA

git commit: Fix Blueprint export handling of properties containing nameservice references

2014-09-24 Thread jspeidel

Repository: ambari
Updated Branches:
  refs/heads/trunk d8af15e8a - 6c6e4fc2e


Fix Blueprint export handling of properties containing nameservice references

This patch implements a fix for AMBARI-7458.

This patch updates the BlueprintConfigurationProcessor such
  that properties that can contain a namservice reference (there
  are only two known properties at this point) are not removed by
  the BlueprintConfigurationProcessor during the export process.

Previously, these properties were removed, since they didn't contain
  hostname information. In a NameNode HA scenario, these particular
  properties will reference nameservices, so this problem would only
  occur when NameNode HA is enabled.

This patch addresses the immediate problem, but in the longer-term
  the stack definitions should be updated to include HA-related
  metadata, so that the Blueprint processor can determine which
  properties are modified in an HA scenario.

This patch also implements a new unit test to verify this change.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6c6e4fc2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6c6e4fc2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6c6e4fc2

Branch: refs/heads/trunk
Commit: 6c6e4fc2eabd1af1c8bd89aa6aebf4068bdcdefb
Parents: d8af15e
Author: Bob Nettleton rnettle...@hortonworks.com
Authored: Tue Sep 23 14:08:38 2014 -0400
Committer: John Speidel jspei...@hortonworks.com
Committed: Wed Sep 24 13:20:09 2014 -0400

--
 .../BlueprintConfigurationProcessor.java| 28 +-
 .../BlueprintConfigurationProcessorTest.java| 54 +++-
 2 files changed, 80 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/6c6e4fc2/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index b3cc098..3e1bcf0 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -19,12 +19,14 @@
 package org.apache.ambari.server.controller.internal;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -75,6 +77,14 @@ public class BlueprintConfigurationProcessor {
   private static Pattern HOSTGROUP_PORT_REGEX = 
Pattern.compile(%HOSTGROUP::(\\w+|\\d+)%:?(\\d+)?);
 
   /**
+   * Statically-defined set of properties that can support using a nameservice 
name
+   *   in the configuration, rather than just a host name.
+   */
+  private static SetString configPropertiesWithHASupport =
+new HashSetString(Arrays.asList(fs.defaultFS, hbase.rootdir));
+
+
+  /**
* Configuration properties to be updated
*/
   private MapString, MapString, String properties;
@@ -290,7 +300,10 @@ public class BlueprintConfigurationProcessor {
   break;
 }
   }
-  if (! matchedHost) {
+  // remove properties that do not contain hostnames,
+  // except in the case of HA-related properties, that
+  // can contain nameservice references instead of hostnames (Fix for 
Bug AMBARI-7458).
+  if (! matchedHost  ! isNameServiceProperty(propertyName)) {
 typeProperties.remove(propertyName);
   }
 }
@@ -299,6 +312,19 @@ public class BlueprintConfigurationProcessor {
   }
 
   /**
+   * Determines if a given property name's value can include
+   *   nameservice references instead of host names.
+   *
+   * @param propertyName name of the property
+   *
+   * @return true if this property can support using nameservice names
+   * false if this property cannot support using nameservice names
+   */
+  private static boolean isNameServiceProperty(String propertyName) {
+return configPropertiesWithHASupport.contains(propertyName);
+  }
+
+  /**
* Update multi host topology configuration properties for blueprint export.
*
* @param hostGroups  cluster host groups

http://git-wip-us.apache.org/repos/asf/ambari/blob/6c6e4fc2/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java

1 2 >

1 - 100 of 123 matches

Mail list logo