[2/2] ambari git commit: AMBARI-15793. Blueprint: Presence/absence of HiveServer2 Interactive component in the template should be validated against related configurations

2016-04-10 Thread smohanty 
AMBARI-15793. Blueprint: Presence/absence of HiveServer2 Interactive component 
in the template should be validated against related configurations


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/88984efd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/88984efd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/88984efd

Branch: refs/heads/trunk
Commit: 88984efde16cd3bfd32616db454abab56f818c7d
Parents: 55e5a78
Author: Sumit Mohanty 
Authored: Sun Apr 10 17:45:20 2016 -0700
Committer: Sumit Mohanty 
Committed: Sun Apr 10 17:45:20 2016 -0700

--
 .../stacks/HDP/2.5/services/stack_advisor.py|  105 +-
 .../python/stacks/2.5/common/host-3-hosts.json  |  269 +
 .../2.5/common/services-normal-his-2-hosts.json | 1131 ++
 .../2.5/common/services-normal-his-valid.json   | 1131 ++
 .../stacks/2.5/common/test_stack_advisor.py |  168 +++
 5 files changed, 2802 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/88984efd/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
--
diff --git 
a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py 
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 9050092..a3009ee 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -19,15 +19,110 @@ limitations under the License.
 
 
 class HDP25StackAdvisor(HDP24StackAdvisor):
+  def createComponentLayoutRecommendations(self, services, hosts):
+parentComponentLayoutRecommendations = super(HDP25StackAdvisor, 
self).createComponentLayoutRecommendations(
+  services, hosts)
+return parentComponentLayoutRecommendations
+
+  def getComponentLayoutValidations(self, services, hosts):
+parentItems = super(HDP25StackAdvisor, 
self).getComponentLayoutValidations(services, hosts)
+servicesList = [service["StackServices"]["service_name"] for service in 
services["services"]]
+componentsListList = [service["components"] for service in 
services["services"]]
+componentsList = [item["StackServiceComponents"] for sublist in 
componentsListList for item in sublist]
+childItems = []
+
+if "HIVE" in servicesList:
+  hsi_hosts = self.__getHosts(componentsList, "HIVE_SERVER_INTERACTIVE")
+  if len(hsi_hosts) > 1:
+message = "Only one host can install HIVE_SERVER_INTERACTIVE. "
+childItems.append(
+  {"type": 'host-component', "level": 'ERROR', "message": message, 
"component-name": 'HIVE_SERVER_INTERACTIVE'})
+print '3329'
+
+parentItems.extend(childItems)
+return parentItems
+
+  def getServiceConfigurationValidators(self):
+parentValidators = super(HDP25StackAdvisor, 
self).getServiceConfigurationValidators()
+childValidators = {
+  "HIVE": {"hive-interactive-env": 
self.validateHiveInteractiveEnvConfigurations}
+}
+self.mergeValidators(parentValidators, childValidators)
+return parentValidators
+
+  def validateHiveInteractiveEnvConfigurations(self, properties, 
recommendedDefaults, configurations, services, hosts):
+hive_site_env_properties = getSiteProperties(configurations, 
"hive-interactive-env")
+servicesList = [service["StackServices"]["service_name"] for service in 
services["services"]]
+componentsListList = [service["components"] for service in 
services["services"]]
+componentsList = [item["StackServiceComponents"] for sublist in 
componentsListList for item in sublist]
+validationItems = []
+
+if "HIVE" in servicesList:
+  hsi_hosts = self.__getHosts(componentsList, "HIVE_SERVER_INTERACTIVE")
+  if len(hsi_hosts) > 0:
+# HIVE_SERVER_INTERACTIVE is mapped to a host
+if 'enable_hive_interactive' not in hive_site_env_properties or (
+'enable_hive_interactive' in hive_site_env_properties and 
hive_site_env_properties[
+  'enable_hive_interactive'].lower() != 'true'):
+  validationItems.append({"config-name": "enable_hive_interactive",
+  "item": self.getWarnItem(
+"HIVE_SERVER_INTERACTIVE requires 
enable_hive_interactive in hive-interactive-env set to true.")})
+if 'hive_server_interactive_host' in hive_site_env_properties:
+  hsi_host = hsi_hosts[0]
+  if hive_site_env_properties['hive_server_interactive_host'].lower() 
!= hsi_host.lower():
+validationItems.append({"config-name": 
"hive_server_interactive_host",
+

[1/2] ambari git commit: AMBARI-15793. Blueprint: Presence/absence of HiveServer2 Interactive component in the template should be validated against related configurations

2016-04-10 Thread smohanty 
Repository: ambari
Updated Branches:
  refs/heads/trunk 55e5a785f -> 88984efde


http://git-wip-us.apache.org/repos/asf/ambari/blob/88984efd/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
--
diff --git 
a/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
 
b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
new file mode 100644
index 000..69c7e55
--- /dev/null
+++ 
b/ambari-server/src/test/python/stacks/2.5/common/services-normal-his-valid.json
@@ -0,0 +1,1131 @@
+{
+  "href" : 
"/api/v1/stacks/HDP/versions/2.5/?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version,services/StackServices/service_name,services/StackServices/service_version,services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy,services/configurations/StackConfigurations/property_depends_on,services/configurations/dependencies/StackConfigurationDependency/dependency_name,services/configurations/dependencies/StackConfigurationDependency/dependency_type,services/configurations/StackConfigurations/type/StackServices/service_name.in(HDFS,YARN,MAPREDUCE2,TEZ,HIVE,PIG,ZOOKEEPER,SLIDER)",
+  "Versions" : {
+"parent_stack_version" : "2.4",
+"stack_name" : "HDP",
+"stack_version" : "2.5",
+"stack_hierarchy" : {
+  "stack_name" : "HDP",
+  "stack_versions" : [ "2.4", "2.3", "2.2", "2.1", "2.0.6" ]
+}
+  },
+  "services" : [ {
+"href" : "/api/v1/stacks/HDP/versions/2.5/services/HDFS",
+"StackServices" : {
+  "service_name" : "HDFS",
+  "service_version" : "2.7.1.2.5",
+  "stack_name" : "HDP",
+  "stack_version" : "2.5"
+},
+"components" : [ {
+  "href" : 
"/api/v1/stacks/HDP/versions/2.5/services/HDFS/components/DATANODE",
+  "StackServiceComponents" : {
+"advertise_version" : true,
+"bulk_commands_display_name" : "DataNodes",
+"bulk_commands_master_component_name" : "NAMENODE",
+"cardinality" : "1+",
+"component_category" : "SLAVE",
+"component_name" : "DATANODE",
+"custom_commands" : [ ],
+"decommission_allowed" : true,
+"display_name" : "DataNode",
+"has_bulk_commands_definition" : true,
+"is_client" : false,
+"is_master" : false,
+"reassign_allowed" : false,
+"recovery_enabled" : false,
+"service_name" : "HDFS",
+"stack_name" : "HDP",
+"stack_version" : "2.5",
+"hostnames" : [ "c6402.ambari.apache.org" ]
+  },
+  "dependencies" : [ ]
+}, {
+  "href" : 
"/api/v1/stacks/HDP/versions/2.5/services/HDFS/components/HDFS_CLIENT",
+  "StackServiceComponents" : {
+"advertise_version" : true,
+"bulk_commands_display_name" : "",
+"bulk_commands_master_component_name" : "",
+"cardinality" : "1+",
+"component_category" : "CLIENT",
+"component_name" : "HDFS_CLIENT",
+"custom_commands" : [ ],
+"decommission_allowed" : false,
+"display_name" : "HDFS Client",
+"has_bulk_commands_definition" : false,
+"is_client" : true,
+"is_master" : false,
+"reassign_allowed" : false,
+"recovery_enabled" : false,
+"service_name" : "HDFS",
+"stack_name" : "HDP",
+"stack_version" : "2.5",
+"hostnames" : [ "c6402.ambari.apache.org" ]
+  },
+  "dependencies" : [ ]
+}, {
+  "href" : 
"/api/v1/stacks/HDP/versions/2.5/services/HDFS/components/JOURNALNODE",
+  "StackServiceComponents" : {
+"advertise_version" : true,
+"bulk_commands_display_name" : "",
+"bulk_commands_master_component_name" : "",
+"cardinality" : "0+",
+"component_category" : "SLAVE",
+"component_name" : "JOURNALNODE",
+"custom_commands" : [ ],
+"decommission_allowed" : false,
+"display_name" : "JournalNode",
+"has_bulk_commands_definition" : false,
+"is_client" : false,
+"is_master" : false,
+"reassign_allowed" : false,
+"recovery_enabled" : false,
+"service_name" : "HDFS",
+"stack_name" : "HDP",
+"stack_version" : "2.5",
+"hostnames" : [ "c6402.ambari.apache.org" ]
+  },
+  "dependencies" : [ {
+"href" : 
"/api/v1/stacks/HDP/versions/2.5/services/HDFS/components/JOURNALNODE/dependencies/HDFS_CLIENT",
+"Dependencies" : {
+  "component_name" : "HDFS_CLIENT",
+  "dependent_component_name" : "JOURNALNODE",
+  "dependent_service_name" : "HDFS",
+  "stack_name" : "HDP",
+  "stack_version" : "2.5"
+}
+  } ]
+}, {
+  "href" : 
"/api/v1/stacks/HDP/versions/2.5/services/HDFS/components/NAMENODE",
+  "StackServiceComponents" : {
+"advertise_version" : true,
+

ambari git commit: AMBARI-14920 - Support Atlas / Storm integration

2016-04-10 Thread tbeerbower 
Repository: ambari
Updated Branches:
  refs/heads/trunk 7b64e964d -> 55e5a785f


AMBARI-14920 - Support Atlas / Storm integration


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/55e5a785
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/55e5a785
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/55e5a785

Branch: refs/heads/trunk
Commit: 55e5a785fea5cff8dceb5dc1d43371b68726b848
Parents: 7b64e96
Author: tbeerbower 
Authored: Sun Apr 10 17:36:22 2016 -0400
Committer: tbeerbower 
Committed: Sun Apr 10 17:36:38 2016 -0400

--
 .../0.9.1.2.1/configuration/storm-site.xml  | 20 +
 .../0.9.1.2.1/package/scripts/params_linux.py   | 14 +
 .../STORM/0.9.1.2.1/package/scripts/storm.py| 15 +-
 .../services/STORM/configuration/storm-env.xml  | 31 
 4 files changed, 74 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/55e5a785/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
--
diff --git 
a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
 
b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
index aada363..cf702c0 100644
--- 
a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
+++ 
b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
@@ -727,4 +727,24 @@
   true
 
   
+  
+atlas.cluster.name
+{{cluster_name}}
+
+  
+application-properties
+atlas.cluster.name
+  
+
+  
+  
+storm.topology.submission.notifier.plugin.class
+org.apache.atlas.storm.hook.StormAtlasHook
+
+  
+application-properties
+atlas.cluster.name
+  
+
+  
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/55e5a785/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
--
diff --git 
a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
 
b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
index e9e8ce5..101ca46 100644
--- 
a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
+++ 
b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
@@ -17,6 +17,7 @@ See the License for the specific language governing 
permissions and
 limitations under the License.
 
 """
+import os
 import re
 import ambari_simplejson as json # simplejson is much faster comparing to 
Python 2.6 json module and has the same functions set.
 
@@ -43,6 +44,8 @@ tmp_dir = Script.get_tmp_dir()
 stack_root = status_params.stack_root
 sudo = AMBARI_SUDO_BINARY
 
+cluster_name = config['clusterName']
+
 stack_name = default("/hostLevelParams/stack_name", None)
 upgrade_direction = default("/commandParams/upgrade_direction", 
Direction.UPGRADE)
 version = default("/commandParams/version", None)
@@ -182,6 +185,17 @@ metrics_report_interval = 
default("/configurations/ams-site/timeline.metrics.sin
 metrics_collection_period = 
default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10)
 metric_collector_sink_jar = "/usr/lib/storm/lib/ambari-metrics-storm-sink*.jar"
 
+jar_jvm_opts = ''
+
+# Atlas related params
+atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', [])
+has_atlas = len(atlas_hosts) > 0
+
+if has_atlas:
+  atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in 
os.environ else stack_root + '/current/atlas-server'
+  atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in 
os.environ else '/etc/atlas/conf'
+  jar_jvm_opts = '-Datlas.conf=' + atlas_conf_dir
+
 # ranger host
 ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
 has_ranger_admin = not len(ranger_admin_hosts) == 0

http://git-wip-us.apache.org/repos/asf/ambari/blob/55e5a785/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
--
diff --git 
a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
 
b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
index d2fafa6..74050d8 100644
--- 
a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
+++