Repository: ambari Updated Branches: refs/heads/trunk 7b64e964d -> 55e5a785f
AMBARI-14920 - Support Atlas / Storm integration Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/55e5a785 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/55e5a785 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/55e5a785 Branch: refs/heads/trunk Commit: 55e5a785fea5cff8dceb5dc1d43371b68726b848 Parents: 7b64e96 Author: tbeerbower <tbeerbo...@hortonworks.com> Authored: Sun Apr 10 17:36:22 2016 -0400 Committer: tbeerbower <tbeerbo...@hortonworks.com> Committed: Sun Apr 10 17:36:38 2016 -0400 ---------------------------------------------------------------------- .../0.9.1.2.1/configuration/storm-site.xml | 20 +++++++++++++ .../0.9.1.2.1/package/scripts/params_linux.py | 14 +++++++++ .../STORM/0.9.1.2.1/package/scripts/storm.py | 15 +++++++++- .../services/STORM/configuration/storm-env.xml | 31 ++++++++++++++++---- 4 files changed, 74 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/55e5a785/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml index aada363..cf702c0 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml @@ -727,4 +727,24 @@ <empty-value-valid>true</empty-value-valid> </value-attributes> </property> + <property> + <name>atlas.cluster.name</name> + <value>{{cluster_name}}</value> + <depends-on> + <property> + <type>application-properties</type> + <name>atlas.cluster.name</name> + </property> + </depends-on> + </property> + <property> + <name>storm.topology.submission.notifier.plugin.class</name> + <value>org.apache.atlas.storm.hook.StormAtlasHook</value> + <depends-on> + <property> + <type>application-properties</type> + <name>atlas.cluster.name</name> + </property> + </depends-on> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/55e5a785/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py index e9e8ce5..101ca46 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py @@ -17,6 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. """ +import os import re import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. @@ -43,6 +44,8 @@ tmp_dir = Script.get_tmp_dir() stack_root = status_params.stack_root sudo = AMBARI_SUDO_BINARY +cluster_name = config['clusterName'] + stack_name = default("/hostLevelParams/stack_name", None) upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE) version = default("/commandParams/version", None) @@ -182,6 +185,17 @@ metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sin metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10) metric_collector_sink_jar = "/usr/lib/storm/lib/ambari-metrics-storm-sink*.jar" +jar_jvm_opts = '' + +# Atlas related params +atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) +has_atlas = len(atlas_hosts) > 0 + +if has_atlas: + atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else stack_root + '/current/atlas-server' + atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' + jar_jvm_opts = '-Datlas.conf=' + atlas_conf_dir + # ranger host ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) has_ranger_admin = not len(ranger_admin_hosts) == 0 http://git-wip-us.apache.org/repos/asf/ambari/blob/55e5a785/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py index d2fafa6..74050d8 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py @@ -52,6 +52,7 @@ def storm(name=None): @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def storm(name=None): import params + import os Directory(params.log_dir, owner=params.storm_user, @@ -93,6 +94,18 @@ def storm(name=None): content=InlineTemplate(params.storm_env_sh_template) ) + if params.has_atlas: + atlas_storm_hook_dir = os.path.join(params.atlas_home_dir, "hook", "storm") + if os.path.exists(atlas_storm_hook_dir): + storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib") + if os.path.exists(storm_extlib_dir): + src_files = os.listdir(atlas_storm_hook_dir) + for file_name in src_files: + atlas_storm_hook_file_name = os.path.join(atlas_storm_hook_dir, file_name) + storm_lib_file_name = os.path.join(storm_extlib_dir, file_name) + if (os.path.isfile(atlas_storm_hook_file_name)): + Link(storm_lib_file_name, to = atlas_storm_hook_file_name) + if params.has_metric_collector: File(format("{conf_dir}/storm-metrics2.properties"), owner=params.storm_user, @@ -127,7 +140,7 @@ def storm(name=None): owner=params.storm_user, content=InlineTemplate(params.storm_worker_log4j_content) ) - + if params.security_enabled: TemplateConfig(format("{conf_dir}/storm_jaas.conf"), owner=params.storm_user http://git-wip-us.apache.org/repos/asf/ambari/blob/55e5a785/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-env.xml index 2bb0862..8d2da69 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-env.xml @@ -26,9 +26,30 @@ <value>true</value> <description></description> </property> - <property> - <name>storm_logs_supported</name> - <value>true</value> - <description></description> - </property> + <property> + <name>storm_logs_supported</name> + <value>true</value> + <description></description> + </property> + <!-- storm-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for storm-env.sh file</description> + <value> + #!/bin/bash + + # Set Storm specific environment variables here. + + # The java implementation to use. + export JAVA_HOME={{java64_home}} + + export STORM_CONF_DIR={{conf_dir}} + export STORM_HOME={{storm_component_home_dir}} + + export STORM_JAR_JVM_OPTS={{jar_jvm_opts}} + </value> + <value-attributes> + <type>content</type> + </value-attributes> + </property> </configuration>