SLIDER-769 Modify Slider app packages for seamless integration with AMS provider (Ambari Metrics Service)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4a667d39 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4a667d39 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4a667d39 Branch: refs/heads/feature/SLIDER-779_Move_to_Groovy_2.4.0 Commit: 4a667d39d1e78d91ae51eef6db21c5b54ee6ba51 Parents: 9e58112 Author: Gour Saha <gourks...@apache.org> Authored: Wed Feb 18 11:27:54 2015 -0800 Committer: Gour Saha <gourks...@apache.org> Committed: Wed Feb 18 11:30:20 2015 -0800 ---------------------------------------------------------------------- app-packages/storm/appConfig-default.json | 15 +++++----- .../storm/appConfig-secured-default.json | 15 +++++----- app-packages/storm/ganglia_metrics.json | 31 -------------------- app-packages/storm/metainfo.xml | 8 ++--- app-packages/storm/package/scripts/params.py | 13 ++++---- .../storm/package/templates/config.yaml.j2 | 27 +++++++++++++---- app-packages/storm/resources-default.json | 8 ++--- app-packages/storm/timeline_metrics.json | 31 ++++++++++++++++++++ 8 files changed, 82 insertions(+), 66 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4a667d39/app-packages/storm/appConfig-default.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/appConfig-default.json b/app-packages/storm/appConfig-default.json index 9fd7130..e04bb9c 100644 --- a/app-packages/storm/appConfig-default.json +++ b/app-packages/storm/appConfig-default.json @@ -10,21 +10,22 @@ "site.global.app_user": "${USER_NAME}", "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}", "site.global.user_group": "hadoop", - "site.global.ganglia_server_host": "${NN_HOST}", - "site.global.ganglia_server_id": "Application2", - "site.global.ganglia_enabled":"true", - "site.global.ganglia_server_port": "8668", + "site.global.metric_collector_host": "${NN_HOST}", + "site.global.metric_collector_port": "6188", + "site.global.metric_collector_lib": "", + + "site.storm-site.metrics.reporter.register" : "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter", "site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}", "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']", "site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}", "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm", "site.storm-site.transactional.zookeeper.root": "/transactional", "site.storm-site.storm.zookeeper.port": "2181", - "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", - "site.storm-site.supervisor.childopts": "-Xmx256m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", + "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/metric_collector_host},port=${@//site/global/metric_collector_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", + "site.storm-site.supervisor.childopts": "-Xmx256m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/metric_collector_host},port=${@//site/global/metric_collector_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", "site.storm-site.ui.childopts": "-Xmx768m", - "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", + "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/metric_collector_host},port=${@//site/global/metric_collector_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${storm.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper", "site.storm-site.drpc.invocations.port": "0", "site.storm-site.storm.zookeeper.root": "${DEFAULT_ZK_PATH}", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4a667d39/app-packages/storm/appConfig-secured-default.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/appConfig-secured-default.json b/app-packages/storm/appConfig-secured-default.json index 4c40ddf..2d51a9b 100644 --- a/app-packages/storm/appConfig-secured-default.json +++ b/app-packages/storm/appConfig-secured-default.json @@ -10,21 +10,22 @@ "site.global.app_user": "${USER_NAME}", "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}", "site.global.user_group": "hadoop", - "site.global.ganglia_server_host": "${NN_HOST}", - "site.global.ganglia_server_id": "Application2", - "site.global.ganglia_enabled":"true", - "site.global.ganglia_server_port": "8668", + "site.global.metric_collector_host": "${NN_HOST}", + "site.global.metric_collector_port": "6188", + "site.global.metric_collector_lib": "", + + "site.storm-site.metrics.reporter.register" : "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter", "site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}", "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']", "site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}", "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm", "site.storm-site.transactional.zookeeper.root": "/transactional", "site.storm-site.storm.zookeeper.port": "2181", - "site.storm-site.nimbus.childopts": "-Xmx1024m -Djava.security.auth.login.config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/conf/storm_jaas.conf -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", - "site.storm-site.supervisor.childopts": "-Xmx256m -Djava.security.auth.login.config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/conf/storm_jaas.conf -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", + "site.storm-site.nimbus.childopts": "-Xmx1024m -Djava.security.auth.login.config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/conf/storm_jaas.conf -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/metric_collector_host},port=${@//site/global/metric_collector_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM", + "site.storm-site.supervisor.childopts": "-Xmx256m -Djava.security.auth.login.config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/conf/storm_jaas.conf -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/metric_collector_host},port=${@//site/global/metric_collector_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM", "site.storm-site.ui.childopts": "-Xmx768m -Djava.security.auth.login.config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/conf/storm_jaas.conf", - "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", + "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/metric_collector_host},port=${@//site/global/metric_collector_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM", "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper", "site.storm-site.drpc.invocations.port": "0", "site.storm-site.storm.zookeeper.root": "${DEFAULT_ZK_PATH}", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4a667d39/app-packages/storm/ganglia_metrics.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/ganglia_metrics.json b/app-packages/storm/ganglia_metrics.json deleted file mode 100644 index 861c4fa..0000000 --- a/app-packages/storm/ganglia_metrics.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "Component": { - "NIMBUS": { - "totalslots": { - "metric": "Total Slots", - "pointInTime": false, - "temporal": true - }, - "totalexecutors": { - "metric": "Total Executors", - "pointInTime": false, - "temporal": true - }, - "topologies": { - "metric": "Topologies", - "pointInTime": false, - "temporal": true - }, - "totaltasks": { - "metric": "Total Tasks", - "pointInTime": false, - "temporal": true - }, - "usedslots": { - "metric": "Used Slots", - "pointInTime": false, - "temporal": true - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4a667d39/app-packages/storm/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml index 28b0e9b..d2625f4 100644 --- a/app-packages/storm/metainfo.xml +++ b/app-packages/storm/metainfo.xml @@ -38,11 +38,7 @@ </export> <export> <name>org.apache.slider.metrics</name> - <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value> - </export> - <export> - <name>org.apache.slider.metrics.ui</name> - <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value> + <value>http://${site.global.metric_collector_host}:${site.global.metric_collector_port}/ws/v1/timeline/metrics</value> </export> <export> <name>nimbus.host_port</name> @@ -78,7 +74,7 @@ <category>MASTER</category> <publishConfig>true</publishConfig> <autoStartOnFailure>true</autoStartOnFailure> - <appExports>QuickLinks-nimbus.host_port,QuickLinks-org.apache.slider.metrics.ui,QuickLinks-org.apache.slider.metrics</appExports> + <appExports>QuickLinks-nimbus.host_port,QuickLinks-org.apache.slider.metrics</appExports> <maxInstanceCount>1</maxInstanceCount> <commandScript> <script>scripts/nimbus.py</script> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4a667d39/app-packages/storm/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py index 1ccba5e..f6cccfb 100644 --- a/app-packages/storm/package/scripts/params.py +++ b/app-packages/storm/package/scripts/params.py @@ -25,6 +25,7 @@ import status_params config = Script.get_config() app_root = config['configurations']['global']['app_root'] +app_name = config['clusterName'] conf_dir = format("{app_root}/conf") storm_user = config['configurations']['global']['app_user'] log_dir = config['configurations']['global']['app_log_dir'] @@ -39,11 +40,13 @@ rest_lib_dir = format("{app_root}/external/storm-rest") storm_bin = format("{app_root}/bin/storm.py") storm_env_sh_template = config['configurations']['storm-env']['content'] -ganglia_installed = config['configurations']['global']['ganglia_enabled'] -if ganglia_installed: - ganglia_report_interval = 60 - ganglia_server = config['configurations']['global']['ganglia_server_host'] - ganglia_port = config['configurations']['global']['ganglia_server_port'] +metric_collector_host = default('/configurations/global/metric_collector_host', '') +metric_collector_port = default('/configurations/global/metric_collector_port', '') +metric_collector_lib = default('/configurations/global/metric_collector_lib', '') + +has_metric_collector = 1 +if not metric_collector_lib: + has_metric_collector = 0 security_enabled = config['configurations']['global']['security_enabled'] http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4a667d39/app-packages/storm/package/templates/config.yaml.j2 ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/templates/config.yaml.j2 b/app-packages/storm/package/templates/config.yaml.j2 index aa4ec46..d9c47a9 100644 --- a/app-packages/storm/package/templates/config.yaml.j2 +++ b/app-packages/storm/package/templates/config.yaml.j2 @@ -16,18 +16,33 @@ nimbusHost: {{nimbus_host}} nimbusPort: {{nimbus_port}} -{% if ganglia_installed %} -enableGanglia: {{ganglia_installed}} +{% if has_metric_collector %} +enableGanglia: False + +ganglia: + reportInterval: 60 + +enableMetricsSink: True + +metrics_collector: + + reportInterval: 60 + + host: "{{metric_collector_host}}" + port: {{metric_collector_port}} + +{% else %} +enableGanglia: True # ganglia configuration (necessary if ganglia reporting is enabled) ganglia: # how often to report to ganglia metrics (in seconds) - reportInterval: {{ganglia_report_interval}} + reportInterval: 60 # the hostname of the gmond server where storm cluster metrics will be sent - host: {{ganglia_server}} - port: {{ganglia_port}} + host: {{metric_collector_host}} + port: {{metric_collector_port}} # address mode # default is MULTICAST @@ -36,4 +51,4 @@ ganglia: # an <IP>:<HOSTNAME> pair to spoof # this allows us to simulate storm cluster metrics coming from a specific host #spoof: "192.168.1.1:storm" -{% endif %} \ No newline at end of file +{% endif %} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4a667d39/app-packages/storm/resources-default.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/resources-default.json b/app-packages/storm/resources-default.json index 65e6579..4b3eb64 100644 --- a/app-packages/storm/resources-default.json +++ b/app-packages/storm/resources-default.json @@ -13,22 +13,22 @@ "NIMBUS": { "yarn.role.priority": "1", "yarn.component.instances": "1", - "yarn.memory": "2048" + "yarn.memory": "256" }, "STORM_UI_SERVER": { "yarn.role.priority": "2", "yarn.component.instances": "1", - "yarn.memory": "1278" + "yarn.memory": "256" }, "DRPC_SERVER": { "yarn.role.priority": "3", "yarn.component.instances": "1", - "yarn.memory": "1278" + "yarn.memory": "256" }, "SUPERVISOR": { "yarn.role.priority": "4", "yarn.component.instances": "1", - "yarn.memory": "3072" + "yarn.memory": "256" } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4a667d39/app-packages/storm/timeline_metrics.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/timeline_metrics.json b/app-packages/storm/timeline_metrics.json new file mode 100644 index 0000000..861c4fa --- /dev/null +++ b/app-packages/storm/timeline_metrics.json @@ -0,0 +1,31 @@ +{ + "Component": { + "NIMBUS": { + "totalslots": { + "metric": "Total Slots", + "pointInTime": false, + "temporal": true + }, + "totalexecutors": { + "metric": "Total Executors", + "pointInTime": false, + "temporal": true + }, + "topologies": { + "metric": "Topologies", + "pointInTime": false, + "temporal": true + }, + "totaltasks": { + "metric": "Total Tasks", + "pointInTime": false, + "temporal": true + }, + "usedslots": { + "metric": "Used Slots", + "pointInTime": false, + "temporal": true + } + } + } +}
