Elukey has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/342592 )
Change subject: Add the Jmxtrans configuration for the MapReduce History server ...................................................................... Add the Jmxtrans configuration for the MapReduce History server Bug: T156272 Change-Id: I815214d1dfd9d6970c99b7485bd40e1a254f67c4 --- A manifests/hadoop/jmxtrans/historyserver.pp M manifests/hadoop/jmxtrans/master.pp 2 files changed, 125 insertions(+), 1 deletion(-) Approvals: Elukey: Looks good to me, approved jenkins-bot: Verified diff --git a/manifests/hadoop/jmxtrans/historyserver.pp b/manifests/hadoop/jmxtrans/historyserver.pp new file mode 100644 index 0000000..5782fb0 --- /dev/null +++ b/manifests/hadoop/jmxtrans/historyserver.pp @@ -0,0 +1,122 @@ +# == Class cdh::hadoop::jmxtrans::historyserver +# Sets up a jmxtrans instance for a Hadoop MapReduce History Server +# running on the current host. +# Note: This requires the jmxtrans puppet module found at +# https://github.com/wikimedia/puppet-jmxtrans. +# +# == Parameters +# $jmx_port - Map Reduce Job History JMX port. Default: 9986 +# $ganglia - Ganglia host:port +# $graphite - Graphite host:port +# $statsd - statsd host:port +# $outfile - outfile to which Kafka stats will be written. +# $objects - objects parameter to pass to jmxtrans::metrics. Only use +# this if you need to override the default ones that this +# class provides. +# +# == Usage +# class { 'cdh::hadoop::jmxtrans::historyserver': +# ganglia => 'ganglia.example.org:8649' +# } +# +class cdh::hadoop::jmxtrans::historyserver( + $jmx_port = $cdh::hadoop::defaults::mapreduce_history_jmxremote_port, + $ganglia = undef, + $graphite = undef, + $statsd = undef, + $outfile = undef, + $objects = undef, +) inherits cdh::hadoop::defaults +{ + $jmx = "${::fqdn}:${jmx_port}" + $group_name = 'Hadoop.JobHistoryServer' + + # query for metrics from Hadoop MapReduce JobHistory's JVM + jmxtrans::metrics::jvm { 'hadoop-mapreduce-historyserver': + jmx => $jmx, + group_prefix => "${group_name}.", + outfile => $outfile, + ganglia => $ganglia, + graphite => $graphite, + statsd => $statsd, + } + + + $historyserver_objects = $objects ? { + # if $objects was not set, then use this as the + # default set of JMX MBean objects to query. + undef => [ + { + 'name' => 'Hadoop:name=JvmMetrics,service=JobHistoryServer', + 'resultAlias' => "${group_name}.JvmMetrics", + 'attrs' => { + 'GcCount' => { 'slope' => 'positive' }, + 'GcCountPS MarkSweep' => { 'slope' => 'positive' }, + 'GcCountPS Scavenge' => { 'slope' => 'positive' }, + 'GcTimeMillis' => { 'slope' => 'both' }, + 'GcTimeMillisPS MarkSweep' => { 'slope' => 'both' }, + 'GcTimeMillisPS Scavenge' => { 'slope' => 'both' }, + 'LogError' => { 'slope' => 'positive' }, + 'LogFatal' => { 'slope' => 'positive' }, + 'LogInfo' => { 'slope' => 'both' }, + 'LogWarn' => { 'slope' => 'positive' }, + 'MemHeapCommittedM' => { 'slope' => 'both' }, + 'MemHeapUsedM' => { 'slope' => 'both' }, + 'MemNonHeapCommittedM' => { 'slope' => 'both' }, + 'MemNonHeapUsedM' => { 'slope' => 'both' }, + 'ThreadsBlocked' => { 'slope' => 'both' }, + 'ThreadsNew' => { 'slope' => 'both' }, + 'ThreadsRunnable' => { 'slope' => 'both' }, + 'ThreadsTerminated' => { 'slope' => 'both' }, + 'ThreadsTimedWaiting' => { 'slope' => 'both' }, + 'ThreadsWaiting' => { 'slope' => 'both' }, + }, + + }, + { + 'name' => 'Hadoop:name=RpcDetailedActivityForPort10020,service=JobHistoryServer', + 'resultAlias' => "${group_name}.RpcDetailedActivityForPort10020", + 'attrs' => { + 'GetCountersAvgTime' => { 'slope' => 'both' }, + 'GetCountersNumOps' => { 'slope' => 'both' }, + 'GetJobReportAvgTime' => { 'slope' => 'both' }, + 'GetJobReportNumOps' => { 'slope' => 'both' }, + 'GetTaskAttemptCompletionEventsAvgTime' => { 'slope' => 'both' }, + 'GetTaskAttemptCompletionEventsNumOps' => { 'slope' => 'both' }, + 'GetTaskReportsAvgTime' => { 'slope' => 'both' }, + 'GetTaskReportsNumOps' => { 'slope' => 'both' }, + }, + }, + { + 'name' => 'Hadoop:name=UgiMetrics,service=JobHistoryServer', + 'resultAlias' => "${group_name}.UgiMetrics", + 'attrs' => { + 'GetGroupsAvgTime' => { 'slope' => 'both' }, + 'GetGroupsNumOps' => { 'slope' => 'both' }, + 'LoginFailureAvgTime' => { 'slope' => 'both' }, + 'LoginFailureNumOps' => { 'slope' => 'positive' }, + 'LoginSuccessAvgTime' => { 'slope' => 'both' }, + 'LoginSuccessNumOps' => { 'slope' => 'positive' }, + 'RenewalFailures' => { 'slope' => 'both' }, + + }, + }, + ], + + # else use $objects + default => $objects, + } + + # query for jmx metrics + jmxtrans::metrics { "hadoop-mapreduce-historyserver-${::hostname}-${jmx_port}": + jmx => $jmx, + outfile => $outfile, + ganglia => $ganglia, + ganglia_group_name => $group_name, + graphite => $graphite, + graphite_root_prefix => $group_name, + statsd => $statsd, + statsd_root_prefix => $group_name, + objects => $historyserver_objects, + } +} \ No newline at end of file diff --git a/manifests/hadoop/jmxtrans/master.pp b/manifests/hadoop/jmxtrans/master.pp index 7b70ddb..a564eb5 100644 --- a/manifests/hadoop/jmxtrans/master.pp +++ b/manifests/hadoop/jmxtrans/master.pp @@ -7,7 +7,9 @@ $outfile = undef, ) { - class { ['cdh::hadoop::jmxtrans::namenode', 'cdh::hadoop::jmxtrans::resourcemanager']: + class { ['cdh::hadoop::jmxtrans::namenode', + 'cdh::hadoop::jmxtrans::resourcemanager', + 'cdh::hadoop::jmxtrans::historyserver']: ganglia => $ganglia, graphite => $graphite, outfile => $outfile, -- To view, visit https://gerrit.wikimedia.org/r/342592 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I815214d1dfd9d6970c99b7485bd40e1a254f67c4 Gerrit-PatchSet: 5 Gerrit-Project: operations/puppet/cdh Gerrit-Branch: master Gerrit-Owner: Elukey <ltosc...@wikimedia.org> Gerrit-Reviewer: Elukey <ltosc...@wikimedia.org> Gerrit-Reviewer: Ottomata <ao...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
