Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/98601
Change subject: Adding varnishkafka::monitoring class to send stats to Ganglia.
......................................................................
Adding varnishkafka::monitoring class to send stats to Ganglia.
This uses logster and a custom VarnishkafkaLogster parser
to read log.statistics.file JSON stats and send to Ganglia.
Change-Id: I83083f078ae947770ae7fbb79a73fb02832d68be
---
M README.md
A files/VarnishkafkaLogster.py
M manifests/defaults.pp
M manifests/init.pp
A manifests/monitoring.pp
M templates/varnishkafka.conf.erb
6 files changed, 92 insertions(+), 4 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet/varnishkafka
refs/changes/01/98601/1
diff --git a/README.md b/README.md
index ea852f2..7a965be 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,8 @@
# Usage:
+## Run varnishkafka
+
```puppet
# Default varnishkafka instance with a two broker Kafka Cluster
class { 'varnishkafka':
@@ -13,7 +15,7 @@
```puppet
# varnishkafka instance using custom JSON output format
-class {
+class { 'varnishkafka':
brokers => ['kafka1.example.org:9092', 'kafka2.example.org:9092'],
format_type => 'json',
format => '%{@hostname}l %{@sequence!num}n %{%FT%T@dt}t
%{Varnish:time_firstbyte@time_firstbyte!num}x %{@ip}h
%{Varnish:handling@cache_status}x %{@http_status}s %{@response_size!num}b
%{@http_method}m %{Host@uri_host}i %{@uri_path}U %{@uri_query}q
%{Content-Type@content_type}o %{Referer@referer}i
%{X-Forwarded-For@x_forwarded_for}i %{User-Agent@user_agent}i
%{Accept-Language@accept_language}i'}
@@ -21,3 +23,12 @@
```
See the ```varnishkafka``` class docs in manifests/init.pp for more parameter
documentation.
+
+## Ganglia Monitoring
+
+```puppet
+# This will install logster and a custom VarnishkafkaLogster
+# logster parser to send JSON stats from the log.statistics.file
+# to Ganglia.
+class { 'varnishkafka::monitoring': }
+```
diff --git a/files/VarnishkafkaLogster.py b/files/VarnishkafkaLogster.py
new file mode 100644
index 0000000..6830e55
--- /dev/null
+++ b/files/VarnishkafkaLogster.py
@@ -0,0 +1,29 @@
+### VarnishkafkaLogster is a subclass of JsonLogster.
+### It is meant to parse varnishkafka
+### (https://github.com/wikimedia/operations-software-varnish-varnishkafka)
+### JSON statistics.
+###
+### Example:
+### sudo ./logster --dry-run --output=ganglia VarnishkafkaLogster
/var/cache/varnishkafka.stats.json
+###
+
+from logster.parsers.JsonLogster import JsonLogster
+
+class VarnishkafkaLogster(JsonLogster):
+ def key_filter(self, key):
+
+ if key == 'varnishkafka':
+ key = 'kafka%svarnishkafka' % self.key_separator
+ elif key == 'kafka':
+ key = 'kafka%srdkafka' % self.key_separator
+ elif 'bootstrap' in key:
+ return False
+ elif self.key_separator in key:
+ # this won't do anything if key_separator is '-'
+ key = key.replace(self.key_separator, '-')
+ elif key.startswith('-'):
+ return False
+ elif key in ['name', 'topic', 'toppars']:
+ return False
+
+ return key
diff --git a/manifests/defaults.pp b/manifests/defaults.pp
index ccc1700..449662f 100644
--- a/manifests/defaults.pp
+++ b/manifests/defaults.pp
@@ -33,6 +33,8 @@
$log_level = 6
$log_stderr = false
$log_syslog = true
+ $log_statistics_file =
'/var/cache/varnishkafka/varnishkafka.stats.json'
+ $log_statistics_interval = 60
$conf_template = 'varnishkafka/varnishkafka.conf.erb'
$default_template = 'varnishkafka/varnishkafka.default.erb'
diff --git a/manifests/init.pp b/manifests/init.pp
index 785c147..21a2087 100644
--- a/manifests/init.pp
+++ b/manifests/init.pp
@@ -53,6 +53,9 @@
# $log_level - varnishkafka log level. Default 6
(info).
# $log_stderr - Boolean. Whether to log to stderr.
Default: true
# $log_syslog - Boolean. Whether to log to syslog.
Default: true
+# $log_statistics_file - Path to varnishkafka JSON statistics
file.
+# Default:
/var/cache/varnishkafka/varnishkafka.stats.json
+# $log_statistics_interval - JSON statistics file output interval in
seconds. Default: 60
#
# $conf_template
# $default_template
@@ -87,6 +90,8 @@
$log_level = $varnishkafka::defaults::log_level,
$log_stderr = $varnishkafka::defaults::log_stderr,
$log_syslog = $varnishkafka::defaults::log_syslog,
+ $log_statistics_file =
$varnishkafka::defaults::log_statistics_file,
+ $log_statistics_interval =
$varnishkafka::defaults::log_statistics_interval,
$conf_template = $varnishkafka::defaults::conf_template,
$default_template = $varnishkafka::defaults::default_template
diff --git a/manifests/monitoring.pp b/manifests/monitoring.pp
new file mode 100644
index 0000000..f55725d
--- /dev/null
+++ b/manifests/monitoring.pp
@@ -0,0 +1,41 @@
+# == Class varnishkafka::monitoring
+# Uses logster (https://github.com/wikimedia/operations-debs-logster)
+# to tail varnishkafka.stats.json file send stats to Ganglia.
+#
+# TODO: Support more than logster ganglia output via
+# class parameters.
+class varnishkafka::monitoring(
+ $ensure = 'present'
+)
+{
+ Class['varnishkafka'] -> Class['varnishkafka::monitoring']
+
+ # varnishkafka monitoring is done via the logster package.
+ package { 'logster':
+ ensure => 'installed',
+ # don't bother doing this unless ganglia is installed
+ require => Package['ganglia']
+ }
+ # put the VarnishkafkaLogster.py module in place
+ if !defined(File['/usr/local/share/logster']) {
+ file { '/usr/local/share/logster':
+ ensure => 'directory',
+ }
+ }
+
+ # Custom JsonLogster parser subclass to filter and transform
+ # a few varnishkafka stats JSON keys.
+ file { '/usr/local/share/logster/VarnishkafkaLogster.py':
+ source => 'puppet:///modules/varnishkafka/VarnishkafkaLogster.py',
+ require => File['/usr/local/share/logster'],
+ }
+
+ # Run logster using the VarnishkafkaLogster parser and send updated stats
to Ganglia.
+ $cron_command = "/usr/bin/logster --output ganglia
--gmetric-options='--group=kafka' VarnishkafkaLogster.VarnishkafkaLogster
${varnishkafka::log_statistics_file}"
+ cron { 'varnishkafka-stats-to-ganglia':
+ ensure => $ensure,
+ command => $cron_command,
+ minute => '*/1',
+ require => [Package['logster'],
File['/usr/local/share/logster/VarnishkafkaLogster.py']]
+ }
+}
diff --git a/templates/varnishkafka.conf.erb b/templates/varnishkafka.conf.erb
index 2cb9f66..6141945 100644
--- a/templates/varnishkafka.conf.erb
+++ b/templates/varnishkafka.conf.erb
@@ -192,11 +192,11 @@
# Statistics output interval
# Defaults to 60 seconds, use 0 to disable.
-#log.statistics.interval = 60
+log.statistics.interval = <%= @log_statistics_interval %>
# Statistics output file
-# Defaults to /var/cache/varnishkafka.stats.json
-#log.statistics.file = /var/cache/varnishkafka.stats.json
+# Defaults to /tmp/varnishkafka.stats.json
+log.statistics.file = <%= @log_statistics_file %>
# daemonize varnishkafka (boolean)
--
To view, visit https://gerrit.wikimedia.org/r/98601
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I83083f078ae947770ae7fbb79a73fb02832d68be
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet/varnishkafka
Gerrit-Branch: master
Gerrit-Owner: Ottomata <o...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
