BryanDavis has uploaded a new change for review.
https://gerrit.wikimedia.org/r/230233
Change subject: logstash: Count MediaWiki log events with statsd
......................................................................
logstash: Count MediaWiki log events with statsd
Count each MediaWiki logstash event by sending an increment command for
the counter "logstash.rate.mediawiki.$CHANNEL.$LEVEL" (eg
"logstash.rate.mediawiki.memcached.ERROR") to a statsd server.
Bug: T100735
Change-Id: I4771abef68a151d08340b63f95cc556c8010416d
---
M hieradata/labs/deployment-prep/common.yaml
M hieradata/role/common/logstash.yaml
M manifests/role/logstash.pp
A modules/logstash/manifests/output/statsd.pp
A modules/logstash/templates/output/statsd.erb
5 files changed, 107 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/33/230233/1
diff --git a/hieradata/labs/deployment-prep/common.yaml
b/hieradata/labs/deployment-prep/common.yaml
index 0879893..f2461ad 100644
--- a/hieradata/labs/deployment-prep/common.yaml
+++ b/hieradata/labs/deployment-prep/common.yaml
@@ -132,6 +132,7 @@
- deployment-elastic08
"elasticsearch::cluster_name": beta-search
"elasticsearch::script_disable_dynamic": false
+role::logstash::statsd_host: labmon1001.eqiad.wmnet
nfs_mounts:
project: false
home: false
diff --git a/hieradata/role/common/logstash.yaml
b/hieradata/role/common/logstash.yaml
index f77985e..b040192 100644
--- a/hieradata/role/common/logstash.yaml
+++ b/hieradata/role/common/logstash.yaml
@@ -39,6 +39,8 @@
- logstash1005.eqiad.wmnet
- logstash1006.eqiad.wmnet
+role::logstash::statsd_host: statsd.eqiad.wmnet
+
# Kibana
role::kibana::vhost: logstash.wmflabs.org
role::kibana::serveradmin: [email protected]
diff --git a/manifests/role/logstash.pp b/manifests/role/logstash.pp
index 2834399..3892ce9 100644
--- a/manifests/role/logstash.pp
+++ b/manifests/role/logstash.pp
@@ -5,7 +5,12 @@
#
# Provisions Logstash and ElasticSearch.
#
-class role::logstash {
+# == Parameters:
+# - $statsd_host: Host to send statsd data to.
+#
+class role::logstash (
+ $statsd_host,
+) {
include ::role::logstash::elasticsearch
include ::logstash
@@ -71,6 +76,7 @@
priority => 70,
}
+ ## Outputs (90)
# Template for Elasticsearch index creation
file { '/etc/logstash/elasticsearch-template.json':
ensure => present,
@@ -88,6 +94,14 @@
template => '/etc/logstash/elasticsearch-template.json',
require => File['/etc/logstash/elasticsearch-template.json'],
}
+
+ logstash::output::statsd { 'MW_channel_rate':
+ host => $statsd_host,
+ guard_condition => '[type] == "mediawiki" and "es" in [tags]',
+ namespace => 'logstash.rate',
+ sender => 'mediawiki',
+ increment => [ '%{channel}.%{level}' ],
+ }
}
# == Class: role::logstash::elasticsearch
diff --git a/modules/logstash/manifests/output/statsd.pp
b/modules/logstash/manifests/output/statsd.pp
new file mode 100644
index 0000000..82dd2ea
--- /dev/null
+++ b/modules/logstash/manifests/output/statsd.pp
@@ -0,0 +1,60 @@
+# == Define: logstash::output::statsd
+#
+# Configure logstash to output to statsd
+#
+# Metric names are formed as "${namespace}.${sender}.${metric}" following the
+# Etsy naming style. $namespace will be omitted if set to an empty string and
+# can be formed using Logstash "%{foo}" style printf substution based on the
+# contents of the Logstash event being processed. $sender will have dots
+# replaced with underscores. These conventions are enforced by the Logstash
+# output plugin, so figure out how deal with them for your use case.
+#
+# == Parameters:
+# - $ensure: Whether the config should exist. Default present.
+# - $host: statsd server. Default '127.0.0.1'.
+# - $port: statsd server port. Default 8125.
+# - $guard_condition: Logstash condition to require to pass events to output.
+# Default undef.
+# - $namespace: The statsd namespace to use for this metric.
+# Default 'logstash'.
+# - $sender: Name of the sender. Dots will be replaced with underscores.
+# Default $host.
+# - $count: Hash of metric_name => count values. Default undef.
+# - $decrement: Array of metric names to decrement. Default undef.
+# - $gauge: Hash of metric_name => gauge values. Default undef.
+# - $increment: Array of metric names to increment. Default undef.
+# - $set: Hash of metric_name => set values. Default undef.
+# - $timing: Hash of metric_name => timing values. Default undef.
+# - $sample_rate: The sample rate for the metric. Default 1.
+#
+# == Sample usage:
+#
+# logstash::output::statsd { 'MW_channel_rate':
+# guard_condition => '[type] == "mediawiki" and "es" in [tags]',
+# namespace => 'logstash.rate',
+# sender => 'mediawiki',
+# increment => [ "%{channel}.%{level}" ],
+# }
+#
+define logstash::output::statsd(
+ $ensure = present,
+ $host = '127.0.0.1',
+ $port = 8125,
+ $guard_condition = undef,
+ $namespace = 'logstash',
+ $sender = $host,
+ $count = undef,
+ $decrement = undef,
+ $gauge = undef,
+ $increment = undef,
+ $set = undef,
+ $timing = undef,
+ $sample_rate = 1,
+) {
+ logstash::conf { "output-statsd-${title}":
+ ensure => $ensure,
+ content => template('logstash/output/statsd.erb'),
+ priority => $priority,
+ }
+}
+# vim:sw=4 ts=4 sts=4 et:
diff --git a/modules/logstash/templates/output/statsd.erb
b/modules/logstash/templates/output/statsd.erb
new file mode 100644
index 0000000..83d031c
--- /dev/null
+++ b/modules/logstash/templates/output/statsd.erb
@@ -0,0 +1,29 @@
+output {
+<% if @guard_condition %>if <%= @guard_condition %> {<% end %>
+ statsd {
+ host => "<%= @host %>"
+ port => <%= @port %>
+ namespace => "<%= @namespace %>"
+ sender => "<%= @sender %>"
+ sample_rate => <%= @sample_rate %>
+ <% if @count %>
+ count => [ "<%= @count.map{|e| e.join('","') }.join('","') %>" ]
+ <% endif %>
+ <% if @decrement %>
+ decrement => [ "<%= @decrement.join('","') %>" ]
+ <% endif %>
+ <% if @gauge %>
+ gauge => [ "<%= @gauge.map{|e| e.join('","') }.join('","') %>" ]
+ <% endif %>
+ <% if @increment %>
+ increment => [ "<%= @increment.join('","') %>" ]
+ <% endif %>
+ <% if @set %>
+ set => [ "<%= @set.map{|e| e.join('","') }.join('","') %>" ]
+ <% endif %>
+ <% if @timing %>
+ timing => [ "<%= @timing.map{|e| e.join('","') }.join('","') %>" ]
+ <% endif %>
+ }
+<% if @guard_condition %>}<% end %>
+}
--
To view, visit https://gerrit.wikimedia.org/r/230233
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I4771abef68a151d08340b63f95cc556c8010416d
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
