Gilles has uploaded a new change for review.
https://gerrit.wikimedia.org/r/234157
Change subject: Send image varnish frontend data from logs to statsd
......................................................................
Send image varnish frontend data from logs to statsd
This is a modified version of varnishrls for varnish thumbnail access stats
Bug: T105681
Change-Id: Iae36f1916f28d568c5a9c76f07c55699a092b63c
---
M modules/role/manifests/cache/statsd.pp
A modules/varnish/files/varnishmedia
A modules/varnish/manifests/logging/media.pp
A modules/varnish/templates/initscripts/varnishmedia.systemd.erb
4 files changed, 194 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/57/234157/1
diff --git a/modules/role/manifests/cache/statsd.pp
b/modules/role/manifests/cache/statsd.pp
index e631820..927fef6 100644
--- a/modules/role/manifests/cache/statsd.pp
+++ b/modules/role/manifests/cache/statsd.pp
@@ -23,4 +23,9 @@
::varnish::logging::rls { 'rls':
statsd_server => 'statsd.eqiad.wmnet',
}
+
+ # Media browser cache hit rate and request volume stats.
+ ::varnish::logging::media { 'media':
+ statsd_server => 'statsd.eqiad.wmnet',
+ }
}
diff --git a/modules/varnish/files/varnishmedia
b/modules/varnish/files/varnishmedia
new file mode 100755
index 0000000..6e58ea8
--- /dev/null
+++ b/modules/varnish/files/varnishmedia
@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ varnishmedia
+ ~~~~~~~~~~~~
+
+ Accumulate browser cache hit ratio and total request volume statistics
+ for media requests and report to StatsD.
+
+ Usage: varnishmedia [--statsd-server SERVER] [--key-prefix PREFIX]
+
+ --statsd-server SERVER statsd server (default: none; echo to stdout)
+ --key-prefix PREFIX metric key prefix (default:
media.thumbnail.varnish)
+
+ Copyright 2015 Ori Livneh <o...@wikimedia.org>
+ Copyright 2015 Gilles Dubuc <gil...@wikimedia.org>
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+"""
+from __future__ import division
+
+import argparse
+import io
+import re
+import socket
+import urlparse
+
+import varnishlog
+
+class MediaVarnishLogProcessor:
+
+ def __init__(self, key_prefix='media.thumbnail.varnish',
statsd_server=None):
+ self.key_prefix = key_prefix
+ self.statsd_server = statsd_server
+ if statsd_server:
+ self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ self.stats = {}
+ self.transactions = {}
+
+ def handle_log_record(self, transaction_id, tag, record, remote_party):
+ """VSL_handler_f callback function."""
+
+ if tag == 'RxURL':
+ # RxURL is the first tag we expect. If there are any existing
+ # records for this transaction ID, we clear them away.
+ self.transactions[transaction_id] = {tag: record}
+ elif tag == 'ReqEnd':
+ # ReqEnd is the last tag we expect. We pop the transaction's
+ # records from the buffer and process it.
+ transaction = self.transactions.pop(transaction_id, None)
+ if transaction is not None:
+ transaction[tag] = record
+ self.process_transaction(transaction)
+ else:
+ # All other tags are buffered.
+ transaction = self.transactions.get(transaction_id)
+ if transaction is not None:
+ transaction[tag] = record
+ return 0
+
+ def process_transaction(self, transaction):
+ """Process a single completed transaction."""
+ status_code = transaction['TxStatus']
+ metric_keys = ['reqs.all', 'resps.' + status_code]
+
+ if 'RxHeader' in transaction:
+ metric_keys.append('reqs.if_none_match')
+
+ if 'TxHeader' in transaction:
+ cache_control_header = transaction.get('TxHeader')
+ cache_control = 'no'
+ if cache_control_header:
+ match = re.search(r'(?<=max-age=)\d+', cache_control_header)
+ if match:
+ cache_control = 'short' if match.group() == '300' else
'long'
+ metric_keys.append('responses.%s_cache_control.%s' %
+ (cache_control, status_code))
+
+ for key in metric_keys:
+ self.stats[key] = self.stats.get(key, 0) + 1
+
+ if self.stats['reqs.all'] > 1:
+ self.flush_stats()
+
+ def flush_stats(self):
+ """Flush metrics to standard out or statsd server."""
+ buf = io.BytesIO()
+ while self.stats:
+ key, value = self.stats.popitem()
+ metric = '%s.%s:%s|c\n' % (self.key_prefix, key, value)
+ buf.write(metric.encode('utf-8'))
+ buf.seek(io.SEEK_SET)
+ if self.statsd_server:
+ self.sock.sendto(buf.read(), self.statsd_server)
+ else:
+ print(buf.read().decode('utf-8').rstrip())
+
+ def start(self):
+ varnishlog.varnishlog((
+ ('m', 'RxURL:/thumb/'), # Only look at thumb requests
+ ('n', 'frontend'), # Consider the frontend Varnish instance
+ ('i', 'TxStatus'), # Get TxStatus for the HTTP status code
+ ('i', 'RxURL'), # Get RxURL to match /w/load.php
+ ('i', 'ReqEnd'), # Get ReqEnd to delimit requests
+ ), self.handle_log_record)
+
+def parse_statsd_server_string(server_string):
+ parsed = urlparse.urlparse('//' + server_string)
+ return parsed.hostname, parsed.port or 8125
+
+
+def parse_prefix_string(key_prefix):
+ key_prefix = key_prefix.strip('.')
+ if not key_prefix:
+ raise ValueError('Key prefix must not be empty')
+ return key_prefix
+
+ap = argparse.ArgumentParser(
+ description='Media Browser Cache Hit Ratio StatsD Reporter',
+ epilog='If no statsd server is specified, prints stats to stdout instead.'
+)
+ap.add_argument('--key-prefix', help='metric key prefix',
+ type=parse_prefix_string, default='media.thumbnail.varnish')
+ap.add_argument('--statsd-server', help='statsd server',
+ type=parse_statsd_server_string, default=None)
+args = ap.parse_args()
+
+lp = MediaVarnishLogProcessor(args.key_prefix, args.statsd_server)
+lp.start()
diff --git a/modules/varnish/manifests/logging/media.pp
b/modules/varnish/manifests/logging/media.pp
new file mode 100644
index 0000000..d6c8f33
--- /dev/null
+++ b/modules/varnish/manifests/logging/media.pp
@@ -0,0 +1,40 @@
+# == Define: varnish::logging::media
+#
+# Accumulate browser cache hit ratio and total request volume statistics
+# for Media requests and report to StatsD.
+#
+# === Parameters
+#
+# [*statsd_server*]
+# StatsD server address, in "host:port" format.
+# Defaults to localhost:8125.
+#
+# === Examples
+#
+# varnish::logging::media {
+# statsd_server => 'statsd.eqiad.wmnet:8125
+# }
+#
+define varnish::logging::media( $statsd_server = 'statsd' ) {
+ include varnish::common
+
+ file { '/usr/local/bin/varnishmedia':
+ source => 'puppet:///modules/varnish/varnishmedia',
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ require =>
File['/usr/local/lib/python2.7/dist-packages/varnishlog.py'],
+ notify => Service['varnishmedia'],
+ }
+
+ base::service_unit { 'varnishmedia':
+ ensure => present,
+ systemd => true,
+ strict => false,
+ template_name => 'varnishmedia',
+ require => File['/usr/local/bin/varnishmedia'],
+ service_params => {
+ enable => true,
+ },
+ }
+}
diff --git a/modules/varnish/templates/initscripts/varnishmedia.systemd.erb
b/modules/varnish/templates/initscripts/varnishmedia.systemd.erb
new file mode 100644
index 0000000..982573d
--- /dev/null
+++ b/modules/varnish/templates/initscripts/varnishmedia.systemd.erb
@@ -0,0 +1,9 @@
+[Unit]
+Description=Media Browser Cache Hit Ratio StatsD Reporter
+After=varnish-frontend.service
+
+[Service]
+ExecStart=/usr/local/bin/varnishmedia --statsd-server=<%= @statsd_server %>
+
+[Install]
+WantedBy=multi-user.target
--
To view, visit https://gerrit.wikimedia.org/r/234157
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Iae36f1916f28d568c5a9c76f07c55699a092b63c
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Gilles <gdu...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
