Ema has submitted this change and it was merged. Change subject: Port varnishprocessor to new VSL API ......................................................................
Port varnishprocessor to new VSL API Bug: T131353 Change-Id: Ife9d80c67ed91b6c247b438ab04a4eff2d773973 --- A modules/varnish/files/varnishprocessor4/__init__.py A modules/varnish/files/varnishprocessor4/varnishprocessor.py M modules/varnish/manifests/common.pp 3 files changed, 117 insertions(+), 1 deletion(-) Approvals: Ema: Verified; Looks good to me, approved diff --git a/modules/varnish/files/varnishprocessor4/__init__.py b/modules/varnish/files/varnishprocessor4/__init__.py new file mode 100644 index 0000000..c73bf42 --- /dev/null +++ b/modules/varnish/files/varnishprocessor4/__init__.py @@ -0,0 +1 @@ +from .varnishprocessor import VarnishLogProcessor # noqa diff --git a/modules/varnish/files/varnishprocessor4/varnishprocessor.py b/modules/varnish/files/varnishprocessor4/varnishprocessor.py new file mode 100644 index 0000000..e286093 --- /dev/null +++ b/modules/varnish/files/varnishprocessor4/varnishprocessor.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +""" + VarnishLogProcessor + ~~~~~~~~~~~~~~~~~~~ + + Processes Varnish log data and sends the output to statsd + + Copyright 2015 Ori Livneh <o...@wikimedia.org> + Copyright 2015 Gilles Dubuc <gil...@wikimedia.org> + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +""" +import argparse +import io +import socket +import urlparse + + +class VarnishLogProcessor: + description = 'Varnish Log Processor' + key_prefix = None + statsd_server = None + + def __init__(self): + ap = argparse.ArgumentParser( + description=self.description, + epilog='If no statsd server is specified, prints stats to stdout.' + ) + + print self.key_prefix + + ap.add_argument('--key-prefix', help='metric key prefix', + type=parse_prefix_string, default=self.key_prefix) + ap.add_argument('--statsd-server', help='statsd server', + type=parse_statsd_server_string, default=None) + args = ap.parse_args() + + if args.key_prefix is not None: + self.key_prefix = args.key_prefix + + if args.statsd_server is not None: + self.statsd_server = args.statsd_server + + if self.statsd_server: + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + self.stats = {} + self.transactions = {} + + self.start() + + def handle_log_record(self, transaction_id, tag, record, remote_party): + """VSL_handler_f callback function.""" + + if tag == 'ReqURL': + # RxURL is the first tag we expect. If there are any existing + # records for this transaction ID, we clear them away. + self.transactions[transaction_id] = {tag: record} + elif tag == 'Timestamp' and 'Resp' in record: + # This is the last tag we expect. We pop the transaction's records + # from the buffer and process it. + transaction = self.transactions.pop(transaction_id, None) + if transaction is not None: + transaction[tag] = record + self.process_transaction(transaction) + else: + # All other tags are buffered. + transaction = self.transactions.get(transaction_id) + if transaction is not None: + transaction[tag] = record + return 0 + + def flush_stats(self): + """Flush metrics to standard out or statsd server.""" + buf = io.BytesIO() + while self.stats: + key, value = self.stats.popitem() + metric = '%s.%s:%s|c\n' % (self.key_prefix, key, value) + buf.write(metric.encode('utf-8')) + buf.seek(io.SEEK_SET) + if self.statsd_server: + self.sock.sendto(buf.read(), self.statsd_server) + else: + print(buf.read().decode('utf-8').rstrip()) + + def process_transaction(self, transaction): + pass + + def start(): + pass + + +def parse_statsd_server_string(server_string): + parsed = urlparse.urlparse('//' + server_string) + return parsed.hostname, parsed.port or 8125 + + +def parse_prefix_string(prefix): + print prefix + + prefix = prefix.strip('.') + if not prefix: + raise ValueError('Key prefix must not be empty') + return prefix diff --git a/modules/varnish/manifests/common.pp b/modules/varnish/manifests/common.pp index cb8e325..53bf532 100644 --- a/modules/varnish/manifests/common.pp +++ b/modules/varnish/manifests/common.pp @@ -36,7 +36,7 @@ } file { '/usr/local/lib/python2.7/dist-packages/varnishprocessor': - source => 'puppet:///modules/varnish/varnishprocessor', + source => "puppet:///modules/varnish/varnishprocessor${varnish4_python_suffix}", owner => 'root', group => 'root', mode => '0755', -- To view, visit https://gerrit.wikimedia.org/r/305525 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ife9d80c67ed91b6c247b438ab04a4eff2d773973 Gerrit-PatchSet: 3 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ema <e...@wikimedia.org> Gerrit-Reviewer: BBlack <bbl...@wikimedia.org> Gerrit-Reviewer: Ema <e...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
