Dzahn has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/382913 )
Change subject: udp2log: remove ganglia monitoring ...................................................................... udp2log: remove ganglia monitoring Bug: T177225 Change-Id: I47bc09db8d15cf7024e44a0b673d2668321ca8f9 --- D modules/udp2log/files/PacketLossLogtailer.py D modules/udp2log/files/rolematcher.py M modules/udp2log/manifests/monitoring.pp 3 files changed, 0 insertions(+), 460 deletions(-) Approvals: Ottomata: Looks good to me, but someone else must approve jenkins-bot: Verified Dzahn: Looks good to me, approved diff --git a/modules/udp2log/files/PacketLossLogtailer.py b/modules/udp2log/files/PacketLossLogtailer.py deleted file mode 100644 index f843902..0000000 --- a/modules/udp2log/files/PacketLossLogtailer.py +++ /dev/null @@ -1,226 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This plugin for logtailer will crunch WMF packet loss logs and return: -# * average percent loss per server role -# * ninetieth percentile loss per server role -# It will throw out -# * packet loss numbers greater than 98% -# * large margins of error -# * margins of error are greater than percent packet loss -# Note that this plugin depends on a certain apache log format, documented in -# __init__. - -import time -import threading -import re - -# local dependencies -from ganglia_logtailer_helper import GangliaMetricObject -from ganglia_logtailer_helper import LogtailerParsingException -from ganglia_logtailer_helper import LogtailerStateException -import rolematcher - - -class PacketLossLogtailer(object): - # only used in daemon mode - period = 30 - - def __init__(self): - '''This function should initialize any data structures or variables - needed for the internal state of the line parser.''' - self.last_update_time = time.time() - self.day_in_seconds = 24 * 60 * 60 - self.reset_state() - self.lock = threading.RLock() - - # a list of rolematchers which are simple object to determine the role - # of a particular server this list is obtained from crawling - # config-master.wikimedia.org/pybal and parse the available - # configurations - self.matchers = rolematcher.init() - # this is what will match the packet loss lines - # packet loss format : - # %[%Y-%m-%dT%H:%M:%S]t %server lost: (%percentloss ± %margin) - # [2011-10-26T21:20:25] sq86.wikimedia.org lost: (3.61446 ± 19.67462)% - # match keys: date, server, percentloss, margin - regex = '^\[(?P<date>[^]]+)\] (?P<server>[^ ]+) ' - regex += 'lost: \((?P<percentloss>[^ ]+) \+\/- (?P<margin>[^)]+)\)%' - self.reg = re.compile(regex) - # assume we're in daemon mode unless set_check_duration gets called - self.dur_override = False - - # example function for parse line - # takes one argument (text) line to be parsed - # returns nothing - def parse_line(self, line): - '''This function should digest the contents of one line at a time, - updating the internal state variables.''' - self.lock.acquire() - try: - regMatch = self.reg.match(line) - if regMatch: - linebits = regMatch.groupdict() - self.num_hits += 1 - # capture data - percentloss = float(linebits['percentloss']) - margin = float(linebits['margin']) - role = self.determine_role(linebits['server']) - # store for 90th % and average calculations - # on ssl servers, sequence numbers are out of order. - # T79967 - if((margin <= 20) and (percentloss <= 98)): - self.percentloss_dict.setdefault(role, []) - self.percentloss_dict[role].append(percentloss) - # keep behavior of PacketLossLogtailer consistent with - # previous version and have one overall metric that - # can be used for sending notifications, for now keep - # filtering out HTTPS/IP6 traffic as the packetloss numbers - # are inaccurate. - if role != 'https': - self.percentloss_dict.setdefault('all_roles', []) - self.percentloss_dict['all_roles'].append(percentloss) - else: - raise LogtailerParsingException("regmatch failed to match") - - except Exception, e: - self.lock.release() - raise LogtailerParsingException( - "regmatch or contents failed with %s" % e) - self.lock.release() - - def update_rolematchers(self): - if (time.time() - self.last_update_time) > self.day_in_seconds: - self.matchers = rolematcher.init() - self.last_update_time = time.time() - if self.matchers == []: - # downloading the roles did not go well apparently - # fall back to hardcoded roles. - rolematcher.manual_init() - - def determine_role(self, hostname): - if hostname == 'total': - return 'total' - # default group for when we were not able to determine the role - role = 'misc' - for matcher in self.matchers: - if matcher == hostname: - role = matcher.get_role() - break - return role - - # example function for deep copy - # takes no arguments - # returns one object - def deep_copy(self): - '''This function should return a copy of the data structure used to - maintain state. This copy should different from the object that is - currently being modified so that the other thread can deal with it - without fear of it changing out from under it. The format of this - object is internal to the plugin.''' - myret = dict(percentloss_dict=self.percentloss_dict) - return myret - - # example function for reset_state - # takes no arguments - # returns nothing - def reset_state(self): - '''This function resets the internal data structure to 0 (saving - whatever state it needs). This function should be called - immediately after deep copy with a lock in place so the internal - data structures can't be modified in between the two calls. If the - time between calls to get_state is necessary to calculate metrics, - reset_state should store now() each time it's called, and get_state - will use the time since that now() to do its calculations''' - self.num_hits = 0 - self.percentloss_dict = dict() - self.last_reset_time = time.time() - self.update_rolematchers() - - # example for keeping track of runtimes - # takes no arguments - # returns float number of seconds for this run - def set_check_duration(self, dur): - '''This function only used if logtailer is in cron mode. If it is - invoked, get_check_duration should use this value instead of - calculating it.''' - self.duration = dur - self.dur_override = True - - def get_check_duration(self): - '''This function should return the time since the last check. If called - from cron mode, this must be set using set_check_duration(). If in - daemon mode, it should be calculated internally.''' - if(self.dur_override): - duration = self.duration - else: - cur_time = time.time() - duration = cur_time - self.last_reset_time - # the duration should be within 10% of period - acceptable_duration_min = self.period - (self.period / 10.0) - acceptable_duration_max = self.period + (self.period / 10.0) - if (duration < acceptable_duration_min or - duration > acceptable_duration_max): - msg = "time calculation problem - " - msg += "duration (%s) > 10%% away from period (%s)" % ( - duration, self.period) - raise LogtailerStateException(msg) - return duration - - # example function for get_state - # takes no arguments - # returns a dictionary of (metric => metric_object) pairs - def get_state(self): - '''This function should acquire a lock, call deep copy, get the - current time if necessary, call reset_state, then do its - calculations. It should return a list of metric objects.''' - # if number of log lines is 0, then return no data - if (self.num_hits == 0): - return list() - - self.lock.acquire() - try: - mydata = self.deep_copy() - self.get_check_duration() - self.reset_state() - self.lock.release() - except LogtailerStateException, e: - # if something went wrong with deep_copy or the duration, reset and - # continue - self.reset_state() - self.lock.release() - raise e - - # calculate 90th % and average request times - percentloss_dict = mydata['percentloss_dict'] - metrics = list() - for role, percentloss_list in percentloss_dict.iteritems(): - - percentloss_list.sort() - num_entries = len(percentloss_list) - if (num_entries != 0): - packetloss_90th = percentloss_list[int(num_entries * 0.9)] - packetloss_ave = sum(percentloss_list) / len(percentloss_list) - else: - # in this event, all data was thrown out in parse_line - packetloss_90th = 99 - packetloss_ave = 99 - # package up the data you want to submit - # setting tmax to 960 seconds as data may take as long as 15 - # minutes to be processed - if (role == 'all_roles'): - packetloss_ave_metric = GangliaMetricObject( - 'packet_loss_average', packetloss_ave, units='%', tmax=960) - packetloss_90th_metric = GangliaMetricObject( - 'packet_loss_90th', packetloss_90th, units='%', tmax=960) - else: - packetloss_ave_metric = GangliaMetricObject( - 'packet_loss_average:%s' % (role), packetloss_ave, - units='%', tmax=960) - packetloss_90th_metric = GangliaMetricObject( - 'packet_loss_90th:%s' % (role), packetloss_90th, - units='%', tmax=960) - metrics.append(packetloss_ave_metric) - metrics.append(packetloss_90th_metric) - - # return a list of metric objects - return metrics diff --git a/modules/udp2log/files/rolematcher.py b/modules/udp2log/files/rolematcher.py deleted file mode 100644 index 52bb03e..0000000 --- a/modules/udp2log/files/rolematcher.py +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -''' -This script parses the packet-loss.log file and for each server entry -it determines what it's role is. Server role data has been obtained -from noc.wikimedia.org/pybal and can either be downloaded or the -hard-coded version can be used. - -Purpose of this script is to use it in conjunction with -PacketLossLogTailer.py and send packetloss metrics per dc/role to -Ganglia instead of having one overall packetloss metric. - -When this script is called directly it runs in testmode, -PacketLossLogTailer.py is the regular point of entry. -''' - -import httplib -import json -import re -import sys -import urllib2 - - -numbers = re.compile('([0-9]+)') -base_url = 'https://config-master.wikimedia.org/pybal' -dcs = { - 'eqiad': ['apaches', 'api', 'bits', 'https', 'mobile', 'rendering', 'text', 'upload'], - 'esams': ['bits', 'https', 'text', 'upload'], -} - - -class RoleMatcher(object): - def __init__(self, role, regex, start=None, end=None): - self.role = role - self.regex = re.compile(regex) - self.start = start - self.end = end - - def __str__(self): - if self.start is not None: - return '%s:%s-%s' % (self.role, self.start, self.end) - else: - return self.role - - def __eq__(self, hostname): - match = self.regex.match(hostname) - if match: - if self.start and self.end: - try: - number = int(match.group(1)) - except IndexError: - number = -1 - return self.start <= number and number <= self.end - else: - return True - else: - return False - - def get_role(self): - return self.role - - -def manual_init(): - ''' - This is the hard-coded version of the rolematchers, it's useful for - testing purposes. - ''' - matchers = [ - RoleMatcher('eqiad_apache_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1017, 1113), - RoleMatcher('eqiad_apache_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1161, 1220), - RoleMatcher('eqiad_api_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1189, 1208), - RoleMatcher('eqiad_api_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1114, 1148), - RoleMatcher('eqiad_text_cp', 'cp(10[0-9]+)', 1001, 1020), - RoleMatcher('eqiad_upload_cp', 'cp(10[0-9]+)', 1021, 1036), - RoleMatcher('eqiad_mobile_cp', 'cp(104[0-9])+', 1041, 1044), - RoleMatcher('eqiad_ssl-ip6_ssl', 'ssl(10[0-9])+', 1001, 1004), - - RoleMatcher('esams_bits_cp', 'cp(30[0-9]+)\.esams', 3019, 3022), - RoleMatcher('esams_upload_cp', 'cp(30[0-9]+)\.esams', 3003, 3010), - RoleMatcher('esams_ssl-ip6_ssl', 'ssl(300[0-9]+)\.esams', 3001, 3004), - RoleMatcher('esams_text_kns', 'knsq([0-9]+)\.esams', 23, 30), - RoleMatcher('esams_upload_knsq', 'knsq([0-9]+)\.esams', 16, 22), - ] - return matchers - - -def parse(data): - sections = [] - section = [] - for row in data: - # pybal outputs python dictionaries but we are not going to use eval(), - # hence make the dictionary JSON compatible. - row = row.strip().replace('"', '').replace("'", '"') - row = row.replace('True', 'true').replace('False', 'false') - if row == '': - sections.append(section) - section = [] - elif not row.endswith('}'): # have to strip out some random comments - pos = row.find('}') - row = row[0:pos] - else: - if row.startswith('{'): - try: - section.append(json.loads(row)) - except ValueError: - pass - sections.append(section) - return sections - - -def determine_server_id(hostname): - match = re.findall(numbers, hostname) - try: - return int(match[0]) - except IndexError: - return None - - -def determine_start_end_range(section, hostname): - if len(section) == 1: - start = determine_server_id(hostname) - end = start - else: - start = determine_server_id(hostname) - end = determine_server_id(section[-1]['host']) - return start, end - - -def determine_hostname_suffix(hostname): - suffix = '.'.join(hostname.split('.')[1:]) - suffix = suffix.replace('.', '\\.') - return suffix - - -def determine_hostname_prefix(hostname): - prefix = hostname.split('.')[0] - prefix = ''.join([i for i in prefix if not i.isdigit()]) - return prefix - - -def fetch_url(url): - req = urllib2.Request(url) - data = [] - try: - response = urllib2.urlopen(req) - data = response.readlines() - except urllib2.HTTPError, e: - sys.stderr.write('HTTPError = %s' % e.code) - except urllib2.URLError, e: - sys.stderr.write('URLError = %s' % e.reason) - except httplib.HTTPException, e: - sys.stderr.write('HTTPException') - except Exception: - ''' - just to be sure that nothing falls through the cracks - ''' - import traceback - sys.stderr.write('Generic exception: %s' % traceback.format_exc()) - return data - - -def init(): - matchers = [] - for dc, roles in dcs.iteritems(): - for role in roles: - url = '/'.join([base_url, dc, role]) - data = fetch_url(url) - sections = parse(data) - for section in sections: - if len(section) > 0: - hostname = section[0]['host'] - start, end = determine_start_end_range(section, hostname) - prefix = determine_hostname_prefix(hostname) - suffix = determine_hostname_suffix(hostname) - matcher = RoleMatcher( - '%s_%s_%s' % (dc, role, prefix), - '%s([0-9]+)\.%s' % (prefix, suffix), start, end) - matchers.append(matcher) - return matchers - - -if __name__ == '__main__': - if len(sys.argv) != 2: - print ('Please specify path to packetloss log file, ' - 'call this file only for testing purposes.') - sys.exit(-1) - else: - path = sys.argv[1] - - matchers = init() - line_matcher = re.compile( - r'^\[(?P<date>[^]]+)\] (?P<server>[^ ]+) ' - r'lost: \((?P<percentloss>[^ ]+) \+\/- (?P<margin>[^)]+)\)%') - fh = open(path, 'r') - for line in fh: - regMatch = line_matcher.match(line) - if regMatch: - fields = regMatch.groupdict() - hostname = fields['server'] - role = 'misc' # default group for when we were not able to determine the role - for matcher in matchers: - if matcher == hostname: - role = matcher.get_role() - break - if hostname == 'total': - role = 'total' - print hostname, role - fh.close() diff --git a/modules/udp2log/manifests/monitoring.pp b/modules/udp2log/manifests/monitoring.pp index 47ff6f6..a466e38 100644 --- a/modules/udp2log/manifests/monitoring.pp +++ b/modules/udp2log/manifests/monitoring.pp @@ -20,29 +20,4 @@ group => 'root', source => 'puppet:///modules/udp2log/check_udp2log_procs', } - - if $::standard::has_ganglia { - - require_package('ganglia-logtailer') - - file { 'rolematcher.py': - path => '/usr/share/ganglia-logtailer/rolematcher.py', - mode => '0444', - owner => 'root', - group => 'root', - source => 'puppet:///modules/udp2log/rolematcher.py', - } - - file { 'PacketLossLogtailer.py': - path => '/usr/share/ganglia-logtailer/PacketLossLogtailer.py', - mode => '0444', - owner => 'root', - group => 'root', - source => 'puppet:///modules/udp2log/PacketLossLogtailer.py', - } - - # send udp2log socket stats to ganglia. - # include general UDP statistic monitoring. - ganglia::plugin::python{ ['udp_stats', 'udp2log_socket']: } - } } -- To view, visit https://gerrit.wikimedia.org/r/382913 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I47bc09db8d15cf7024e44a0b673d2668321ca8f9 Gerrit-PatchSet: 2 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Dzahn <dz...@wikimedia.org> Gerrit-Reviewer: Dzahn <dz...@wikimedia.org> Gerrit-Reviewer: Ottomata <ao...@wikimedia.org> Gerrit-Reviewer: Volans <rcocci...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits