Dzahn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/382913 )
Change subject: udp2log: remove ganglia monitoring
......................................................................
udp2log: remove ganglia monitoring
Bug: T177225
Change-Id: I47bc09db8d15cf7024e44a0b673d2668321ca8f9
---
D modules/udp2log/files/PacketLossLogtailer.py
D modules/udp2log/files/rolematcher.py
M modules/udp2log/manifests/monitoring.pp
3 files changed, 0 insertions(+), 460 deletions(-)
Approvals:
Ottomata: Looks good to me, but someone else must approve
jenkins-bot: Verified
Dzahn: Looks good to me, approved
diff --git a/modules/udp2log/files/PacketLossLogtailer.py
b/modules/udp2log/files/PacketLossLogtailer.py
deleted file mode 100644
index f843902..0000000
--- a/modules/udp2log/files/PacketLossLogtailer.py
+++ /dev/null
@@ -1,226 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# This plugin for logtailer will crunch WMF packet loss logs and return:
-# * average percent loss per server role
-# * ninetieth percentile loss per server role
-# It will throw out
-# * packet loss numbers greater than 98%
-# * large margins of error
-# * margins of error are greater than percent packet loss
-# Note that this plugin depends on a certain apache log format, documented in
-# __init__.
-
-import time
-import threading
-import re
-
-# local dependencies
-from ganglia_logtailer_helper import GangliaMetricObject
-from ganglia_logtailer_helper import LogtailerParsingException
-from ganglia_logtailer_helper import LogtailerStateException
-import rolematcher
-
-
-class PacketLossLogtailer(object):
- # only used in daemon mode
- period = 30
-
- def __init__(self):
- '''This function should initialize any data structures or variables
- needed for the internal state of the line parser.'''
- self.last_update_time = time.time()
- self.day_in_seconds = 24 * 60 * 60
- self.reset_state()
- self.lock = threading.RLock()
-
- # a list of rolematchers which are simple object to determine the role
- # of a particular server this list is obtained from crawling
- # config-master.wikimedia.org/pybal and parse the available
- # configurations
- self.matchers = rolematcher.init()
- # this is what will match the packet loss lines
- # packet loss format :
- # %[%Y-%m-%dT%H:%M:%S]t %server lost: (%percentloss ± %margin)
- # [2011-10-26T21:20:25] sq86.wikimedia.org lost: (3.61446 ± 19.67462)%
- # match keys: date, server, percentloss, margin
- regex = '^\[(?P<date>[^]]+)\] (?P<server>[^ ]+) '
- regex += 'lost: \((?P<percentloss>[^ ]+) \+\/- (?P<margin>[^)]+)\)%'
- self.reg = re.compile(regex)
- # assume we're in daemon mode unless set_check_duration gets called
- self.dur_override = False
-
- # example function for parse line
- # takes one argument (text) line to be parsed
- # returns nothing
- def parse_line(self, line):
- '''This function should digest the contents of one line at a time,
- updating the internal state variables.'''
- self.lock.acquire()
- try:
- regMatch = self.reg.match(line)
- if regMatch:
- linebits = regMatch.groupdict()
- self.num_hits += 1
- # capture data
- percentloss = float(linebits['percentloss'])
- margin = float(linebits['margin'])
- role = self.determine_role(linebits['server'])
- # store for 90th % and average calculations
- # on ssl servers, sequence numbers are out of order.
- # T79967
- if((margin <= 20) and (percentloss <= 98)):
- self.percentloss_dict.setdefault(role, [])
- self.percentloss_dict[role].append(percentloss)
- # keep behavior of PacketLossLogtailer consistent with
- # previous version and have one overall metric that
- # can be used for sending notifications, for now keep
- # filtering out HTTPS/IP6 traffic as the packetloss numbers
- # are inaccurate.
- if role != 'https':
- self.percentloss_dict.setdefault('all_roles', [])
- self.percentloss_dict['all_roles'].append(percentloss)
- else:
- raise LogtailerParsingException("regmatch failed to match")
-
- except Exception, e:
- self.lock.release()
- raise LogtailerParsingException(
- "regmatch or contents failed with %s" % e)
- self.lock.release()
-
- def update_rolematchers(self):
- if (time.time() - self.last_update_time) > self.day_in_seconds:
- self.matchers = rolematcher.init()
- self.last_update_time = time.time()
- if self.matchers == []:
- # downloading the roles did not go well apparently
- # fall back to hardcoded roles.
- rolematcher.manual_init()
-
- def determine_role(self, hostname):
- if hostname == 'total':
- return 'total'
- # default group for when we were not able to determine the role
- role = 'misc'
- for matcher in self.matchers:
- if matcher == hostname:
- role = matcher.get_role()
- break
- return role
-
- # example function for deep copy
- # takes no arguments
- # returns one object
- def deep_copy(self):
- '''This function should return a copy of the data structure used to
- maintain state. This copy should different from the object that is
- currently being modified so that the other thread can deal with it
- without fear of it changing out from under it. The format of this
- object is internal to the plugin.'''
- myret = dict(percentloss_dict=self.percentloss_dict)
- return myret
-
- # example function for reset_state
- # takes no arguments
- # returns nothing
- def reset_state(self):
- '''This function resets the internal data structure to 0 (saving
- whatever state it needs). This function should be called
- immediately after deep copy with a lock in place so the internal
- data structures can't be modified in between the two calls. If the
- time between calls to get_state is necessary to calculate metrics,
- reset_state should store now() each time it's called, and get_state
- will use the time since that now() to do its calculations'''
- self.num_hits = 0
- self.percentloss_dict = dict()
- self.last_reset_time = time.time()
- self.update_rolematchers()
-
- # example for keeping track of runtimes
- # takes no arguments
- # returns float number of seconds for this run
- def set_check_duration(self, dur):
- '''This function only used if logtailer is in cron mode. If it is
- invoked, get_check_duration should use this value instead of
- calculating it.'''
- self.duration = dur
- self.dur_override = True
-
- def get_check_duration(self):
- '''This function should return the time since the last check. If
called
- from cron mode, this must be set using set_check_duration(). If in
- daemon mode, it should be calculated internally.'''
- if(self.dur_override):
- duration = self.duration
- else:
- cur_time = time.time()
- duration = cur_time - self.last_reset_time
- # the duration should be within 10% of period
- acceptable_duration_min = self.period - (self.period / 10.0)
- acceptable_duration_max = self.period + (self.period / 10.0)
- if (duration < acceptable_duration_min or
- duration > acceptable_duration_max):
- msg = "time calculation problem - "
- msg += "duration (%s) > 10%% away from period (%s)" % (
- duration, self.period)
- raise LogtailerStateException(msg)
- return duration
-
- # example function for get_state
- # takes no arguments
- # returns a dictionary of (metric => metric_object) pairs
- def get_state(self):
- '''This function should acquire a lock, call deep copy, get the
- current time if necessary, call reset_state, then do its
- calculations. It should return a list of metric objects.'''
- # if number of log lines is 0, then return no data
- if (self.num_hits == 0):
- return list()
-
- self.lock.acquire()
- try:
- mydata = self.deep_copy()
- self.get_check_duration()
- self.reset_state()
- self.lock.release()
- except LogtailerStateException, e:
- # if something went wrong with deep_copy or the duration, reset and
- # continue
- self.reset_state()
- self.lock.release()
- raise e
-
- # calculate 90th % and average request times
- percentloss_dict = mydata['percentloss_dict']
- metrics = list()
- for role, percentloss_list in percentloss_dict.iteritems():
-
- percentloss_list.sort()
- num_entries = len(percentloss_list)
- if (num_entries != 0):
- packetloss_90th = percentloss_list[int(num_entries * 0.9)]
- packetloss_ave = sum(percentloss_list) / len(percentloss_list)
- else:
- # in this event, all data was thrown out in parse_line
- packetloss_90th = 99
- packetloss_ave = 99
- # package up the data you want to submit
- # setting tmax to 960 seconds as data may take as long as 15
- # minutes to be processed
- if (role == 'all_roles'):
- packetloss_ave_metric = GangliaMetricObject(
- 'packet_loss_average', packetloss_ave, units='%', tmax=960)
- packetloss_90th_metric = GangliaMetricObject(
- 'packet_loss_90th', packetloss_90th, units='%', tmax=960)
- else:
- packetloss_ave_metric = GangliaMetricObject(
- 'packet_loss_average:%s' % (role), packetloss_ave,
- units='%', tmax=960)
- packetloss_90th_metric = GangliaMetricObject(
- 'packet_loss_90th:%s' % (role), packetloss_90th,
- units='%', tmax=960)
- metrics.append(packetloss_ave_metric)
- metrics.append(packetloss_90th_metric)
-
- # return a list of metric objects
- return metrics
diff --git a/modules/udp2log/files/rolematcher.py
b/modules/udp2log/files/rolematcher.py
deleted file mode 100644
index 52bb03e..0000000
--- a/modules/udp2log/files/rolematcher.py
+++ /dev/null
@@ -1,209 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-'''
-This script parses the packet-loss.log file and for each server entry
-it determines what it's role is. Server role data has been obtained
-from noc.wikimedia.org/pybal and can either be downloaded or the
-hard-coded version can be used.
-
-Purpose of this script is to use it in conjunction with
-PacketLossLogTailer.py and send packetloss metrics per dc/role to
-Ganglia instead of having one overall packetloss metric.
-
-When this script is called directly it runs in testmode,
-PacketLossLogTailer.py is the regular point of entry.
-'''
-
-import httplib
-import json
-import re
-import sys
-import urllib2
-
-
-numbers = re.compile('([0-9]+)')
-base_url = 'https://config-master.wikimedia.org/pybal'
-dcs = {
- 'eqiad': ['apaches', 'api', 'bits', 'https', 'mobile', 'rendering',
'text', 'upload'],
- 'esams': ['bits', 'https', 'text', 'upload'],
-}
-
-
-class RoleMatcher(object):
- def __init__(self, role, regex, start=None, end=None):
- self.role = role
- self.regex = re.compile(regex)
- self.start = start
- self.end = end
-
- def __str__(self):
- if self.start is not None:
- return '%s:%s-%s' % (self.role, self.start, self.end)
- else:
- return self.role
-
- def __eq__(self, hostname):
- match = self.regex.match(hostname)
- if match:
- if self.start and self.end:
- try:
- number = int(match.group(1))
- except IndexError:
- number = -1
- return self.start <= number and number <= self.end
- else:
- return True
- else:
- return False
-
- def get_role(self):
- return self.role
-
-
-def manual_init():
- '''
- This is the hard-coded version of the rolematchers, it's useful for
- testing purposes.
- '''
- matchers = [
- RoleMatcher('eqiad_apache_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1017, 1113),
- RoleMatcher('eqiad_apache_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1161, 1220),
- RoleMatcher('eqiad_api_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1189, 1208),
- RoleMatcher('eqiad_api_mw', 'mw([0-9]+)\.eqiad\.wmnet', 1114, 1148),
- RoleMatcher('eqiad_text_cp', 'cp(10[0-9]+)', 1001, 1020),
- RoleMatcher('eqiad_upload_cp', 'cp(10[0-9]+)', 1021, 1036),
- RoleMatcher('eqiad_mobile_cp', 'cp(104[0-9])+', 1041, 1044),
- RoleMatcher('eqiad_ssl-ip6_ssl', 'ssl(10[0-9])+', 1001, 1004),
-
- RoleMatcher('esams_bits_cp', 'cp(30[0-9]+)\.esams', 3019, 3022),
- RoleMatcher('esams_upload_cp', 'cp(30[0-9]+)\.esams', 3003, 3010),
- RoleMatcher('esams_ssl-ip6_ssl', 'ssl(300[0-9]+)\.esams', 3001, 3004),
- RoleMatcher('esams_text_kns', 'knsq([0-9]+)\.esams', 23, 30),
- RoleMatcher('esams_upload_knsq', 'knsq([0-9]+)\.esams', 16, 22),
- ]
- return matchers
-
-
-def parse(data):
- sections = []
- section = []
- for row in data:
- # pybal outputs python dictionaries but we are not going to use eval(),
- # hence make the dictionary JSON compatible.
- row = row.strip().replace('"', '').replace("'", '"')
- row = row.replace('True', 'true').replace('False', 'false')
- if row == '':
- sections.append(section)
- section = []
- elif not row.endswith('}'): # have to strip out some random comments
- pos = row.find('}')
- row = row[0:pos]
- else:
- if row.startswith('{'):
- try:
- section.append(json.loads(row))
- except ValueError:
- pass
- sections.append(section)
- return sections
-
-
-def determine_server_id(hostname):
- match = re.findall(numbers, hostname)
- try:
- return int(match[0])
- except IndexError:
- return None
-
-
-def determine_start_end_range(section, hostname):
- if len(section) == 1:
- start = determine_server_id(hostname)
- end = start
- else:
- start = determine_server_id(hostname)
- end = determine_server_id(section[-1]['host'])
- return start, end
-
-
-def determine_hostname_suffix(hostname):
- suffix = '.'.join(hostname.split('.')[1:])
- suffix = suffix.replace('.', '\\.')
- return suffix
-
-
-def determine_hostname_prefix(hostname):
- prefix = hostname.split('.')[0]
- prefix = ''.join([i for i in prefix if not i.isdigit()])
- return prefix
-
-
-def fetch_url(url):
- req = urllib2.Request(url)
- data = []
- try:
- response = urllib2.urlopen(req)
- data = response.readlines()
- except urllib2.HTTPError, e:
- sys.stderr.write('HTTPError = %s' % e.code)
- except urllib2.URLError, e:
- sys.stderr.write('URLError = %s' % e.reason)
- except httplib.HTTPException, e:
- sys.stderr.write('HTTPException')
- except Exception:
- '''
- just to be sure that nothing falls through the cracks
- '''
- import traceback
- sys.stderr.write('Generic exception: %s' % traceback.format_exc())
- return data
-
-
-def init():
- matchers = []
- for dc, roles in dcs.iteritems():
- for role in roles:
- url = '/'.join([base_url, dc, role])
- data = fetch_url(url)
- sections = parse(data)
- for section in sections:
- if len(section) > 0:
- hostname = section[0]['host']
- start, end = determine_start_end_range(section, hostname)
- prefix = determine_hostname_prefix(hostname)
- suffix = determine_hostname_suffix(hostname)
- matcher = RoleMatcher(
- '%s_%s_%s' % (dc, role, prefix),
- '%s([0-9]+)\.%s' % (prefix, suffix), start, end)
- matchers.append(matcher)
- return matchers
-
-
-if __name__ == '__main__':
- if len(sys.argv) != 2:
- print ('Please specify path to packetloss log file, '
- 'call this file only for testing purposes.')
- sys.exit(-1)
- else:
- path = sys.argv[1]
-
- matchers = init()
- line_matcher = re.compile(
- r'^\[(?P<date>[^]]+)\] (?P<server>[^ ]+) '
- r'lost: \((?P<percentloss>[^ ]+) \+\/- (?P<margin>[^)]+)\)%')
- fh = open(path, 'r')
- for line in fh:
- regMatch = line_matcher.match(line)
- if regMatch:
- fields = regMatch.groupdict()
- hostname = fields['server']
- role = 'misc' # default group for when we were not able to
determine the role
- for matcher in matchers:
- if matcher == hostname:
- role = matcher.get_role()
- break
- if hostname == 'total':
- role = 'total'
- print hostname, role
- fh.close()
diff --git a/modules/udp2log/manifests/monitoring.pp
b/modules/udp2log/manifests/monitoring.pp
index 47ff6f6..a466e38 100644
--- a/modules/udp2log/manifests/monitoring.pp
+++ b/modules/udp2log/manifests/monitoring.pp
@@ -20,29 +20,4 @@
group => 'root',
source => 'puppet:///modules/udp2log/check_udp2log_procs',
}
-
- if $::standard::has_ganglia {
-
- require_package('ganglia-logtailer')
-
- file { 'rolematcher.py':
- path => '/usr/share/ganglia-logtailer/rolematcher.py',
- mode => '0444',
- owner => 'root',
- group => 'root',
- source => 'puppet:///modules/udp2log/rolematcher.py',
- }
-
- file { 'PacketLossLogtailer.py':
- path => '/usr/share/ganglia-logtailer/PacketLossLogtailer.py',
- mode => '0444',
- owner => 'root',
- group => 'root',
- source => 'puppet:///modules/udp2log/PacketLossLogtailer.py',
- }
-
- # send udp2log socket stats to ganglia.
- # include general UDP statistic monitoring.
- ganglia::plugin::python{ ['udp_stats', 'udp2log_socket']: }
- }
}
--
To view, visit https://gerrit.wikimedia.org/r/382913
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I47bc09db8d15cf7024e44a0b673d2668321ca8f9
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Dzahn <dz...@wikimedia.org>
Gerrit-Reviewer: Dzahn <dz...@wikimedia.org>
Gerrit-Reviewer: Ottomata <ao...@wikimedia.org>
Gerrit-Reviewer: Volans <rcocci...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
