Dzahn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/382922 )
Change subject: memcached: remove ganglia monitoring
......................................................................
memcached: remove ganglia monitoring
Bug: T177225
Change-Id: I46403f6307e4c5a1b49bb66b417ee21cc4435ea5
---
M hieradata/role/common/memcached.yaml
D modules/memcached/files/ganglia/gmond_memcached.py
D modules/memcached/files/ganglia/gmond_memcached.pyconf
D modules/memcached/manifests/ganglia.pp
M modules/memcached/manifests/init.pp
D modules/memcached/manifests/monitoring.pp
6 files changed, 1 insertion(+), 537 deletions(-)
Approvals:
jenkins-bot: Verified
Dzahn: Looks good to me, approved
diff --git a/hieradata/role/common/memcached.yaml
b/hieradata/role/common/memcached.yaml
index b5cf27c..93b1384 100644
--- a/hieradata/role/common/memcached.yaml
+++ b/hieradata/role/common/memcached.yaml
@@ -7,3 +7,4 @@
profile::memcached::extended_options:
- 'slab_reassign'
profile::memcached::port: '11211'
+standard::has_ganglia: false
diff --git a/modules/memcached/files/ganglia/gmond_memcached.py
b/modules/memcached/files/ganglia/gmond_memcached.py
deleted file mode 100644
index c85110d..0000000
--- a/modules/memcached/files/ganglia/gmond_memcached.py
+++ /dev/null
@@ -1,373 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import sys
-import traceback
-import os
-import threading
-import time
-import socket
-import select
-
-descriptors = list()
-Desc_Skel = {}
-_Worker_Thread = None
-_Lock = threading.Lock() # synchronization lock
-Debug = False
-
-
-def dprint(f, *v):
- if Debug:
- print >>sys.stderr, "DEBUG: "+f % v
-
-
-def floatable(str):
- try:
- float(str)
- return True
- except:
- return False
-
-
-class UpdateMetricThread(threading.Thread):
-
- def __init__(self, params):
- threading.Thread.__init__(self)
- self.running = False
- self.shuttingdown = False
- self.refresh_rate = 15
- if "refresh_rate" in params:
- self.refresh_rate = int(params["refresh_rate"])
- self.metric = {}
- self.last_metric = {}
- self.timeout = 2
-
- self.host = "localhost"
- self.port = 11211
- if "host" in params:
- self.host = params["host"]
- if "port" in params:
- self.port = int(params["port"])
- self.type = params["type"]
- self.mp = params["metrix_prefix"]
-
- def shutdown(self):
- self.shuttingdown = True
- if not self.running:
- return
- self.join()
-
- def run(self):
- self.running = True
-
- while not self.shuttingdown:
- _Lock.acquire()
- self.update_metric()
- _Lock.release()
- time.sleep(self.refresh_rate)
-
- self.running = False
-
- def update_metric(self):
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- msg = ""
- self.last_metric = self.metric.copy()
- try:
- dprint("connect %s:%d", self.host, self.port)
- sock.connect((self.host, self.port))
- sock.send("stats\r\n")
-
- while True:
- rfd, wfd, xfd = select.select([sock], [], [], self.timeout)
- if not rfd:
- print >>sys.stderr, "ERROR: select timeout"
- break
-
- for fd in rfd:
- if fd == sock:
- data = fd.recv(8192)
- msg += data
-
- if msg.find("END"):
- break
-
- sock.close()
- except socket.error, e:
- print >>sys.stderr, "ERROR: %s" % e
-
- for m in msg.split("\r\n"):
- d = m.split(" ")
- if len(d) == 3 and d[0] == "STAT" and floatable(d[2]):
- self.metric[self.mp+"_"+d[1]] = float(d[2])
-
- def metric_of(self, name):
- val = 0
- mp = name.split("_")[0]
- name_tail = name.rsplit("_", 1)
- if (name_tail[1] == "rate" and
- name_tail[0] in self.metric):
- _Lock.acquire()
- name = name_tail[0]
- if name in self.last_metric:
- num = self.metric[name]-self.last_metric[name]
- period = self.metric[mp+"_time"]-self.last_metric[mp+"_time"]
- try:
- val = num/period
- except ZeroDivisionError:
- val = 0
- _Lock.release()
- elif (name_tail[1] == "ratio" and
- "hits" in name_tail[0] and
- name_tail[0] in self.metric and
- name_tail[0].replace("hits", "misses") in self.metric):
- _Lock.acquire()
- hits_name = name_tail[0]
- misses_name = hits_name.replace("hits", "misses")
- if (hits_name in self.last_metric and
- misses_name in self.last_metric):
- hits = self.metric[hits_name]-self.last_metric[hits_name]
- misses = self.metric[misses_name]-self.last_metric[misses_name]
- try:
- val = hits/(hits+misses)
- except ZeroDivisionError:
- val = 0
- _Lock.release()
- elif name in self.metric:
- _Lock.acquire()
- val = self.metric[name]
- _Lock.release()
- return val
-
-
-def metric_init(params):
- global descriptors, Desc_Skel, _Worker_Thread, Debug
-
- print '[memcached] memcached protocol "stats"'
- if "type" not in params:
- params["type"] = "memcached"
-
- if "metrix_prefix" not in params:
- if params["type"] == "memcached":
- params["metrix_prefix"] = "mc"
- elif params["type"] == "Tokyo Tyrant":
- params["metrix_prefix"] = "tt"
-
- print params
-
- # initialize skeleton of descriptors
- Desc_Skel = {
- 'name': 'XXX',
- 'call_back': metric_of,
- 'time_max': 60,
- 'value_type': 'float',
- 'format': '%.0f',
- 'units': 'XXX',
- 'slope': 'XXX', # zero|positive|negative|both
- 'description': 'XXX',
- 'groups': params["type"],
- }
-
- if "refresh_rate" not in params:
- params["refresh_rate"] = 15
- if "debug" in params:
- Debug = params["debug"]
- dprint("%s", "Debug mode on")
-
- _Worker_Thread = UpdateMetricThread(params)
- _Worker_Thread.start()
-
- # IP:HOSTNAME
- if "spoof_host" in params:
- Desc_Skel["spoof_host"] = params["spoof_host"]
-
- mp = params["metrix_prefix"]
-
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_curr_items",
- "units": "items",
- "slope": "both",
- "description": "Current number of items stored",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_cmd_get",
- "units": "commands",
- "slope": "positive",
- "description": "Cumulative number of retrieval reqs",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_cmd_set",
- "units": "commands",
- "slope": "positive",
- "description": "Cumulative number of storage reqs",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_bytes_read",
- "units": "bytes",
- "slope": "positive",
- "description": "Total number of bytes read by this server from
network",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_bytes_written",
- "units": "bytes",
- "slope": "positive",
- "description": "Total number of bytes sent by this server to network",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_bytes",
- "units": "bytes",
- "slope": "both",
- "description": "Current number of bytes used to store items",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_limit_maxbytes",
- "units": "bytes",
- "slope": "both",
- "description": "Number of bytes this server is allowed to use for
storage",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_curr_connections",
- "units": "connections",
- "slope": "both",
- "description": "Number of open connections",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_evictions",
- "units": "items",
- "slope": "both",
- "description": "Number of valid items removed from cache to free
memory for new items",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_get_hits",
- "units": "items",
- "slope": "positive",
- "description": "Number of keys that have been requested and found
present",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_get_misses",
- "units": "items",
- "slope": "positive",
- "description": "Number of items that have been requested and not
found",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_get_hits_rate",
- "units": "items",
- "slope": "both",
- "description": "Hits per second",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_get_misses_rate",
- "units": "items",
- "slope": "both",
- "description": "Misses per second",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_cmd_get_rate",
- "units": "commands",
- "slope": "both",
- "description": "Gets per second",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_cmd_set_rate",
- "units": "commands",
- "slope": "both",
- "description": "Sets per second",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_get_hits_ratio",
- "units": "items",
- "slope": "positive",
- "description": "Get hits ratio (hits/(hits+misses)",
- }))
-
- # Tokyo Tyrant
- if "type" in params and params["type"].lower().find("tokyo") == 0:
- dtmp = descriptors[:]
- for d in dtmp:
- if d["name"] in [
- mp+"_bytes_read",
- mp+"_bytes_written",
- mp+"_limit_maxbytes",
- mp+"_curr_connections",
- mp+"_evictions",
- ]:
- descriptors.remove(d)
- for d in descriptors:
- if d["name"] == mp+"_get_hits":
- d["name"] = mp+"_cmd_get_hits"
- if d["name"] == mp+"_get_misses":
- d["name"] = mp+"_cmd_get_misses"
-
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_cmd_set_hits",
- "units": "items",
- "slope": "positive",
- "description": "Number of keys that have been stored and found
present",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_cmd_set_misses",
- "units": "items",
- "slope": "positive",
- "description": "Number of items that have been stored and not
found",
- }))
-
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_cmd_delete",
- "units": "commands",
- "slope": "positive",
- "description": "Cumulative number of delete reqs",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_cmd_delete_hits",
- "units": "items",
- "slope": "positive",
- "description": "Number of keys that have been deleted and found
present ",
- }))
- descriptors.append(create_desc(Desc_Skel, {
- "name": mp+"_cmd_delete_misses",
- "units": "items",
- "slope": "positive",
- "description": "Number of items that have been deleted and not
found",
- }))
-
- return descriptors
-
-
-def create_desc(skel, prop):
- d = skel.copy()
- for k, v in prop.iteritems():
- d[k] = v
- return d
-
-
-def metric_of(name):
- return _Worker_Thread.metric_of(name)
-
-
-def metric_cleanup():
- _Worker_Thread.shutdown()
-
-
-if __name__ == '__main__':
- try:
- params = {
- "host": "localhost",
- "port": 11211,
- # "host": "tt101",
- # "port": 1978,
- # "type": "Tokyo Tyrant",
- # "metrix_prefix": "tt101",
- "debug": True,
- }
- metric_init(params)
-
- while True:
- for d in descriptors:
- v = d['call_back'](d['name'])
- print ('value for %s is '+d['format']) % (d['name'], v)
- time.sleep(5)
- except KeyboardInterrupt:
- time.sleep(0.2)
- os._exit(1)
- except:
- traceback.print_exc()
- os._exit(1)
diff --git a/modules/memcached/files/ganglia/gmond_memcached.pyconf
b/modules/memcached/files/ganglia/gmond_memcached.pyconf
deleted file mode 100644
index c915c13..0000000
--- a/modules/memcached/files/ganglia/gmond_memcached.pyconf
+++ /dev/null
@@ -1,109 +0,0 @@
-modules {
- module {
- name = "gmond_memcached"
- language = "python"
-
- param host {
- value = "localhost"
- }
- param port {
- value = 11211
- }
- param type {
- value = "memcached"
- }
-
- param refresh_rate {
- value = 15
- }
-
- # param metrix_prefix {
- # value = "mc"
- # }
- # param spoof_host {
- # value = "__IPADDRESS__:__HOSTNAME__"
- # }
- }
-}
-
-collection_group {
- collect_every = 20
- time_threshold = 90
-
- metric {
- name = "mc_curr_items"
- title = "Current number of items stored"
- value_threshold = 0
- }
- metric {
- name = "mc_cmd_get"
- title = "Cumulative number of retrieval reqs"
- value_threshold = 0
- }
- metric {
- name = "mc_cmd_set"
- title = "Cumulative number of storage reqs"
- value_threshold = 0
- }
- metric {
- name = "mc_cmd_get_rate"
- title = "Retrieval reqs/sec"
- value_threshold = 0
- }
- metric {
- name = "mc_cmd_set_rate"
- title = "Storage reqs/sec"
- value_threshold = 0
- }
- metric {
- name = "mc_bytes_read"
- title = "Total number of bytes read by this server from network"
- value_threshold = 0
- }
- metric {
- name = "mc_bytes_written"
- title = "Total number of bytes sent by this server to network"
- value_threshold = 0
- }
- metric {
- name = "mc_bytes"
- title = "Current number of bytes used to store items"
- value_threshold = 0
- }
- metric {
- name = "mc_limit_maxbytes"
- title = "Number of bytes this server is allowed to use for storage"
- value_threshold = 0
- }
- metric {
- name = "mc_curr_connections"
- title = "Number of open connections"
- value_threshold = 0
- }
- metric {
- name = "mc_evictions"
- title = "Number of valid items removed from cache to free memory for new
items"
- value_threshold = 0
- }
- metric {
- name = "mc_get_hits"
- title = "Number of keys that have been requested and found present "
- value_threshold = 0
- }
- metric {
- name = "mc_get_misses"
- title = "Number of items that have been requested and not found"
- value_threshold = 0
- }
- metric {
- name = "mc_get_hits_rate"
- title = "Hits/sec"
- value_threshold = 0
- }
- metric {
- name = "mc_get_misses_rate"
- title = "Misses/sec"
- value_threshold = 0
- }
-
-}
diff --git a/modules/memcached/manifests/ganglia.pp
b/modules/memcached/manifests/ganglia.pp
deleted file mode 100644
index e4b2af9..0000000
--- a/modules/memcached/manifests/ganglia.pp
+++ /dev/null
@@ -1,26 +0,0 @@
-# memcached/ganglia.pp
-
-class memcached::ganglia {
- file { '/usr/lib/ganglia/python_modules/memcached.py':
- ensure => absent,
- }
- file { '/usr/lib/ganglia/python_modules/gmond_memcached.py':
- owner => 'root',
- group => 'root',
- mode => '0444',
- source =>
"puppet:///modules/${module_name}/ganglia/gmond_memcached.py",
- require => File['/usr/lib/ganglia/python_modules'],
- notify => Service['ganglia-monitor'],
- }
- file { '/usr/lib/ganglia/python_modules/memcached.pyconf':
- ensure => absent,
- }
- file { '/etc/ganglia/conf.d/gmond_memcached.pyconf':
- owner => 'root',
- group => 'root',
- mode => '0444',
- source =>
"puppet:///modules/${module_name}/ganglia/gmond_memcached.pyconf",
- require => File['/usr/lib/ganglia/python_modules/gmond_memcached.py'],
- notify => Service['ganglia-monitor'],
- }
-}
diff --git a/modules/memcached/manifests/init.pp
b/modules/memcached/manifests/init.pp
index 37863ff..f80754b 100644
--- a/modules/memcached/manifests/init.pp
+++ b/modules/memcached/manifests/init.pp
@@ -89,9 +89,6 @@
}
include ::standard
- if $::standard::has_ganglia {
- include ::memcached::ganglia
- }
diamond::collector { 'Memcached':
settings => {
diff --git a/modules/memcached/manifests/monitoring.pp
b/modules/memcached/manifests/monitoring.pp
deleted file mode 100644
index ca2ce0d..0000000
--- a/modules/memcached/manifests/monitoring.pp
+++ /dev/null
@@ -1,26 +0,0 @@
-# == Class: memcached::monitoring
-#
-# Provisions Ganglia metric-gathering modules for memcached.
-#
-class memcached::monitoring {
- include ::ganglia
-
- file { '/usr/lib/ganglia/python_modules/memcached.py':
- source => 'puppet:///modules/memcached/ganglia/memcached.py',
- owner => 'root',
- group => 'root',
- mode => '0444',
- before => File['/etc/ganglia/conf.d/memcached.pyconf'],
- require => Package['ganglia-monitor'],
- notify => Service['ganglia-monitor'],
- }
-
- file { '/etc/ganglia/conf.d/memcached.pyconf':
- source => 'puppet:///modules/memcached/ganglia/memcached.pyconf',
- owner => 'root',
- group => 'root',
- mode => '0444',
- require => Package['ganglia-monitor'],
- notify => Service['ganglia-monitor'],
- }
-}
--
To view, visit https://gerrit.wikimedia.org/r/382922
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I46403f6307e4c5a1b49bb66b417ee21cc4435ea5
Gerrit-PatchSet: 4
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Dzahn <dz...@wikimedia.org>
Gerrit-Reviewer: Dzahn <dz...@wikimedia.org>
Gerrit-Reviewer: Volans <rcocci...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
