Dzahn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/385482 )
Change subject: Revert "hhvm: remove ganglia monitoring"
......................................................................
Revert "hhvm: remove ganglia monitoring"
This reverts commit cae83e114ce3a2464948cf53a4a926bb1e6b8585.
Change-Id: I9e3191e725809b7c47b2a868905cad870be4c35d
---
A modules/hhvm/files/monitoring/hhvm_health.py
A modules/hhvm/files/monitoring/hhvm_health.pyconf
A modules/hhvm/files/monitoring/hhvm_mem.py
A modules/hhvm/files/monitoring/hhvm_mem.pyconf
M modules/hhvm/manifests/monitoring.pp
5 files changed, 378 insertions(+), 1 deletion(-)
Approvals:
Dzahn: Verified; Looks good to me, approved
diff --git a/modules/hhvm/files/monitoring/hhvm_health.py
b/modules/hhvm/files/monitoring/hhvm_health.py
new file mode 100644
index 0000000..763dfcd
--- /dev/null
+++ b/modules/hhvm/files/monitoring/hhvm_health.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ Ganglia metric-gathering module for HHVM health stats
+
+"""
+import json
+import logging
+import re
+import sys
+import time
+import urllib2
+
+
+logging.basicConfig(level=logging.INFO, stream=sys.stderr)
+
+
+def flatten(mapping, prefix=''):
+ flat = {}
+ for k, v in mapping.items():
+ k = prefix + re.sub('\W', '', k.replace(' ', '_'))
+ flat.update(flatten(v, k + '.') if isinstance(v, dict) else {k: v})
+ return flat
+
+
+class HealthStats(object):
+ def __init__(self, url, expiry=5):
+ self.url = url
+ self.expiry = expiry
+ self.data = {}
+ self.update()
+
+ def update(self):
+ try:
+ req = urllib2.urlopen(self.url)
+ res = flatten(json.load(req), 'HHVM.')
+ self.data.update(res)
+ self.last_fetched = time.time()
+ except (AttributeError, EnvironmentError, ValueError):
+ logging.exception('Failed to update stats:')
+
+ def expired(self):
+ return time.time() - self.last_fetched > self.expiry
+
+ def get(self, stat):
+ if self.expired():
+ self.update()
+ return self.data[stat]
+
+
+def guess_unit(metric):
+ if 'size' in metric or 'capac' in metric or 'byte' in metric:
+ return 'bytes'
+ return 'count'
+
+
+def metric_init(params):
+ url = params.get('url', 'http://localhost:9002/check-health')
+ stats = HealthStats(url)
+ return [{
+ 'name': str(key),
+ 'value_type': 'uint',
+ 'format': '%u',
+ 'units': guess_unit(key),
+ 'slope': 'both',
+ 'groups': 'HHVM',
+ 'call_back': stats.get,
+ } for key in stats.data]
+
+
+def metric_cleanup():
+ pass
+
+
+def self_test():
+ params = dict(arg.split('=') for arg in sys.argv[1:])
+ metrics = metric_init(params)
+ while 1:
+ for metric in metrics:
+ name = metric['name']
+ call_back = metric['call_back']
+ logging.info('%s: %s', name, call_back(name))
+ time.sleep(5)
+
+
+if __name__ == '__main__':
+ self_test()
diff --git a/modules/hhvm/files/monitoring/hhvm_health.pyconf
b/modules/hhvm/files/monitoring/hhvm_health.pyconf
new file mode 100644
index 0000000..6c898a3
--- /dev/null
+++ b/modules/hhvm/files/monitoring/hhvm_health.pyconf
@@ -0,0 +1,96 @@
+# Ganglia metric module for HHVM health statistics.
+
+modules {
+ module {
+ name = "hhvm_health"
+ language = "python"
+ }
+}
+
+collection_group {
+ collect_every = 20
+ time_threshold = 60
+
+ metric {
+ name = "HHVM.targetcache"
+ title = "Target Cache"
+ value_threshold = 1.0
+ }
+
+
+ metric {
+ name = "HHVM.tcprofsize"
+ title = "Translation Cache - Prof Size"
+ value_threshold = 1.0
+ }
+
+
+ metric {
+ name = "HHVM.units"
+ title = "Units"
+ value_threshold = 1.0
+ }
+
+
+ metric {
+ name = "HHVM.tcfrozensize"
+ title = "Translation Cache - Frozen Size"
+ value_threshold = 1.0
+ }
+
+
+ metric {
+ name = "HHVM.hhbcroarenacapac"
+ title = "HHBC RO Arena Capacity"
+ value_threshold = 1.0
+ }
+
+
+ metric {
+ name = "HHVM.rds"
+ title = "RDS"
+ value_threshold = 1.0
+ }
+
+
+ metric {
+ name = "HHVM.tchotsize"
+ title = "Translation Cache - Hot Size"
+ value_threshold = 1.0
+ }
+
+
+ metric {
+ name = "HHVM.funcs"
+ title = "Funcs"
+ value_threshold = 1.0
+ }
+
+
+ metric {
+ name = "HHVM.queued"
+ title = "Queued"
+ value_threshold = 1.0
+ }
+
+
+ metric {
+ name = "HHVM.tcsize"
+ title = "Translation Cache - Size"
+ value_threshold = 1.0
+ }
+
+
+ metric {
+ name = "HHVM.tccoldsize"
+ title = "Translation Cache - Cold Size"
+ value_threshold = 1.0
+ }
+
+
+ metric {
+ name = "HHVM.load"
+ title = "Load"
+ value_threshold = 1.0
+ }
+}
diff --git a/modules/hhvm/files/monitoring/hhvm_mem.py
b/modules/hhvm/files/monitoring/hhvm_mem.py
new file mode 100644
index 0000000..137b70c
--- /dev/null
+++ b/modules/hhvm/files/monitoring/hhvm_mem.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ Ganglia metric-gathering module for HHVM memory stats
+
+"""
+import json
+import logging
+import re
+import sys
+import time
+import urllib2
+
+
+logging.basicConfig(level=logging.INFO, stream=sys.stderr)
+
+
+def flatten(mapping, prefix=''):
+ flat = {}
+ for k, v in mapping.items():
+ k = prefix + re.sub('\W', '', k.replace(' ', '_'))
+ flat.update(flatten(v, k + '.') if isinstance(v, dict) else {k: v})
+ return flat
+
+
+class MemoryStats(object):
+ def __init__(self, url, expiry=5):
+ self.url = url
+ self.expiry = expiry
+ self.data = {}
+ self.update()
+
+ def update(self):
+ try:
+ req = urllib2.urlopen(self.url)
+ res = flatten(json.load(req)['Memory'], 'HHVM.')
+ self.data.update(res)
+ self.last_fetched = time.time()
+ except (AttributeError, EnvironmentError, ValueError):
+ logging.exception('Failed to update stats:')
+
+ def expired(self):
+ return time.time() - self.last_fetched > self.expiry
+
+ def get(self, stat):
+ if self.expired():
+ self.update()
+ return self.data[stat]
+
+
+def metric_init(params):
+ url = params.get('url', 'http://localhost:9002/memory.json')
+ stats = MemoryStats(url)
+ return [{
+ 'name': str(key),
+ 'value_type': 'uint',
+ 'format': '%u',
+ 'units': 'count' if 'count' in key.lower() else 'bytes',
+ 'slope': 'both',
+ 'groups': 'HHVM',
+ 'call_back': stats.get,
+ } for key in stats.data]
+
+
+def metric_cleanup():
+ pass
+
+
+def self_test():
+ params = dict(arg.split('=') for arg in sys.argv[1:])
+ metrics = metric_init(params)
+ while 1:
+ for metric in metrics:
+ name = metric['name']
+ call_back = metric['call_back']
+ logging.info('%s: %s', name, call_back(name))
+ time.sleep(5)
+
+
+if __name__ == '__main__':
+ self_test()
diff --git a/modules/hhvm/files/monitoring/hhvm_mem.pyconf
b/modules/hhvm/files/monitoring/hhvm_mem.pyconf
new file mode 100644
index 0000000..2856f02
--- /dev/null
+++ b/modules/hhvm/files/monitoring/hhvm_mem.pyconf
@@ -0,0 +1,67 @@
+# Ganglia metric module for HHVM memory statistics.
+
+modules {
+ module {
+ name = "hhvm_mem"
+ language = "python"
+ }
+}
+
+collection_group {
+ collect_every = 20
+ time_threshold = 60
+
+ metric {
+ name = "HHVM.Process_Stats_bytes.Shared"
+ title = "Memory - Process - Shared"
+ value_threshold = 1.0
+ }
+
+ metric {
+ name = "HHVM.Process_Stats_bytes.TextCode"
+ title = "Memory - Process - TextCode"
+ value_threshold = 1.0
+ }
+
+ metric {
+ name = "HHVM.Process_Stats_bytes.Data"
+ title = "Memory - Process - Data"
+ value_threshold = 1.0
+ }
+
+ metric {
+ name = "HHVM.Process_Stats_bytes.VmRss"
+ title = "Memory - Process - VmRss"
+ value_threshold = 1.0
+ }
+
+ metric {
+ name = "HHVM.Breakdown.Static_Strings.Bytes"
+ title = "Memory - Static Strings"
+ value_threshold = 1.0
+ }
+
+ metric {
+ name = "HHVM.Process_Stats_bytes.VmSize"
+ title = "Memory - VmSize"
+ value_threshold = 1.0
+ }
+
+ metric {
+ name = "HHVM.Breakdown.Unknown"
+ title = "Memory - Unknown"
+ value_threshold = 1.0
+ }
+
+ metric {
+ name = "HHVM.Breakdown.Code.Bytes"
+ title = "Memory - Code"
+ value_threshold = 1.0
+ }
+
+ metric {
+ name = "HHVM.Breakdown.Static_Strings.Count"
+ title = "Memory - Static Strings Count"
+ value_threshold = 1.0
+ }
+}
diff --git a/modules/hhvm/manifests/monitoring.pp
b/modules/hhvm/manifests/monitoring.pp
index 868f2f4..27e3691 100644
--- a/modules/hhvm/manifests/monitoring.pp
+++ b/modules/hhvm/manifests/monitoring.pp
@@ -1,6 +1,6 @@
# == Class: hhvm::monitoring
#
-# Provisions Diamond collector for HHVM.
+# Provisions Ganglia metric-gathering modules for HHVM.
#
class hhvm::monitoring {
include ::standard
@@ -10,4 +10,50 @@
diamond::collector { 'HhvmApc':
source => 'puppet:///modules/hhvm/monitoring/hhvm_apc.py',
}
+
+ if $::standard::has_ganglia {
+
+ include ::ganglia
+
+ file { '/usr/lib/ganglia/python_modules/hhvm_mem.py':
+ source => 'puppet:///modules/hhvm/monitoring/hhvm_mem.py',
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ require => Package['ganglia-monitor'],
+ }
+
+ file { '/etc/ganglia/conf.d/hhvm_mem.pyconf':
+ source => 'puppet:///modules/hhvm/monitoring/hhvm_mem.pyconf',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ require => File['/usr/lib/ganglia/python_modules/hhvm_mem.py'],
+ notify => Service['ganglia-monitor'],
+ }
+
+ }
+
+ ## Health statistics
+
+ if $::standard::has_ganglia {
+
+ file { '/usr/lib/ganglia/python_modules/hhvm_health.py':
+ source => 'puppet:///modules/hhvm/monitoring/hhvm_health.py',
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ require => Package['ganglia-monitor'],
+ }
+
+ file { '/etc/ganglia/conf.d/hhvm_health.pyconf':
+ source => 'puppet:///modules/hhvm/monitoring/hhvm_health.pyconf',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ require => File['/usr/lib/ganglia/python_modules/hhvm_health.py'],
+ notify => Service['ganglia-monitor'],
+ }
+
+ }
}
--
To view, visit https://gerrit.wikimedia.org/r/385482
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9e3191e725809b7c47b2a868905cad870be4c35d
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Dzahn <dz...@wikimedia.org>
Gerrit-Reviewer: Dzahn <dz...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: Volans <rcocci...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
