Hashar has submitted this change and it was merged.
Change subject: 0.1.1-wmf2: support python-statsd >= 3.x
......................................................................
0.1.1-wmf2: support python-statsd >= 3.x
Bug: T107268
Change-Id: I3f67cee2defe3e181415572c3e651e8e088f5b52
---
M debian/changelog
A debian/patches/0002-Debug-dying-task-managers.patch
A debian/patches/0003-Convert-to-use-latest-statsd-version.patch
A debian/patches/0004-Convert-timing-metrics-to-milliseconds.patch
M debian/patches/series
5 files changed, 269 insertions(+), 0 deletions(-)
Approvals:
Hashar: Verified; Looks good to me, approved
diff --git a/debian/changelog b/debian/changelog
index f84dbae..d64805f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,22 @@
+nodepool (0.1.1-wmf2) jessie-wikimedia; urgency=medium
+
+ * Support python-statsd >= 3.x
+ * Patches:
+
+ 0002-Debug-dying-task-managers.patch
+ [757d45b] Debug dying task managers
+ Clark Boylan <clark.boy...@gmail.com>
+
+ 0003-Convert-to-use-latest-statsd-version.patch
+ [8e6b1b9] Convert to use latest statsd version
+ Ian Wienand <iwien...@redhat.com>
+
+ 0004-Convert-timing-metrics-to-milliseconds.patch
+ [bd3c218] Convert timing metrics to milliseconds
+ Timothy Chavez <timothy.cha...@hp.com>
+
+ -- Antoine Musso <has...@free.fr> Thu, 03 Sep 2015 09:51:57 +0000
+
nodepool (0.1.1-wmf1) jessie-wikimedia; urgency=medium
* Bump to upstream 0.1.0
diff --git a/debian/patches/0002-Debug-dying-task-managers.patch
b/debian/patches/0002-Debug-dying-task-managers.patch
new file mode 100644
index 0000000..50d9ca7
--- /dev/null
+++ b/debian/patches/0002-Debug-dying-task-managers.patch
@@ -0,0 +1,77 @@
+From: Clark Boylan <clark.boy...@gmail.com>
+Date: Wed, 27 May 2015 09:50:55 -0700
+Subject: Debug dying task managers
+
+Change-Id: I81684026071b5abd6162e9f117e7fad08da61ea2
+---
+ nodepool/task_manager.py | 56 ++++++++++++++++++++++++++----------------------
+ 1 file changed, 30 insertions(+), 26 deletions(-)
+
+diff --git a/nodepool/task_manager.py b/nodepool/task_manager.py
+index e48de1a..6e9cc95 100644
+--- a/nodepool/task_manager.py
++++ b/nodepool/task_manager.py
+@@ -79,33 +79,37 @@ class TaskManager(threading.Thread):
+
+ def run(self):
+ last_ts = 0
+- while True:
+- task = self.queue.get()
+- if not task:
+- if not self._running:
+- break
+- continue
++ try:
+ while True:
+- delta = time.time() - last_ts
+- if delta >= self.rate:
+- break
+- time.sleep(self.rate - delta)
+- self.log.debug("Manager %s running task %s (queue: %s)" %
+- (self.name, task, self.queue.qsize()))
+- start = time.time()
+- self.runTask(task)
+- last_ts = time.time()
+- dt = last_ts - start
+- self.log.debug("Manager %s ran task %s in %ss" %
+- (self.name, task, dt))
+- if statsd:
+- #nodepool.task.PROVIDER.subkey
+- subkey = type(task).__name__
+- key = 'nodepool.task.%s.%s' % (self.name, subkey)
+- statsd.timing(key, dt)
+- statsd.incr(key)
+-
+- self.queue.task_done()
++ task = self.queue.get()
++ if not task:
++ if not self._running:
++ break
++ continue
++ while True:
++ delta = time.time() - last_ts
++ if delta >= self.rate:
++ break
++ time.sleep(self.rate - delta)
++ self.log.debug("Manager %s running task %s (queue: %s)" %
++ (self.name, task, self.queue.qsize()))
++ start = time.time()
++ self.runTask(task)
++ last_ts = time.time()
++ dt = last_ts - start
++ self.log.debug("Manager %s ran task %s in %ss" %
++ (self.name, task, dt))
++ if statsd:
++ #nodepool.task.PROVIDER.subkey
++ subkey = type(task).__name__
++ key = 'nodepool.task.%s.%s' % (self.name, subkey)
++ statsd.timing(key, dt)
++ statsd.incr(key)
++
++ self.queue.task_done()
++ except Exception:
++ self.log.exception("Task manager died.")
++ raise
+
+ def submitTask(self, task):
+ if not self._running:
diff --git a/debian/patches/0003-Convert-to-use-latest-statsd-version.patch
b/debian/patches/0003-Convert-to-use-latest-statsd-version.patch
new file mode 100644
index 0000000..d28fc55
--- /dev/null
+++ b/debian/patches/0003-Convert-to-use-latest-statsd-version.patch
@@ -0,0 +1,129 @@
+From: Ian Wienand <iwien...@redhat.com>
+Date: Thu, 9 Apr 2015 09:53:19 +1000
+Subject: Convert to use latest statsd version
+
+statsd >= 3.0 changed the way it initializes itself; to start-up from
+environment variables you need to import from 'statsd.defaults.env'.
+It is also slightly different in that it provides default values; so
+we check if the environment variable is set and avoid importing it if
+statsd isn't configured.
+
+This moves the statsd object creation into a common module so it can
+be shared rather than create multiple clients.
+
+Documentation is also updated to describe how to configure this
+
+Change-Id: I4791c9d26f2309f78a556de42af5b9945005aa46
+---
+ doc/source/installation.rst | 7 +++++--
+ nodepool/nodepool.py | 2 +-
+ nodepool/stats.py | 34 ++++++++++++++++++++++++++++++++++
+ nodepool/task_manager.py | 2 +-
+ requirements.txt | 2 +-
+ 5 files changed, 42 insertions(+), 5 deletions(-)
+ create mode 100644 nodepool/stats.py
+
+diff --git a/doc/source/installation.rst b/doc/source/installation.rst
+index 2b49519..3b3d47b 100644
+--- a/doc/source/installation.rst
++++ b/doc/source/installation.rst
+@@ -41,8 +41,11 @@ number of nodes you expect to be in use at once.
+ Statsd and Graphite
+ ~~~~~~~~~~~~~~~~~~~
+
+-If you have a Graphite system with statsd, Nodepool can be configured
+-to send information to statsd.
++If you have a Graphite system with ``statsd``, Nodepool can be
++configured to send information to it. Set the environment variable
++``STATSD_HOST`` to the ``statsd`` hostname (and optionally
++``STATSD_PORT`` if this should be different to the default ``8125``)
++for the Nodepool daemon to enable this support.
+
+ Install Nodepool
+ ----------------
+diff --git a/nodepool/nodepool.py b/nodepool/nodepool.py
+index 691f35f..21f4c4e 100644
+--- a/nodepool/nodepool.py
++++ b/nodepool/nodepool.py
+@@ -16,7 +16,6 @@
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+-from statsd import statsd
+ import apscheduler.scheduler
+ import gear
+ import json
+@@ -38,6 +37,7 @@ import jenkins_manager
+ import nodedb
+ import nodeutils as utils
+ import provider_manager
++from stats import statsd
+
+ MINS = 60
+ HOURS = 60 * MINS
+diff --git a/nodepool/stats.py b/nodepool/stats.py
+new file mode 100644
+index 0000000..b926c70
+--- /dev/null
++++ b/nodepool/stats.py
+@@ -0,0 +1,34 @@
++#!/usr/bin/env python
++#
++# Licensed under the Apache License, Version 2.0 (the "License"); you may
++# not use this file except in compliance with the License. You may obtain
++# a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
++# License for the specific language governing permissions and limitations
++# under the License.
++
++"""
++Import and set `statsd` if STATSD_HOST is present in the
++environment, else set it to None. This mirrors the behaviour of old
++releases of upstream statsd and avoids us having to change anything
++else.
++"""
++
++import os
++import logging
++
++log = logging.getLogger("nodepool.stats")
++
++if os.getenv('STATSD_HOST', None):
++ from statsd.defaults.env import statsd
++ log.info("Statsd reporting to %s:%s" %
++ (os.getenv('STATSD_HOST'),
++ os.getenv('STATSD_PORT', '8125')))
++else:
++ log.info("Statsd reporting disabled")
++ statsd = None
+diff --git a/nodepool/task_manager.py b/nodepool/task_manager.py
+index 6e9cc95..487e28a 100644
+--- a/nodepool/task_manager.py
++++ b/nodepool/task_manager.py
+@@ -21,7 +21,7 @@ import threading
+ from six.moves import queue as Queue
+ import logging
+ import time
+-from statsd import statsd
++from stats import statsd
+ import requests.exceptions
+
+
+diff --git a/requirements.txt b/requirements.txt
+index 39629dd..ad5c2dd 100644
+--- a/requirements.txt
++++ b/requirements.txt
+@@ -6,7 +6,7 @@ python-jenkins
+ paramiko>1.9.0
+ python-daemon>=2.0.4
+ extras
+-statsd>=1.0.0,<3.0
++statsd>=3.0
+ apscheduler>=2.1.1,<3.0
+ sqlalchemy>=0.8.2,<0.9.0
+ pyzmq>=13.1.0,<14.0.0
diff --git a/debian/patches/0004-Convert-timing-metrics-to-milliseconds.patch
b/debian/patches/0004-Convert-timing-metrics-to-milliseconds.patch
new file mode 100644
index 0000000..2d20dcf
--- /dev/null
+++ b/debian/patches/0004-Convert-timing-metrics-to-milliseconds.patch
@@ -0,0 +1,41 @@
+From: Timothy Chavez <timothy.cha...@hp.com>
+Date: Tue, 2 Jun 2015 11:28:49 -0500
+Subject: Convert timing metrics to milliseconds
+
+Timing metrics which are sent to statsd must be in milliseconds[1].
+
+[1] http://statsd.readthedocs.org/en/latest/types.html#timers
+
+Change-Id: Iaaa40b88aa670aaef41c4dab0231c6cb8f5498bc
+---
+ nodepool/nodepool.py | 3 +--
+ nodepool/task_manager.py | 2 +-
+ 2 files changed, 2 insertions(+), 3 deletions(-)
+
+diff --git a/nodepool/nodepool.py b/nodepool/nodepool.py
+index 21f4c4e..d0a20ff 100644
+--- a/nodepool/nodepool.py
++++ b/nodepool/nodepool.py
+@@ -127,8 +127,7 @@ class NodeCompleteThread(threading.Thread):
+
+ if statsd and self.result == 'SUCCESS':
+ start = node.state_time
+- end = time.time()
+- dt = end - start
++ dt = int((time.time() - start) * 1000)
+
+ # nodepool.job.tempest
+ key = 'nodepool.job.%s' % self.jobname
+diff --git a/nodepool/task_manager.py b/nodepool/task_manager.py
+index 487e28a..48108d7 100644
+--- a/nodepool/task_manager.py
++++ b/nodepool/task_manager.py
+@@ -103,7 +103,7 @@ class TaskManager(threading.Thread):
+ #nodepool.task.PROVIDER.subkey
+ subkey = type(task).__name__
+ key = 'nodepool.task.%s.%s' % (self.name, subkey)
+- statsd.timing(key, dt)
++ statsd.timing(key, int(dt * 1000))
+ statsd.incr(key)
+
+ self.queue.task_done()
diff --git a/debian/patches/series b/debian/patches/series
index 2117df4..d16f52d 100644
--- a/debian/patches/series
+++ b/debian/patches/series
@@ -1 +1,4 @@
0001-Stop-all-threads-on-SIGUSR1.patch
+0002-Debug-dying-task-managers.patch
+0003-Convert-to-use-latest-statsd-version.patch
+0004-Convert-timing-metrics-to-milliseconds.patch
--
To view, visit https://gerrit.wikimedia.org/r/235696
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I3f67cee2defe3e181415572c3e651e8e088f5b52
Gerrit-PatchSet: 1
Gerrit-Project: operations/debs/nodepool
Gerrit-Branch: debian
Gerrit-Owner: Hashar <has...@free.fr>
Gerrit-Reviewer: Hashar <has...@free.fr>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
