Zhou Zheng Sheng has uploaded a new change for review. Change subject: AdvancedStatsThread: Throttle duplicated exception log ......................................................................
AdvancedStatsThread: Throttle duplicated exception log When the same exception is raised in the stats function many times, the log message is the same as well, so extra message does not help us on the problem. If the problem is not solved, the same exception will be raised again and again, then the log will contain a lot of garbages. This patch proposes a way to throttle duplicated exception log in stats functions. For the same exception in the same stats function, if it is raised too many times, just suppress the log. Bug-Url: https://bugzilla.redhat.com/show_bug.cgi?id=853725 Change-Id: I2ee04d8d82e2a14b0a003627981c28e5e64a46ab Signed-off-by: Zhou Zheng Sheng <[email protected]> --- M vdsm/utils.py 1 file changed, 45 insertions(+), 2 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/84/8884/1 diff --git a/vdsm/utils.py b/vdsm/utils.py index 002094a..7fab19a 100644 --- a/vdsm/utils.py +++ b/vdsm/utils.py @@ -291,6 +291,45 @@ return bgn_sample, end_sample, (end_time - bgn_time) + +class _ExceptionLogThrottle(object): + ''' + Record previous exception for each statsFunction, throttle the log + if the same exception is raised too many times. + ''' + + def __init__(self, log, dupThreshold=3): + self._log = log + self._dupThreshold = dupThreshold + self._exceptions = {} + + def _exceptionEqual(self, exc1, exc2): + return exc1.__class__ == exc2.__class__ and exc1.args == exc2.args + + def needSuppress(self, statsFunction, exceptObj): + if (statsFunction not in self._exceptions or + not self._exceptionEqual( + self._exceptions[statsFunction]['exception'], exceptObj)): + + self._exceptions[statsFunction] = {'exception': exceptObj, + 'dupCount': 0} + return False + + dupCount = self._exceptions[statsFunction]['dupCount'] + + if dupCount <= self._dupThreshold: + dupCount += 1 + if dupCount == self._dupThreshold: + self._log.debug( + "Log will be throttled for the same exception: %s" % + exceptObj.message) + self._exceptions[statsFunction] = {'exception': exceptObj, + 'dupCount': dupCount} + return False + + return True + + class AdvancedStatsThread(threading.Thread): """ A thread that runs the registered AdvancedStatsFunction objects @@ -311,6 +350,8 @@ self._statsTime = None self._statsFunctions = [] + + self._logThrottle = _ExceptionLogThrottle(log) def addStatsFunction(self, *args): """ @@ -395,8 +436,10 @@ statsFunction() except Exception, e: if not self.handleStatsException(e): - self._log.error("Stats function failed: %s", - statsFunction, exc_info=True) + if not self._logThrottle.needSuppress( + statsFunction, e): + self._log.error("Stats function failed: %s", + statsFunction, exc_info=True) self._stopEvent.wait(waitInterval) intervalAccum = (intervalAccum + waitInterval) % maxInterval -- To view, visit http://gerrit.ovirt.org/8884 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I2ee04d8d82e2a14b0a003627981c28e5e64a46ab Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Zhou Zheng Sheng <[email protected]> _______________________________________________ vdsm-patches mailing list [email protected] https://lists.fedorahosted.org/mailman/listinfo/vdsm-patches
