jenkins-bot has submitted this change and it was merged.
Change subject: Rework logging infrastructure
......................................................................
Rework logging infrastructure
- Consistently log using logging.* instead of print
- Use a consistent log format, including timestamps
- Add consistent --logfile parameter to switch between
stdout logging and logfile logging. The logfile is
rotated nightly and 7 historical files are kept
(using centralized argparse parser)
- Make sure log files are created in mode 600 instead of something
world-readable, in case sensitive information is logged
Minor changes:
- Kill irc3's logging config by adding logging_config = {'version': 1}
- Clean up wikibugs.py arg handling using the new argparse-based system
- Make SGE tasks use --logfile
Bug: T1232
Change-Id: Ic60419aaf31664f558555247359a82b20dd6b55e
---
M channelfilter.py
M fabfile.py
M redis2irc.py
A wblogging.py
M wikibugs.py
5 files changed, 85 insertions(+), 25 deletions(-)
Approvals:
Legoktm: Looks good to me, approved
jenkins-bot: Verified
diff --git a/channelfilter.py b/channelfilter.py
index bd9251d..6fac33f 100644
--- a/channelfilter.py
+++ b/channelfilter.py
@@ -6,6 +6,10 @@
import yaml
import re
+import logging
+
+logger = logging.getLogger('wikibugs2.channelfilter')
+
class ChannelFilter(object):
def __init__(self, path=None):
@@ -24,7 +28,7 @@
self.parse_regexps()
self.time = time.time()
self.mtime = os.path.getmtime(self.path)
- print(self.config)
+ logger.info(self.config)
def parse_regexps(self):
chan_proj_map = self.config['channels']
diff --git a/fabfile.py b/fabfile.py
index 2ef5dfe..a54801a 100644
--- a/fabfile.py
+++ b/fabfile.py
@@ -14,8 +14,8 @@
python = '{}/py-wikibugs2/bin/python'.format(home_dir)
jobs = {
- 'wb2-phab': '{python} {code_dir}/wikibugs.py',
- 'wb2-irc': '{python} {code_dir}/redis2irc.py',
+ 'wb2-phab': '{python} {code_dir}/wikibugs.py --logfile
{home_dir}/wikibugs.log',
+ 'wb2-irc': '{python} {code_dir}/redis2irc.py --logfile
{home_dir}/redis2irc.log',
}
jsub = '/usr/bin/jsub'
diff --git a/redis2irc.py b/redis2irc.py
index 674c1ef..13c989b 100644
--- a/redis2irc.py
+++ b/redis2irc.py
@@ -5,16 +5,26 @@
import asyncio_redis.encoders
import json
import irc3
-import traceback
+import logging
import channelfilter
import configfetcher
import messagebuilder
+from wblogging import LoggingSetupParser
__version__ = '3.0alpha'
+parser = LoggingSetupParser(
+ description="Read bugs from redis, format them and send them to irc",
+)
+args = parser.parse_args()
+
+logger = logging.getLogger('wikibugs.wb2-irc')
+
class Redis2Irc(irc3.IrcBot):
+ logging_config = {'version': 1}
+
def __init__(self, conf, builder, chanfilter, **kwargs):
"""
:type conf: configfetcher.ConfigFetcher
@@ -72,6 +82,7 @@
updated = bot.chanfilter.update()
if updated:
bot.privmsg('#wikimedia-labs', '!log tools.wikibugs Updated
channels.yaml to: %s' % updated)
+ logger.info('Updated channels.yaml to: %s' % updated)
channels = bot.chanfilter.channels_for(useful_info['projects'])
for chan in channels:
@@ -87,8 +98,7 @@
try:
yield from redislistener(bot)
except Exception:
- bot.log.critical(traceback.format_exc())
- bot.log.info("...restarting Redis listener in a few seconds.")
+ logger.exception("Redis listener crashed; restarting in a few
seconds.")
yield from asyncio.sleep(5)
@@ -109,8 +119,7 @@
useful_info = json.loads(future.value)
asyncio.Task(handle_useful_info(bot, useful_info)) # Do not wait
for response
except Exception:
- bot.log.critical(traceback.format_exc())
- yield from asyncio.sleep(1)
+ logger.exception("Redis configuration failed; retrying.")
def main():
@@ -133,14 +142,13 @@
'irc3.plugins.autojoins',
__name__, # this register MyPlugin
],
- verbose=True,
ctcp={
'version': 'wikibugs2 %s running on irc3 {version}. See {url} for
more details.' % __version__,
'userinfo': '{userinfo}',
'ping': 'PONG',
}
)
- asyncio.Task(redisrunner(bot))
+ asyncio.Task(redisrunner())
bot.run()
if __name__ == '__main__':
diff --git a/wblogging.py b/wblogging.py
new file mode 100644
index 0000000..945adc9
--- /dev/null
+++ b/wblogging.py
@@ -0,0 +1,46 @@
+import os
+import sys
+import logging
+import logging.handlers
+import argparse
+
+
+def private_open(file, flags, dir_fd=None):
+ return os.open(file, flags, mode=0o600, dir_fd=dir_fd)
+
+
+class
PrivateTimedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler):
+ def _open(self):
+ return open(self.baseFilename, self.mode, encoding=self.encoding,
opener=private_open)
+
+
+class LoggingSetupParser(argparse.ArgumentParser):
+ def __init__(self, *args, **kwargs):
+ super(LoggingSetupParser, self).__init__(*args, **kwargs)
+ self.add_argument(
+ "--logfile", dest='logfile', default=None,
+ help="Log to this (rotated) log file; if not provided, log to
stdout",
+ )
+
+ def parse_args(self, *args, **kwargs):
+ result = super(LoggingSetupParser, self).parse_args(*args, **kwargs)
+ self.set_up_logging(result)
+ return result
+
+ def set_up_logging(self, args):
+ logger = logging.getLogger()
+ logger.setLevel(logging.DEBUG)
+
+ if args.logfile:
+ handler = PrivateTimedRotatingFileHandler(
+ args.logfile,
+ when='midnight', backupCount=7,
+ utc=True
+ )
+ print("Logging to %s" % args.logfile)
+ else:
+ handler = logging.StreamHandler(sys.stdout)
+ handler.setLevel(logging.DEBUG)
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s
- %(message)s')
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
diff --git a/wikibugs.py b/wikibugs.py
index d8fd17c..f081585 100644
--- a/wikibugs.py
+++ b/wikibugs.py
@@ -4,20 +4,25 @@
import logging
import phabricator
import time
-import sys
import json
from bs4 import BeautifulSoup
import configfetcher
import rqueue
+from wblogging import LoggingSetupParser
-logger = logging.getLogger()
-logger.setLevel(logging.DEBUG)
+parser = LoggingSetupParser(
+ description="Read bugs from redis, format them and send them to irc",
+)
+parser.add_argument('--raise', dest='raise_errors', action='store_true',
+ help="Raise exceptions instead of just logging them")
+parser.add_argument('files', metavar='file', nargs='*',
+ help="XACT files to parse (listen to phabricator
otherwise)")
-handler = logging.StreamHandler(sys.stdout)
-handler.setLevel(logging.DEBUG)
-formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s -
%(message)s')
-handler.setFormatter(formatter)
-logger.addHandler(handler)
+args = parser.parse_args()
+
+logging.getLogger('requests').setLevel(logging.INFO)
+
+logger = logging.getLogger('wikibugs.wb2-phab')
class Wikibugs2(object):
@@ -281,7 +286,7 @@
info[_type] = None
useful_event_metadata['assignee'] = info
- print(useful_event_metadata)
+ logger.debug(useful_event_metadata)
self.rqueue.put(useful_event_metadata)
@@ -293,12 +298,9 @@
# python -i wikibugs.py ~/errors/XACT-anchor/PHID-TASK-qmkysswakxnzzausyzlv
# then import pdb; pdb.pm() to enter the debugger
- bugs.raise_errors = False
- for file in sys.argv[1:]:
- if file == "--raise":
- bugs.raise_errors = True
- continue
- print("Processing {f}".format(f=file))
+ bugs.raise_errors = args.raise_errors
+ for file in args.files:
+ logger.info("Processing {f}".format(f=file))
from collections import OrderedDict # noqa
bugs.process_event(eval(open(file).readline()))
--
To view, visit https://gerrit.wikimedia.org/r/189290
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ic60419aaf31664f558555247359a82b20dd6b55e
Gerrit-PatchSet: 3
Gerrit-Project: labs/tools/wikibugs2
Gerrit-Branch: master
Gerrit-Owner: Merlijn van Deen <[email protected]>
Gerrit-Reviewer: Legoktm <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
