Legoktm has uploaded a new change for review. https://gerrit.wikimedia.org/r/181433
Change subject: Use RotatingFileHandler for logging, set to ~100MB ...................................................................... Use RotatingFileHandler for logging, set to ~100MB Change-Id: I25c23aa98cb7a912340f15e9bf56fb61432249ea --- M nightly.py 1 file changed, 27 insertions(+), 24 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/labs/tools/extdist refs/changes/33/181433/1 diff --git a/nightly.py b/nightly.py index d07806a..d6d1c53 100644 --- a/nightly.py +++ b/nightly.py @@ -21,6 +21,7 @@ import glob import json import logging +import logging.handlers import os import subprocess import sys @@ -42,7 +43,15 @@ self._repo_list = None self._extension_config = None self.force = force - pass + + # Set up logging + self.logger = logging.getLogger(__file__) + self.logger.setLevel(logging.DEBUG) + handler = logging.handlers.RotatingFileHandler(self.LOG_FILE, maxBytes=100000) + handler.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s %(levelname)s:%(message)s') + handler.setFormatter(formatter) + self.logger.addHandler(handler) @property def repo_list(self): @@ -58,7 +67,7 @@ Does an API request to get the complete list of extensions. Do not call directly. """ - logging.debug('Fetching list of all %s...' % self.REPO_TYPE) + self.logger.debug('Fetching list of all %s...' % self.REPO_TYPE) data = { 'action': 'query', 'list': 'extdistrepos', @@ -83,7 +92,7 @@ Fetch the ExtensionDistributor configuration from the API Do not call this directly. """ - logging.debug('Fetching ExtensionDistributor config from API...') + self.logger.debug('Fetching ExtensionDistributor config from API...') data = { 'action': 'query', 'meta': 'siteinfo', @@ -103,12 +112,6 @@ """ Does basic initialization """ - # Set up logging - logging.basicConfig( - filename=self.LOG_FILE, - level=logging.DEBUG, - format='%(asctime)s %(levelname)s:%(message)s' - ) # Check to make sure nightly.py isn't already running if os.path.exists(self.PID_FILE): @@ -116,7 +119,7 @@ old_pid = f.read() if self.check_pid(int(old_pid)): - logging.warning('Another process of nightly.py is still running, quitting this one') + self.logger.warning('Another process of nightly.py is still running, quitting this one') quit() self.create_pid_file() @@ -141,15 +144,15 @@ create new tarballs if needed """ full_path = os.path.join(self.EXT_PATH, ext) - logging.info('Starting update for %s' % ext) + self.logger.info('Starting update for %s' % ext) if not os.path.exists(full_path): os.chdir(self.EXT_PATH) - logging.debug('Cloning %s' % ext) + self.logger.debug('Cloning %s' % ext) self.shell_exec(['git', 'clone', self.GIT_URL % ext, ext]) pass for branch in self.supported_versions: os.chdir(full_path) - logging.info('Creating %s for %s' % (branch, ext)) + self.logger.info('Creating %s for %s' % (branch, ext)) # Update remotes self.shell_exec(['git', 'fetch']) try: @@ -161,7 +164,7 @@ self.shell_exec(['git', 'checkout', 'origin/%s' % branch]) except subprocess.CalledProcessError: # Just a warning because this is expected for some extensions - logging.warning('could not checkout origin/%s' % branch) + self.logger.warning('could not checkout origin/%s' % branch) continue # Reset everything, again. self.shell_exec(['git', 'clean', '-ffd']) @@ -173,10 +176,10 @@ rev = self.shell_exec(['git', 'rev-parse', '--short', 'HEAD']).strip() tarball_fname = '%s-%s-%s.tar.gz' % (ext, branch, rev) if not self.force and os.path.exists(os.path.join(self.DIST_PATH, tarball_fname)): - logging.debug('No updates to branch, tarball already exists.') + self.logger.debug('No updates to branch, tarball already exists.') continue if self.COMPOSER and os.path.exists('composer.json'): - logging.debug('Running composer install for %s' % ext) + self.logger.debug('Running composer install for %s' % ext) self.shell_exec([self.COMPOSER, 'install']) pass # Create a 'version' file with basic info about the tarball @@ -185,19 +188,19 @@ f.write(self.shell_exec(['date', '+%Y-%m-%dT%H:%M:%S']) + '\n') # TODO: Do this in python f.write(rev + '\n') old_tarballs = glob.glob(os.path.join(self.DIST_PATH, '%s-%s-*.tar.gz' % (ext, branch))) - logging.debug('Deleting old tarballs...') + self.logger.debug('Deleting old tarballs...') for old in old_tarballs: # FIXME: Race condition, we should probably do this later on... os.unlink(old) os.chdir(self.EXT_PATH) # Finally, create the new tarball self.shell_exec(['tar', '--exclude', '.git', '-czPf', tarball_fname, ext]) - logging.debug('Moving new tarballs into dist/') + self.logger.debug('Moving new tarballs into dist/') tarballs = glob.glob(os.path.join(self.EXT_PATH, '*.tar.gz')) for tar in tarballs: fname = tar.split('/')[-1] os.rename(tar, os.path.join(self.DIST_PATH, fname)) - logging.info('Finished update for %s' % ext) + self.logger.info('Finished update for %s' % ext) def check_pid(self, pid): """ @@ -219,20 +222,20 @@ """ with open(self.PID_FILE, 'w') as f: f.write(str(os.getpid())) - logging.info('Creating pid file') + self.logger.info('Creating pid file') def run(self, repos=None): self.init() if repos is None: repos = self.repo_list - logging.info('Processing %s %s' % (len(repos), self.REPO_TYPE)) - logging.info('Starting update of all %s...' % self.REPO_TYPE) + self.logger.info('Processing %s %s' % (len(repos), self.REPO_TYPE)) + self.logger.info('Starting update of all %s...' % self.REPO_TYPE) for repo in repos: try: self.update_extension(repo) except: - logging.error('Updating %s failed, skipping' % repo) - logging.info('Finished update of all %s!' % self.REPO_TYPE) + self.logger.error('Updating %s failed, skipping' % repo) + self.logger.info('Finished update of all %s!' % self.REPO_TYPE) def main(): -- To view, visit https://gerrit.wikimedia.org/r/181433 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I25c23aa98cb7a912340f15e9bf56fb61432249ea Gerrit-PatchSet: 1 Gerrit-Project: labs/tools/extdist Gerrit-Branch: master Gerrit-Owner: Legoktm <legoktm.wikipe...@gmail.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits