Chad has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/384898 )
Change subject: Scap prep: Clean up everything, fix up StartProfiler symlink mess ...................................................................... Scap prep: Clean up everything, fix up StartProfiler symlink mess Basically refactor all of this to not rely on the class state nearly so much. Fixes a ton of pylint suggestions, plus is just easier to follow While we're here, stop making StartProfiler.php a symlink, instead include it like we do with LocalSettings.php Bug: T126306 Change-Id: I88fa235104927e683a9a8a0754d10c16103431f8 --- M scap/plugins/prep.py 1 file changed, 82 insertions(+), 79 deletions(-) Approvals: Chad: Looks good to me, approved 20after4: Looks good to me, but someone else must approve jenkins-bot: Verified diff --git a/scap/plugins/prep.py b/scap/plugins/prep.py index 3db4040..c87d8f0 100644 --- a/scap/plugins/prep.py +++ b/scap/plugins/prep.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +""" + scap.plugins.prep + ~~~~~~~~~~~~~~~~~ + Scap plugin for setting up a new version of MediaWiki for deployment +""" import argparse import multiprocessing import os @@ -8,33 +14,81 @@ import scap.git as git import scap.utils as utils +GERRIT_URL = 'https://gerrit.wikimedia.org/r/' -def VersionParser(v): + +def version_parser(ver): + """Validation our version number formats""" try: - return re.match("(\d+\.\d+(\.\d+-)?wmf\.?\d+|master)", v).group(0) + return re.match("(\d+\.\d+(\.\d+-)?wmf\.?\d+|master)", ver).group(0) except: raise argparse.ArgumentTypeError( - "Branch '%s' does not match required format" % (v,)) + "Branch '%s' does not match required format" % ver) + + +def update_update_strategy(path): + """For all submodules, update the merge strategy""" + with utils.cd(path): + base_cmd = '/usr/bin/git -C %s config ' % path + base_cmd += 'submodule.$name.update rebase' + cmd = "/usr/bin/git submodule foreach --recursive '%s'" % base_cmd + subprocess.call(cmd, shell=True) + + +def write_settings_stub(dest, include): + """Write a silly little PHP file that includes another""" + file_stub = ( + '<?php\n' + + '# Managed by scap (mediawiki-config:/scap/plugins/prep.py)\n' + + '# WARNING: This file is publically viewable on the web. ' + + 'Do not put private data here.\n' + + 'include_once( "%s" );' % include + ) + with open(dest, 'w+') as destfile: + destfile.write(file_stub) + + +def master_stuff(dest_dir): + """If we're operating on a master branch, do some extra weird stuff""" + repos = { + 'extensions': 'mediawiki/extensions', + 'vendor': 'mediawiki/vendor', + 'skins': 'mediawiki/skins', + } + + for dest, upstream in repos.items(): + path = os.path.join(dest_dir, dest) + url = GERRIT_URL + upstream + + if os.path.exists(path): + with utils.cd(path): + subprocess.check_call( + ['/usr/bin/git', 'init'] + ) + subprocess.check_call( + ['/usr/bin/git', 'remote', 'add', 'origin', url] + ) + + git.fetch(path, url) + git.checkout(path, 'master') + git.update_submodules(path, use_upstream=True) + update_update_strategy(path) @cli.command('prep', help='Checkout MediaWiki version to staging') class CheckoutMediaWiki(cli.Application): - gerrit = 'https://gerrit.wikimedia.org/r/p/' - dest_dir = '' - """ Scap sub-command to manage checkout new MediaWiki versions """ @cli.argument('-p', '--prefix', nargs=1, required=False, default='php-', metavar='PREFIX', help='Directory prefix to checkout version to.') - @cli.argument('branch', metavar='BRANCH', type=VersionParser, + @cli.argument('branch', metavar='BRANCH', type=version_parser, help='The name of the branch to operate on.') def main(self, *extra_args): """ Checkout next MediaWiki """ - self.branch = self.arguments.branch - self.dest_dir = os.path.join( + dest_dir = os.path.join( self.config['stage_dir'], - '{}{}'.format(self.arguments.prefix, self.branch) + '{}{}'.format(self.arguments.prefix, self.arguments.branch) ) old_branch = self.active_wikiversions().keys()[0] copy_dir = os.path.join( @@ -42,13 +96,13 @@ '{}{}'.format(self.arguments.prefix, old_branch) ) - if os.path.isdir(self.dest_dir): + if os.path.isdir(dest_dir): self.get_logger().info('Version already checked out') return 0 - git.fetch(self.dest_dir, self.gerrit + 'mediawiki/core', copy_dir) + git.fetch(dest_dir, GERRIT_URL + 'mediawiki/core', copy_dir) - with utils.cd(self.dest_dir): + with utils.cd(dest_dir): if subprocess.call(['/usr/bin/git', 'config', 'branch.autosetuprebase', 'always']) != 0: self.get_logger().warn('Unable to setup auto-rebase') @@ -59,81 +113,30 @@ self.get_logger().warn('Unable to setup submodule fetch jobs') checkout_version = 'master' - if self.branch != 'master': - checkout_version = 'wmf/%s' % self.branch + if self.arguments.branch != 'master': + checkout_version = 'wmf/%s' % self.arguments.branch - git.checkout(self.dest_dir, checkout_version) + git.checkout(dest_dir, checkout_version) if checkout_version == 'master': - self.master_stuff() + master_stuff(dest_dir) else: - git.update_submodules(self.dest_dir, use_upstream=True) - self.update_submodule_update_strategy(self.dest_dir) + git.update_submodules(dest_dir, use_upstream=True) + update_update_strategy(dest_dir) - self.write_localsettings() - self.create_startprofiler_symlink() + write_settings_stub( + os.path.join(dest_dir, 'LocalSettings.php'), + os.path.join(self.config['deploy_dir'], 'wmf-config', + 'CommonSettings.php')) + write_settings_stub( + os.path.join(dest_dir, 'StartProfiler.php'), + os.path.join(self.config['deploy_dir'], 'wmf-config', + 'StartProfiler.php')) - cache_dir = os.path.join(self.dest_dir, 'cache') + cache_dir = os.path.join(dest_dir, 'cache') os.chmod(cache_dir, 0o777) utils.sudo_check_call('l10nupdate', 'mkdir "%s"' % os.path.join(cache_dir, 'l10n')) self.get_logger().info('MediaWiki %s successfully checked out.' % checkout_version) - - def create_startprofiler_symlink(self): - path = os.path.join(self.dest_dir, 'StartProfiler.php') - log = self.get_logger() - if not os.path.exists(path): - os.symlink('../wmf-config/StartProfiler.php', path) - log.info('Created StartProfiler symlink') - else: - log.warning('StartProfiler symlink already exists') - - def write_localsettings(self): - ls_file = os.path.join(self.dest_dir, 'LocalSettings.php') - cs_file = os.path.join(self.config['deploy_dir'], - 'wmf-config', 'CommonSettings.php') - ls_stub = ( - '<?php\n' + - '# Managed by scap (mediawiki-config:/scap/plugins/prep.py)\n' + - '# WARNING: This file is publically viewable on the web. ' + - 'Do not put private data here.\n' + - 'include_once( "%s" );' % cs_file - ) - - with open(ls_file, 'w+') as f: - f.write(ls_stub) - - def update_submodule_update_strategy(self, path): - with utils.cd(path): - base_cmd = '/usr/bin/git -C %s config ' % path - base_cmd += 'submodule.$name.update rebase' - cmd = "/usr/bin/git submodule foreach --recursive '{}'".format( - base_cmd) - subprocess.call(cmd, shell=True) - - def master_stuff(self): - repos = { - 'extensions': 'mediawiki/extensions', - 'vendor': 'mediawiki/vendor', - 'skins': 'mediawiki/skins', - } - - for dest, upstream in repos.items(): - path = os.path.join(self.dest_dir, dest) - url = self.gerrit + upstream - - if os.path.exists(path): - with utils.cd(path): - subprocess.check_call( - ['/usr/bin/git', 'init'] - ) - subprocess.check_call( - ['/usr/bin/git', 'remote', 'add', 'origin', url] - ) - - git.fetch(path, url) - git.checkout(path, 'master') - git.update_submodules(path, use_upstream=True) - self.update_submodule_update_strategy(path) -- To view, visit https://gerrit.wikimedia.org/r/384898 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I88fa235104927e683a9a8a0754d10c16103431f8 Gerrit-PatchSet: 2 Gerrit-Project: operations/mediawiki-config Gerrit-Branch: master Gerrit-Owner: Chad <ch...@wikimedia.org> Gerrit-Reviewer: 20after4 <mmod...@wikimedia.org> Gerrit-Reviewer: Chad <ch...@wikimedia.org> Gerrit-Reviewer: Thcipriani <tcipri...@wikimedia.org> Gerrit-Reviewer: Urbanecm <martin.urba...@wikimedia.cz> Gerrit-Reviewer: Zoranzoki21 <zorandori4...@gmail.com> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
