Thcipriani has uploaded a new change for review. https://gerrit.wikimedia.org/r/210618
Change subject: Add script from jenkins beta-update-databases ...................................................................... Add script from jenkins beta-update-databases Change-Id: Ie063022b7b4bfc7fa4e1dac2fd15f32bf3a02d3d Bugs: T96199, T98342 --- A modules/beta/files/wmf-beta-update-databases.py M modules/beta/manifests/autoupdater.pp 2 files changed, 64 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/18/210618/1 diff --git a/modules/beta/files/wmf-beta-update-databases.py b/modules/beta/files/wmf-beta-update-databases.py new file mode 100755 index 0000000..e6b83c6 --- /dev/null +++ b/modules/beta/files/wmf-beta-update-databases.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +####################################################################### +# WARNING: this file is managed by Puppet +# puppet:///modules/beta/wmf-beta-update-databases.py +####################################################################### + +""" +Updates update.php on all dbs listed in all-labs.dblist +""" +import os +import sys +import errno +import multiprocessing +import subprocess + +import time + +def get_staging_dir(): + return os.getenv("MEDIAWIKI_STAGING_DIR", "/srv/mediawiki-staging") + +def do_wait(procs): + """ + wait for command in procs array to execute, dump their output + """ + for p, f, cmd in procs: + if p.wait() > 0: + raise Exception("command: ", cmd, "output: ", f.read()) + f.seek(0) + print f.read().strip() + f.close() + +def run_updates(staging, cores): + """ + run update.php on each wiki found in all-labs.dblist + """ + procs = [] + with open(staging, 'r') as dblist: + for db in dblist: + db = db.strip() + f = os.tmpfile() + cmd = '/usr/local/bin/mwscript update.php --wiki=%s --quick' % db + p = subprocess.Popen(cmd, stdout=f, stderr=f, shell=True) + procs.append((p, f, cmd)) + if (len(procs) >= cores): + do_wait(procs) + procs = [] + +def main(): + staging = os.path.join(get_staging_dir(), 'all-labs.dblist') + if not os.path.exists(staging): + raise IOError(errno.ENOENT, 'Labs dblist not found', staging) + + cores = max(multiprocessing.cpu_count() / 2, 1) + run_updates(staging, cores) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/modules/beta/manifests/autoupdater.pp b/modules/beta/manifests/autoupdater.pp index 3ca9821..9990b7c 100644 --- a/modules/beta/manifests/autoupdater.pp +++ b/modules/beta/manifests/autoupdater.pp @@ -30,6 +30,13 @@ content => template('beta/wmf-beta-mwconfig-update.erb'), } + file { '/usr/local/bin/wmf-beta-update-databases.py': + owner => 'root', + group => 'root', + mode => '0555', + source => 'puppet:///modules/beta/wmf-beta-update-databases.py', + } + git::clone { 'mediawiki/core': directory => "${stage_dir}/php-master", branch => 'master', -- To view, visit https://gerrit.wikimedia.org/r/210618 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ie063022b7b4bfc7fa4e1dac2fd15f32bf3a02d3d Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Thcipriani <tcipri...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits