Legoktm has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/375738 )
Change subject: Add 'libraries' repo group ...................................................................... Add 'libraries' repo group And a bunch of internal refactoring to make it easier Change-Id: I122ebf2314cb1cc838c327ed31d294d670f8f6c0 --- M change_password.py A gerrit.py M mw.py M upgrade.py 4 files changed, 103 insertions(+), 18 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/labs/libraryupgrader refs/changes/38/375738/1 diff --git a/change_password.py b/change_password.py index 795aa0b..eb605de 100755 --- a/change_password.py +++ b/change_password.py @@ -23,28 +23,22 @@ from requests.auth import HTTPDigestAuth import sys +import gerrit import upgrade s = requests.Session() - - -def gerrit_request(method, path, **kwargs): - r = s.request(method, 'https://gerrit.wikimedia.org/r/a/' + path, **kwargs) - r.raise_for_status() - - return json.loads(r.text[4:]) def main(): pw = getpass.getpass('HTTP Password for %s: ' % upgrade.GERRIT_USER) auth = HTTPDigestAuth('libraryupgrader', pw) # Check that we're logged in as the right user - account = gerrit_request('GET', path='accounts/self', auth=auth) + account = gerrit.make_request('GET', path='accounts/self', auth=auth) if account['username'] != upgrade.GERRIT_USER: print('Error, logged in as %(username)s (%(email)s)??' % account) sys.exit(1) print('Successfully logged in as %(username)s' % account) - new_password = gerrit_request( + new_password = gerrit.make_request( 'PUT', path='accounts/self/password.http', auth=auth, diff --git a/gerrit.py b/gerrit.py new file mode 100644 index 0000000..9e70ad5 --- /dev/null +++ b/gerrit.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +""" +Common functions for Gerrit things +Copyright (C) 2017 Kunal Mehta <[email protected]> + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. +""" + +import json +import requests + +s = requests.Session() + + +def make_request(method, path, **kwargs): + base = 'https://gerrit.wikimedia.org/r/' + if 'auth' in kwargs: + base += 'a/' + r = s.request(method, base + path, **kwargs) + r.raise_for_status() + + return json.loads(r.text[4:]) + + +def list_projects(prefix=None): + params = {} + if prefix is not None: + params['p'] = prefix + data = make_request('GET', 'projects/', params=params) + repos = set() + for repo, info in data.items(): + if info['state'] != 'ACTIVE': + continue + repos.add(repo) + + yield from sorted(repos) diff --git a/mw.py b/mw.py index 405f5dd..cfa716c 100644 --- a/mw.py +++ b/mw.py @@ -24,16 +24,24 @@ s = requests.session() -@prefetch_generator.background() def get_extension_list(library: str, version_match=None): r = s.get('https://www.mediawiki.org/w/api.php?action=query&list=extdistrepos&formatversion=2&format=json') + repos = set() for type_ in ('extensions', 'skins'): for ext in r.json()['query']['extdistrepos'][type_]: repo = 'mediawiki/' + type_ + '/' + ext - version = repo_info(repo, library) - if version: - if not version_match or version_match != version: - yield {'repo': repo, 'version': version} + repos.add(repo) + + yield from filter_repo_list(sorted(repos), library, version_match=version_match) + + +@prefetch_generator.background() +def filter_repo_list(repos, library, version_match=None): + for repo in repos: + version = repo_info(repo, library) + if version: + if not version_match or version_match != version: + yield {'repo': repo, 'version': version} def repo_info(repo: str, library: str): diff --git a/upgrade.py b/upgrade.py index db76944..d5c1e62 100755 --- a/upgrade.py +++ b/upgrade.py @@ -22,6 +22,7 @@ import sys import docker +import gerrit import mw @@ -36,6 +37,28 @@ BLACKLIST = [ # Per https://gerrit.wikimedia.org/r/375513 'mediawiki/extensions/MediaWikiFarm', +] +# Gerrit repos not under mediawiki/libs/ +OTHER_LIBRARIES = [ + 'AhoCorasick', + 'CLDRPluralRuleParser', + 'HtmlFormatter', + 'IPSet', + 'RelPath', + 'RunningStat', + 'VisualEditor/VisualEditor', + 'WrappedString', + 'at-ease', + 'base-convert', + 'cdb', + 'css-sanitizer', + 'oojs', + 'oojs/ui', + 'php-session-serializer', + 'purtle', + 'testing-access-wrapper', + 'unicodejs', + 'utfnormal', ] @@ -62,10 +85,15 @@ return logs -def get_extension_list(library, version_match): - for info in mw.get_extension_list(library, version_match): +def preprocess_filter(gen): + for info in gen: if info['repo'] not in BLACKLIST: yield info['repo'] + + +def get_library_list(): + yield from gerrit.list_projects('mediawiki/libs/') + yield from OTHER_LIBRARIES def main(): @@ -77,9 +105,17 @@ repo = sys.argv[3] pw = getpass.getpass('HTTP Password for %s: ' % GERRIT_USER) if repo == 'extensions': - repos = get_extension_list(library, version_match=version) + repos = preprocess_filter( + mw.get_extension_list(library, version_match=version) + ) elif repo == 'canaries': - repos = CANARIES + repos = preprocess_filter( + mw.filter_repo_list(CANARIES, library, version_match=version) + ) + elif repo == 'libraries': + repos = preprocess_filter( + mw.filter_repo_list(get_library_list(), library, version_match=version) + ) else: repos = [repo] processed = set() -- To view, visit https://gerrit.wikimedia.org/r/375738 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I122ebf2314cb1cc838c327ed31d294d670f8f6c0 Gerrit-PatchSet: 1 Gerrit-Project: labs/libraryupgrader Gerrit-Branch: master Gerrit-Owner: Legoktm <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
