[gentoo-commits] proj/arch-tools:master commit in: /
commit: 85a5fa0e24d89b5b8a41cc63f4c59e2e46faa4a6 Author: Pawel Hajdan, Jr gentoo org> AuthorDate: Sat Jun 14 09:41:35 2014 + Commit: Paweł Hajdan gentoo org> CommitDate: Sat Jun 14 09:41:35 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/arch-tools.git;a=commit;h=85a5fa0e Fix issue in batch-stabilize leading to errors from Bugzilla xmlrpclib.Fault: --- batch-stabilize.py| 3 ++- common.py | 5 + maintainer-timeout.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/batch-stabilize.py b/batch-stabilize.py index 2093da5..053cd19 100755 --- a/batch-stabilize.py +++ b/batch-stabilize.py @@ -68,7 +68,7 @@ if __name__ == "__main__": url = 'https://bugs.gentoo.org/xmlrpc.cgi' print 'You will be prompted for your Gentoo Bugzilla username and password (%s).' % url bugzilla = BugzillaProxy(url) - login(bugzilla) + user, login_data = login(bugzilla) with open(options.input_filename, "r") as input_file: stabilization_dict = {} @@ -188,6 +188,7 @@ if __name__ == "__main__": # We don't close bugs which still have other arches for obvious reasons, # and security bugs because stabilization is not the last step for them. params = {} + params['token'] = login_data['token'] params['ids'] = [bug_id] params['cc'] = {} params['cc']['remove'] = ['%s...@gentoo.org' % options.arch] diff --git a/common.py b/common.py index cf89eb7..578b0e0 100644 --- a/common.py +++ b/common.py @@ -69,10 +69,7 @@ def login(bugzilla): params = {} params['login'] = user params['password'] = password - print 'Logging in' - bugzilla.User.login(params) - - return user + return user, bugzilla.User.login(params) class Bug: diff --git a/maintainer-timeout.py b/maintainer-timeout.py index ba8c322..76c7e59 100755 --- a/maintainer-timeout.py +++ b/maintainer-timeout.py @@ -29,7 +29,7 @@ if __name__ == "__main__": url = 'https://bugs.gentoo.org/xmlrpc.cgi' print 'You will be prompted for your Gentoo Bugzilla username and password (%s).' % url bugzilla = BugzillaProxy(url) - user = login(bugzilla) + user, login_data = login(bugzilla) bugs = bugzilla.Bug.search({'reporter': user, 'summary': ['stabilize', 'stabilization', 'stable'], 'resolution': ''})['bugs'] comments = bugzilla.Bug.comments({'ids': [bug['id'] for bug in bugs]})
[gentoo-commits] proj/arch-tools:master commit in: /
commit: adb66a9d3f80f8f660d362ca5c8363cf85102a0d Author: Samuli Suominen gentoo org> AuthorDate: Mon Apr 7 12:34:37 2014 + Commit: Samuli Suominen gentoo org> CommitDate: Mon Apr 7 12:34:37 2014 + URL: http://git.overlays.gentoo.org/gitweb/?p=proj/arch-tools.git;a=commit;h=adb66a9d List packages that have xfce in metadata.xml for skipping. List girara and zathura for skipping, see bug 502714 and gentoo-dev ML for discussions --- stabilization-candidates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stabilization-candidates.py b/stabilization-candidates.py index 8798518..52c88f1 100755 --- a/stabilization-candidates.py +++ b/stabilization-candidates.py @@ -24,7 +24,7 @@ if __name__ == "__main__": parser.add_option("--days", dest="days", type=int, default=30, help="Number of days in the tree after stabilization is possible.") parser.add_option("--repo", dest="repo", help="Path to portage CVS repository") parser.add_option("--category", dest="category", help="Portage category filter (default is all categories)") - parser.add_option("--exclude", dest="exclude", default=".*(kde-base|sci|lisp|perl-core|virtual|gnome|ruby|x11|mono|dotnet|games|xfce|leechcraft).*", help="Regular expression for excluded packages.") + parser.add_option("--exclude", dest="exclude", default=".*(kde-base|sci|lisp|perl-core|virtual|gnome|ruby|x11|mono|dotnet|games|xfce|xfburn|mousepad|orage|xfbib|thunar|ristretto|pragha|xfmpc|parole|midori|gigolo|rodent|xfwm|girara|zathura|leechcraft).*", help="Regular expression for excluded packages.") parser.add_option("-o", "--output", dest="output_filename", default="stabilization-candidates.txt", help="Output filename for generated stabilization candidates list.") (options, args) = parser.parse_args()
[gentoo-commits] proj/arch-tools:master commit in: /
commit: eed2243011824ddb3525a75409fae1398a11fb11 Author: Pawel Hajdan, Jr gentoo org> AuthorDate: Sun Jul 9 14:42:21 2017 + Commit: Paweł Hajdan gentoo org> CommitDate: Sun Jul 9 14:42:21 2017 + URL:https://gitweb.gentoo.org/proj/arch-tools.git/commit/?id=eed22430 stabilization-candidates: update for git stabilization-candidates.py | 24 +++- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/stabilization-candidates.py b/stabilization-candidates.py index d324bdb..2537ab4 100755 --- a/stabilization-candidates.py +++ b/stabilization-candidates.py @@ -20,7 +20,7 @@ if __name__ == "__main__": parser = optparse.OptionParser() parser.add_option("--arch", dest="arch", action="append", help="Gentoo arch to use, e.g. x86, amd64, ... Can be passed multiple times.") parser.add_option("--days", dest="days", type=int, default=30, help="Number of days in the tree after stabilization is possible.") - parser.add_option("--repo", dest="repo", help="Path to portage CVS repository") + parser.add_option("--repo", dest="repo", help="Path to portage git repository") parser.add_option("--category", dest="category", help="Portage category filter (default is all categories)") parser.add_option("--exclude", dest="exclude", default=".*(kde-base|sci|lisp|perl-core|virtual|gnome|ruby|x11|mono|dotnet|games|xfce|xfburn|mousepad|orage|xfbib|thunar|ristretto|pragha|xfmpc|parole|midori|gigolo|rodent|xfwm|girara|zathura|leechcraft).*", help="Regular expression for excluded packages.") parser.add_option("-o", "--output", dest="output_filename", default="stabilization-candidates.txt", help="Output filename for generated stabilization candidates list.") @@ -116,18 +116,16 @@ if __name__ == "__main__": pv = portage.versions.catsplit(best_candidate)[1] try: - with open(os.path.join(options.repo, cp, 'ChangeLog')) as changelog_file: - regex = '\*%s \((.*)\)' % re.escape(pv) - match = re.search(regex, changelog_file.read()) - if not match: - print('error parsing ChangeLog') - continue - changelog_date = datetime.datetime.strptime(match.group(1), '%d %b %Y') - if now - changelog_date < datetime.timedelta(days=options.days): - print('not old enough') - continue - except IOError as e: - print(e) + git_log = subprocess.check_output( + ['git', 'log', '--date-order', '--date=short', '--format=%cd', '%s.ebuild' % pv], + cwd=os.path.join(options.repo, cp)) + changelog_date_str = git_log.splitlines()[0].decode('utf-8') + changelog_date = datetime.datetime.strptime(changelog_date_str, '%Y-%m-%d') + if now - changelog_date < datetime.timedelta(days=options.days): + print('not old enough') + continue + except subprocess.CalledProcessError as ex: + print('git error: ' + ex.output) continue keywords = portage.db["/"]["porttree"].dbapi.aux_get(best_candidate, ['KEYWORDS'])[0]
[gentoo-commits] proj/arch-tools:master commit in: /
commit: 064b7b3dbf4277b9c57add479a478386504fc0d2 Author: Pawel Hajdan, Jr gentoo org> AuthorDate: Mon Jul 10 20:28:50 2017 + Commit: Paweł Hajdan gentoo org> CommitDate: Mon Jul 10 20:28:50 2017 + URL:https://gitweb.gentoo.org/proj/arch-tools.git/commit/?id=064b7b3d stabilization-candidates: retry network errors common.py | 19 +++ stabilization-candidates.py | 26 +- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/common.py b/common.py index 89e4996..8e0c14c 100644 --- a/common.py +++ b/common.py @@ -1,10 +1,12 @@ # Copyright 2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import functools import io import datetime import getpass import re +import time import portage @@ -17,6 +19,23 @@ def chunks(iterable, length): yield iterable[i:i + length] +def retry(ExceptionToCheck, tries=4, delay=3, backoff=2): + def deco_retry(f): + @functools.wraps(f) + def f_retry(*args, **kwargs): + mtries, mdelay = tries, delay + while mtries > 1: + try: + return f(*args, **kwargs) + except ExceptionToCheck: + time.sleep(mdelay) + mtries -= 1 + mdelay *= backoff + return f(*args, **kwargs) + return f_retry + return deco_retry + + # Snippet from http://bugs.python.org/issue9584 def expand_braces(orig): r = r'.*(\{.+?[^\\]\})' diff --git a/stabilization-candidates.py b/stabilization-candidates.py index 2537ab4..0cedac4 100755 --- a/stabilization-candidates.py +++ b/stabilization-candidates.py @@ -7,6 +7,7 @@ import optparse import os.path import random import re +import socket import subprocess import xmlrpc.client @@ -14,7 +15,7 @@ from portage.package.ebuild.getmaskingstatus import getmaskingstatus from portage.xml.metadata import MetaDataXML import portage.versions -from common import login +from common import login, retry if __name__ == "__main__": parser = optparse.OptionParser() @@ -139,19 +140,26 @@ if __name__ == "__main__": continue # Do not risk trying to stabilize a package with known bugs. - params = {} - params['Bugzilla_token'] = login_data['token'] - params['summary'] = cp - bugs = [x for x in bugzilla.Bug.search(params)['bugs'] if x['is_open'] and x['severity'] not in ['enhancement', 'QA']] + @retry(socket.error) + def get_package_bugs(): + params = {} + params['Bugzilla_token'] = login_data['token'] + params['summary'] = cp + return [x for x in bugzilla.Bug.search(params)['bugs'] + if x['is_open'] and x['severity'] not in ['enhancement', 'QA']] + bugs = get_package_bugs() if bugs: print('has bugs') continue # Protection against filing a stabilization bug twice. - params = {} - params['Bugzilla_token'] = login_data['token'] - params['summary'] = best_candidate - bugs = bugzilla.Bug.search(params)['bugs'] + @retry(socket.error) + def get_package_bugs(): + params = {} + params['Bugzilla_token'] = login_data['token'] + params['summary'] = best_candidate + return bugzilla.Bug.search(params)['bugs'] + bugs = get_package_bugs() if bugs: print('version has bugs') continue
[gentoo-commits] proj/arch-tools:master commit in: /
commit: 842f3d251d81ffc274fa01a1d9ba9b5794a2d714 Author: William Hubbs gmail com> AuthorDate: Sat Dec 13 19:33:09 2014 + Commit: Paweł Hajdan gentoo org> CommitDate: Mon Dec 29 14:52:27 2014 + URL: http://sources.gentoo.org/gitweb/?p=proj/arch-tools.git;a=commit;h=842f3d25 Port most tools to python 3 This commit ports the following tools to python 3. - common code - batch-stabilize - file-stabilization-bugs - maintainer-timeout - reverse-dependencies - stabilization-candidates This is a list of the changes: - Remove the dependency on pybugz; these tools now use xmlrpc.client directly. This is needed because the next release of pybugz does the same thing; we no longer have a bugzilla module since it was just a wrapper. Handle tokens correctly: - When we log into Bugzilla, we are given a token we must pass back to it with every command we issue. - Log out at the end of every session to expire the token. Fix the state file processing: - The pickle module handles binary files, not text files, so read and write the state file in binary mode. - Add fix_imports=True when reading the state file so a python-2 compatible state file can be read. - Do not set the URL for a stabilization request X-Gentoo-Bug: 532054 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=532054 X-Gentoo-Bug: 532368 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=532368 --- batch-stabilize.py | 43 common.py | 4 ++-- file-stabilization-bugs.py | 42 ++- maintainer-timeout.py | 31 ++ reverse-dependencies.py | 6 ++--- stabilization-candidates.py | 54 ++--- 6 files changed, 104 insertions(+), 76 deletions(-) diff --git a/batch-stabilize.py b/batch-stabilize.py index 053cd19..155af38 100755 --- a/batch-stabilize.py +++ b/batch-stabilize.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -11,8 +11,8 @@ import re import shutil import subprocess import sys +import xmlrpc.client -from bugz.bugzilla import BugzillaProxy from common import login import portage.versions @@ -20,7 +20,7 @@ BUG_REGEX = re.compile("[Bb]ug #?(\d+)") def print_and_log(message, log): try: - print message + print(message) log.write(message + '\n') finally: log.flush() @@ -40,7 +40,7 @@ def run_command(args, cwd, log): log.flush() def save_state(done_bugs): - with open('batch-stabilize.state', 'w') as state_file: + with open('batch-stabilize.state', 'wb') as state_file: pickle.dump(done_bugs, state_file) if __name__ == "__main__": @@ -62,12 +62,12 @@ if __name__ == "__main__": done_bugs = [] if os.path.exists('batch-stabilize.state'): - with open('batch-stabilize.state', 'r') as state_file: - done_bugs = pickle.load(state_file) + with open('batch-stabilize.state', 'rb') as state_file: + done_bugs = pickle.load(state_file, fix_imports=True) url = 'https://bugs.gentoo.org/xmlrpc.cgi' - print 'You will be prompted for your Gentoo Bugzilla username and password (%s).' % url - bugzilla = BugzillaProxy(url) + print('You will be prompted for your Gentoo Bugzilla username and password (%s).' % url) + bugzilla = xmlrpc.client.ServerProxy(url) user, login_data = login(bugzilla) with open(options.input_filename, "r") as input_file: @@ -83,14 +83,14 @@ if __name__ == "__main__": if line.startswith("#"): match = BUG_REGEX.search(line, re.IGNORECASE) if not match: - print 'Ignoring comment line [%s]...' % line + print('Ignoring comment line [%s]...' % line) continue else: bug_id = int(match.group(1)) continue if bug_id == -1: - print 'Could not recognize bug id' + print('Could not recognize bug id') sys.exit(1) # Drop the leading '='. @@ -109,10 +109,10 @@ if __name__ == "__main__": for (pn, ebuild_name) in stabilization_dict[bug_id]: ebuild_path = os.path.join(options.repo, pn, ebuild_name) if not os.path.exists(ebuild_path): - print '%s: file does not exist' % ebu
[gentoo-commits] proj/arch-tools:master commit in: /
commit: e484b1da3bd10769d331ab4e914fa45c5660e726 Author: William Hubbs gmail com> AuthorDate: Tue Dec 30 16:46:24 2014 + Commit: Paweł Hajdan gentoo org> CommitDate: Mon Dec 29 14:58:15 2014 + URL: http://sources.gentoo.org/gitweb/?p=proj/arch-tools.git;a=commit;h=e484b1da batch-stabilize: use include-arches option --- batch-stabilize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/batch-stabilize.py b/batch-stabilize.py index 81f97ba..14cd69d 100755 --- a/batch-stabilize.py +++ b/batch-stabilize.py @@ -166,7 +166,8 @@ if __name__ == "__main__": if run_command(["repoman", "manifest"], cvs_path, log_file)[0] != 0: print('!!! repoman manifest failed') sys.exit(1) - if run_command(["repoman", "commit", "--ignore-arches", "-m", commit_message], cvs_path, log_file)[0] != 0: + if run_command(["repoman", "commit", + "--include-arches", options.arch, "-m", commit_message], cvs_path, log_file)[0] != 0: print('!!! repoman commit failed') sys.exit(1) params = {}
[gentoo-commits] proj/arch-tools:master commit in: /
commit: 40f058957e72e82f4e54d2f453c02e51c62f37c5 Author: William Hubbs gmail com> AuthorDate: Tue Dec 23 16:29:12 2014 + Commit: Paweł Hajdan gentoo org> CommitDate: Mon Dec 29 14:57:12 2014 + URL: http://sources.gentoo.org/gitweb/?p=proj/arch-tools.git;a=commit;h=40f05895 Fix subprocess handling in batch-stabilize --- batch-stabilize.py | 22 +- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/batch-stabilize.py b/batch-stabilize.py index 155af38..81f97ba 100755 --- a/batch-stabilize.py +++ b/batch-stabilize.py @@ -26,19 +26,23 @@ def print_and_log(message, log): log.flush() def run_command(args, cwd, log): - try: - message = "Running %r in %s...\n" % (args, cwd) - sys.stdout.write(message) - log.write(message) + message = "Running %r in %s...\n" % (args, cwd) + returncode = 0 + sys.stdout.write(message) + log.write(message) - cmd = subprocess.Popen(args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = cmd.communicate()[0] - log.write("Finished with exit code %d\n" % cmd.returncode) - log.write(output) - return (cmd.returncode, output) + try: + output = subprocess.check_output(args, cwd=cwd, + stderr=subprocess.STDOUT, universal_newlines=True) + except subprocess.CalledProcessError as e: + output = e.output + returncode = e.returncode finally: + log.write("Finished with exit code %d\n" % returncode) + log.write(output) log.flush() + return (returncode, output) def save_state(done_bugs): with open('batch-stabilize.state', 'wb') as state_file: pickle.dump(done_bugs, state_file)