commit: 842f3d251d81ffc274fa01a1d9ba9b5794a2d714 Author: William Hubbs <w.d.hubbs <AT> gmail <DOT> com> AuthorDate: Sat Dec 13 19:33:09 2014 +0000 Commit: Paweł Hajdan <phajdan.jr <AT> gentoo <DOT> org> CommitDate: Mon Dec 29 14:52:27 2014 +0000 URL: http://sources.gentoo.org/gitweb/?p=proj/arch-tools.git;a=commit;h=842f3d25
Port most tools to python 3 This commit ports the following tools to python 3. - common code - batch-stabilize - file-stabilization-bugs - maintainer-timeout - reverse-dependencies - stabilization-candidates This is a list of the changes: - Remove the dependency on pybugz; these tools now use xmlrpc.client directly. This is needed because the next release of pybugz does the same thing; we no longer have a bugzilla module since it was just a wrapper. Handle tokens correctly: - When we log into Bugzilla, we are given a token we must pass back to it with every command we issue. - Log out at the end of every session to expire the token. Fix the state file processing: - The pickle module handles binary files, not text files, so read and write the state file in binary mode. - Add fix_imports=True when reading the state file so a python-2 compatible state file can be read. - Do not set the URL for a stabilization request X-Gentoo-Bug: 532054 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=532054 X-Gentoo-Bug: 532368 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=532368 --- batch-stabilize.py | 43 ++++++++++++++++++++---------------- common.py | 4 ++-- file-stabilization-bugs.py | 42 ++++++++++++++++++----------------- maintainer-timeout.py | 31 ++++++++++++++++++-------- reverse-dependencies.py | 6 ++--- stabilization-candidates.py | 54 ++++++++++++++++++++++++++------------------- 6 files changed, 104 insertions(+), 76 deletions(-) diff --git a/batch-stabilize.py b/batch-stabilize.py index 053cd19..155af38 100755 --- a/batch-stabilize.py +++ b/batch-stabilize.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -11,8 +11,8 @@ import re import shutil import subprocess import sys +import xmlrpc.client -from bugz.bugzilla import BugzillaProxy from common import login import portage.versions @@ -20,7 +20,7 @@ BUG_REGEX = re.compile("[Bb]ug #?(\d+)") def print_and_log(message, log): try: - print message + print(message) log.write(message + '\n') finally: log.flush() @@ -40,7 +40,7 @@ def run_command(args, cwd, log): log.flush() def save_state(done_bugs): - with open('batch-stabilize.state', 'w') as state_file: + with open('batch-stabilize.state', 'wb') as state_file: pickle.dump(done_bugs, state_file) if __name__ == "__main__": @@ -62,12 +62,12 @@ if __name__ == "__main__": done_bugs = [] if os.path.exists('batch-stabilize.state'): - with open('batch-stabilize.state', 'r') as state_file: - done_bugs = pickle.load(state_file) + with open('batch-stabilize.state', 'rb') as state_file: + done_bugs = pickle.load(state_file, fix_imports=True) url = 'https://bugs.gentoo.org/xmlrpc.cgi' - print 'You will be prompted for your Gentoo Bugzilla username and password (%s).' % url - bugzilla = BugzillaProxy(url) + print('You will be prompted for your Gentoo Bugzilla username and password (%s).' % url) + bugzilla = xmlrpc.client.ServerProxy(url) user, login_data = login(bugzilla) with open(options.input_filename, "r") as input_file: @@ -83,14 +83,14 @@ if __name__ == "__main__": if line.startswith("#"): match = BUG_REGEX.search(line, re.IGNORECASE) if not match: - print 'Ignoring comment line [%s]...' % line + print('Ignoring comment line [%s]...' % line) continue else: bug_id = int(match.group(1)) continue if bug_id == -1: - print 'Could not recognize bug id' + print('Could not recognize bug id') sys.exit(1) # Drop the leading '='. @@ -109,10 +109,10 @@ if __name__ == "__main__": for (pn, ebuild_name) in stabilization_dict[bug_id]: ebuild_path = os.path.join(options.repo, pn, ebuild_name) if not os.path.exists(ebuild_path): - print '%s: file does not exist' % ebuild_path + print('%s: file does not exist' % ebuild_path) success = False if not success: - print 'Sanity check failed. Please make sure your CVS repo is up to date (cvs up).' + print('Sanity check failed. Please make sure your CVS repo is up to date (cvs up).') sys.exit(1) with open('batch-stabilize.log', 'a') as log_file: @@ -132,20 +132,20 @@ if __name__ == "__main__": try: shutil.rmtree(cvs_path) except OSError: - print '!!! rmtree %s failed' % cvs_path + print('!!! rmtree %s failed' % cvs_path) sys.exit(1) if run_command(["cvs", "up", pn], options.repo, log_file)[0] != 0: - print '!!! cvs up failed' + print('!!! cvs up failed') sys.exit(1) for (pn, ebuild_name) in stabilization_dict[bug_id]: cvs_path = os.path.join(options.repo, pn) print_and_log('Working in %s...' % cvs_path, log_file) if run_command(["ekeyword", options.arch, ebuild_name], cvs_path, log_file)[0] != 0: - print '!!! ekeyword failed' + print('!!! ekeyword failed') sys.exit(1) if run_command(["repoman", "manifest"], cvs_path, log_file)[0] != 0: - print '!!! repoman manifest failed' + print('!!! repoman manifest failed') sys.exit(1) for (pn, ebuild_name) in stabilization_dict[bug_id]: cvs_path = os.path.join(options.repo, pn) @@ -160,12 +160,13 @@ if __name__ == "__main__": if run_command(["echangelog", commit_message], cvs_path, log_file)[0] != 0: print_and_log('echangelog failed, maybe just the Manifest is being updated; continuing', log_file) if run_command(["repoman", "manifest"], cvs_path, log_file)[0] != 0: - print '!!! repoman manifest failed' + print('!!! repoman manifest failed') sys.exit(1) if run_command(["repoman", "commit", "--ignore-arches", "-m", commit_message], cvs_path, log_file)[0] != 0: - print '!!! repoman commit failed' + print('!!! repoman commit failed') sys.exit(1) params = {} + params['Bugzilla_token'] = login_data['token'] params['ids'] = [bug_id] bug_xml = bugzilla.Bug.get(params)['bugs'][0] has_my_arch = False @@ -188,7 +189,7 @@ if __name__ == "__main__": # We don't close bugs which still have other arches for obvious reasons, # and security bugs because stabilization is not the last step for them. params = {} - params['token'] = login_data['token'] + params['Bugzilla_token'] = login_data['token'] params['ids'] = [bug_id] params['cc'] = {} params['cc']['remove'] = ['%s...@gentoo.org' % options.arch] @@ -209,3 +210,7 @@ if __name__ == "__main__": if os.path.exists('batch-stabilize.state'): os.remove('batch-stabilize.state') + + params = {} + params['Bugzilla_token'] = login_data['token'] + bugzilla.User.logout(params) diff --git a/common.py b/common.py index 578b0e0..89e4996 100644 --- a/common.py +++ b/common.py @@ -1,7 +1,7 @@ # Copyright 2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import cStringIO +import io import datetime import getpass import re @@ -53,7 +53,7 @@ def detect_cpvs(text): def get_input(prompt): - return raw_input(prompt) + return input(prompt) def login(bugzilla): diff --git a/file-stabilization-bugs.py b/file-stabilization-bugs.py index 1c5bbe9..f9b170c 100755 --- a/file-stabilization-bugs.py +++ b/file-stabilization-bugs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -11,16 +11,14 @@ import re import shutil import subprocess import sys -import urllib -import xmlrpclib +import xmlrpc.client -from bugz.bugzilla import BugzillaProxy from common import login import portage.versions from portage.xml.metadata import MetaDataXML def save_state(done_cpvs): - with open('file-stabilization-bugs.state', 'w') as state_file: + with open('file-stabilization-bugs.state', 'wb') as state_file: pickle.dump(done_cpvs, state_file) if __name__ == "__main__": @@ -40,13 +38,13 @@ if __name__ == "__main__": done_cpvs = [] if os.path.exists('file-stabilization-bugs.state'): - with open('file-stabilization-bugs.state', 'r') as state_file: - done_cpvs = pickle.load(state_file) + with open('file-stabilization-bugs.state', 'rb') as state_file: + done_cpvs = pickle.load(state_file, fix_imports=True) url = 'https://bugs.gentoo.org/xmlrpc.cgi' - print 'You will be prompted for your Gentoo Bugzilla username and password (%s).' % url - bugzilla = BugzillaProxy(url) - login(bugzilla) + print('You will be prompted for your Gentoo Bugzilla username and password (%s).' % url) + bugzilla = xmlrpc.client.ServerProxy(url) + user, login_data = login(bugzilla) with open(options.input_filename, "r") as input_file: for line in input_file: @@ -58,7 +56,7 @@ if __name__ == "__main__": cpv = line if cpv in done_cpvs: - print 'Skipping %s because it\'s marked as done' % cpv + print('Skipping %s because it\'s marked as done' % cpv) continue cp = portage.versions.pkgsplit(cpv)[0] @@ -75,37 +73,41 @@ if __name__ == "__main__": description = ('Is it OK to stabilize =%s ?\n' % cpv + 'If so, please CC all arches which have stable keywords\n' + 'for older versions of this package.') - url = 'http://packages.gentoo.org/package/%s?arches=linux' % urllib.quote(cp) params = {} + params['Bugzilla_token'] = login_data['token'] params['product'] = 'Gentoo Linux' params['version'] = 'unspecified' params['component'] = 'Keywording and Stabilization' params['summary'] = '%s: stabilization request' % cpv params['description'] = description - params['url'] = url params['assigned_to'] = maintainer params['cc'] = other_maintainers params['severity'] = 'enhancement' try: bug_id = bugzilla.Bug.create(params)['id'] - print 'Submitted bug #%d for %s. ;-)' % (bug_id, cpv) + print('Submitted bug #%d for %s. ;-)' % (bug_id, cpv)) done_cpvs += cpv save_state(done_cpvs) try: params = {} + params['Bugzilla_token'] = login_data['token'] params['ids'] = [bug_id] params['keywords'] = {'set': 'STABLEREQ'} bugzilla.Bug.update(params) - except xmlrpclib.Fault, f: + except xmlrpc.client.Fault as f: exit_code = 1 - print f - print 'Failed to add STABLEREQ keyword for %s. :-/' % cpv - except xmlrpclib.Fault, f: + print(f) + print('Failed to add STABLEREQ keyword for %s. :-/' % cpv) + except xmlrpc.client.Fault as f: exit_code = 1 - print f - print 'Failed to submit bug for %s. :-(' % cpv + print(f) + print('Failed to submit bug for %s. :-(' % cpv) if exit_code == 0 and os.path.exists('file-stabilization-bugs.state'): os.remove('file-stabilization-bugs.state') + params = {} + params['Bugzilla_token'] = login_data['token'] + bugzilla.User.logout(params) + sys.exit(exit_code) diff --git a/maintainer-timeout.py b/maintainer-timeout.py index 76c7e59..54b4044 100755 --- a/maintainer-timeout.py +++ b/maintainer-timeout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -6,11 +6,11 @@ import datetime import optparse import os.path import sys +import xmlrpc.client -from bugz.bugzilla import BugzillaProxy import portage.versions -from common import Bug, chunks, login, detect_cpvs +from common import login, detect_cpvs def keyword_to_email(keyword): @@ -27,12 +27,20 @@ if __name__ == "__main__": parser.error("unrecognized command-line args") url = 'https://bugs.gentoo.org/xmlrpc.cgi' - print 'You will be prompted for your Gentoo Bugzilla username and password (%s).' % url - bugzilla = BugzillaProxy(url) + print('You will be prompted for your Gentoo Bugzilla username and password (%s).' % url) + bugzilla = xmlrpc.client.ServerProxy(url) user, login_data = login(bugzilla) - bugs = bugzilla.Bug.search({'reporter': user, 'summary': ['stabilize', 'stabilization', 'stable'], 'resolution': ''})['bugs'] - comments = bugzilla.Bug.comments({'ids': [bug['id'] for bug in bugs]}) + params = {} + params['Bugzilla_token'] = login_data['token'] + params['reporter'] = user + params['summary'] = ['stabilize', 'stabilization', 'stable'] + params['resolution'] = '' + bugs = bugzilla.Bug.search(params)['bugs'] + params = {} + params['Bugzilla_token'] = login_data['token'] + params['ids'] = [bug['id'] for bug in bugs] + comments = bugzilla.Bug.comments(params) for bug in bugs: # Skip bugs where stabilization seems to be already in progress. arch_found = False @@ -68,6 +76,7 @@ if __name__ == "__main__": target_keywords.add(keyword) params = {} + params['Bugzilla_token'] = login_data['token'] params['ids'] = [bug['id']] params['cc'] = {} params['cc']['add'] = list(set(keyword_to_email(k) for k in target_keywords)) @@ -75,7 +84,11 @@ if __name__ == "__main__": params['comment']['body'] = 'Maintainer timeout (%d days). Arches please go ahead.' % options.days bugzilla.Bug.update(params) - print 'Updated bug #%s (%s). Target KEYWORDS: %s ;-)' % ( + print('Updated bug #%s (%s). Target KEYWORDS: %s ;-)' % ( bug['id'], bug['summary'], - ', '.join(list(target_keywords))) + ', '.join(list(target_keywords)))) + + params = {} + params['Bugzilla_token'] = login_data['token'] + bugzilla.User.logout(params) diff --git a/reverse-dependencies.py b/reverse-dependencies.py index 96233ea..624b668 100755 --- a/reverse-dependencies.py +++ b/reverse-dependencies.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -16,7 +16,7 @@ def process_deps(deps, use=None): use = [] result = [] for index, x in enumerate(deps): - if type(x) == types.ListType: + if type(x) == list: continue elif x == "||": # Any-of. result.extend(process_deps(deps[index + 1], use)) @@ -82,4 +82,4 @@ if __name__ == "__main__": output_file.write("#\t%s\n" % ",".join(use_combination)) output_file.write("%s\n" % cp) elif options.verbose: - print 'No reverse dependencies for %s' % pn + print('No reverse dependencies for %s' % pn) diff --git a/stabilization-candidates.py b/stabilization-candidates.py index 52c88f1..d324bdb 100755 --- a/stabilization-candidates.py +++ b/stabilization-candidates.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -8,10 +8,8 @@ import os.path import random import re import subprocess -import xmlrpclib -import urllib +import xmlrpc.client -from bugz.bugzilla import BugzillaProxy from portage.package.ebuild.getmaskingstatus import getmaskingstatus from portage.xml.metadata import MetaDataXML import portage.versions @@ -36,9 +34,9 @@ if __name__ == "__main__": parser.error("unrecognized command-line args") url = 'https://bugs.gentoo.org/xmlrpc.cgi' - print 'You will be prompted for your Gentoo Bugzilla username and password (%s).' % url - bugzilla = BugzillaProxy(url) - login(bugzilla) + print('You will be prompted for your Gentoo Bugzilla username and password (%s).' % url) + bugzilla = xmlrpc.client.ServerProxy(url) + user, login_data = login(bugzilla) final_candidates = [] now = datetime.datetime.now() @@ -77,7 +75,7 @@ if __name__ == "__main__": best_stable = portage.versions.best(portage.portdb.match(cp)) if not best_stable: continue - print 'Working on %s...' % cp, + print('Working on %s...' % cp, end=' ') candidates = [] for cpv in portage.portdb.cp_list(cp): # Only consider higher versions than best stable. @@ -100,12 +98,12 @@ if __name__ == "__main__": continue # Eliminate hard masked packages among others. - if getmaskingstatus(cpv) not in [[u'~%s keyword' % arch] for arch in options.arch]: + if getmaskingstatus(cpv) not in [['~%s keyword' % arch] for arch in options.arch]: continue candidates.append(cpv) if not candidates: - print 'no candidates' + print('no candidates') continue candidates.sort(key=portage.versions.cpv_sort_key()) @@ -122,14 +120,14 @@ if __name__ == "__main__": regex = '\*%s \((.*)\)' % re.escape(pv) match = re.search(regex, changelog_file.read()) if not match: - print 'error parsing ChangeLog' + print('error parsing ChangeLog') continue changelog_date = datetime.datetime.strptime(match.group(1), '%d %b %Y') if now - changelog_date < datetime.timedelta(days=options.days): - print 'not old enough' + print('not old enough') continue - except IOError, e: - print e + except IOError as e: + print(e) continue keywords = portage.db["/"]["porttree"].dbapi.aux_get(best_candidate, ['KEYWORDS'])[0] @@ -139,19 +137,25 @@ if __name__ == "__main__": missing_arch = True break if missing_arch: - print 'not keyworded ~arch' + print('not keyworded ~arch') continue # Do not risk trying to stabilize a package with known bugs. - bugs = [x for x in bugzilla.Bug.search({'summary': cp})['bugs'] if x['is_open'] and x['severity'] not in ['enhancement', 'QA']] + params = {} + params['Bugzilla_token'] = login_data['token'] + params['summary'] = cp + bugs = [x for x in bugzilla.Bug.search(params)['bugs'] if x['is_open'] and x['severity'] not in ['enhancement', 'QA']] if bugs: - print 'has bugs' + print('has bugs') continue # Protection against filing a stabilization bug twice. - bugs = bugzilla.Bug.search({'summary': best_candidate})['bugs'] + params = {} + params['Bugzilla_token'] = login_data['token'] + params['summary'] = best_candidate + bugs = bugzilla.Bug.search(params)['bugs'] if bugs: - print 'version has bugs' + print('version has bugs') continue ebuild_name = portage.versions.catsplit(best_candidate)[1] + ".ebuild" @@ -160,8 +164,8 @@ if __name__ == "__main__": try: original_contents = open(ebuild_path).read() manifest_contents = open(manifest_path).read() - except IOError, e: - print e + except IOError as e: + print(e) continue try: for arch in options.arch: @@ -169,7 +173,7 @@ if __name__ == "__main__": subprocess.check_output(["repoman", "manifest"], cwd=cvs_path) subprocess.check_output(["repoman", "full"], cwd=cvs_path) except subprocess.CalledProcessError: - print 'repoman error' + print('repoman error') continue finally: f = open(ebuild_path, "w") @@ -182,4 +186,8 @@ if __name__ == "__main__": with open(options.output_filename, 'a') as f: f.write('# %s %s\n' % (maintainer, ', '.join(other_maintainers))) f.write('%s\n' % best_candidate) - print (best_candidate, maintainer, other_maintainers) + print((best_candidate, maintainer, other_maintainers)) + + params = {} + params['Bugzilla_token'] = login_data['token'] + bugzilla.User.logout(params)
