Hello community, here is the log from the commit of package openSUSE-release-tools for openSUSE:Factory checked in at 2018-07-21 10:25:19 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old) and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openSUSE-release-tools" Sat Jul 21 10:25:19 2018 rev:114 rq:624221 version:20180720.7920baa Changes: -------- --- /work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes 2018-07-19 23:01:53.232667934 +0200 +++ /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new/openSUSE-release-tools.changes 2018-07-21 10:25:20.910955619 +0200 @@ -2 +2,24 @@ -Thu Jul 19 06:15:37 UTC 2018 - opensuse-packag...@opensuse.org +Fri Jul 20 08:10:35 UTC 2018 - opensuse-releaset...@opensuse.org + +- Update to version 20180720.7920baa: + * pkglistgen: Commit a summary.yml into 000product-summary if existant + * pkglist: Review the proper package (000release-packages) + * pkglistgen: allow to pass staging:a as scope + * pkglistgen: remove default-support-status option + +------------------------------------------------------------------- +Thu Jul 19 20:06:19 UTC 2018 - opensuse-releaset...@opensuse.org + +- Update to version 20180719.9951b45: + * Move expanded_repos into staging api + * repo_checker: Expand repositories of target project + * repo_checker: Fix the review loop + +------------------------------------------------------------------- +Thu Jul 19 14:18:48 UTC 2018 - opensuse-releaset...@opensuse.org + +- Update to version 20180719.c9030ef: + * Revert "dist/ci/Dockerfile: add python2-pyOpenSSL as dependency." + +------------------------------------------------------------------- +Thu Jul 19 06:15:37 UTC 2018 - opensuse-releaset...@opensuse.org Old: ---- openSUSE-release-tools-20180718.2f192f1.obscpio New: ---- openSUSE-release-tools-20180720.7920baa.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openSUSE-release-tools.spec ++++++ --- /var/tmp/diff_new_pack.Ndad1c/_old 2018-07-21 10:25:21.378955506 +0200 +++ /var/tmp/diff_new_pack.Ndad1c/_new 2018-07-21 10:25:21.378955506 +0200 @@ -20,7 +20,7 @@ %define source_dir openSUSE-release-tools %define announcer_filename factory-package-news Name: openSUSE-release-tools -Version: 20180718.2f192f1 +Version: 20180720.7920baa Release: 0 Summary: Tools to aid in staging and release work for openSUSE/SUSE License: GPL-2.0-or-later AND MIT ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.Ndad1c/_old 2018-07-21 10:25:21.406955499 +0200 +++ /var/tmp/diff_new_pack.Ndad1c/_new 2018-07-21 10:25:21.410955499 +0200 @@ -1,6 +1,6 @@ <servicedata> <service name="tar_scm"> <param name="url">https://github.com/openSUSE/openSUSE-release-tools.git</param> - <param name="changesrevision">56b6fe1fa578c18f5cc753687037fee9f81d0998</param> + <param name="changesrevision">fb114a75d437351b61a849b746d82b7cbbbb871a</param> </service> -</servicedata> \ No newline at end of file +</servicedata> ++++++ openSUSE-release-tools-20180718.2f192f1.obscpio -> openSUSE-release-tools-20180720.7920baa.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180718.2f192f1/dist/ci/Dockerfile new/openSUSE-release-tools-20180720.7920baa/dist/ci/Dockerfile --- old/openSUSE-release-tools-20180718.2f192f1/dist/ci/Dockerfile 2018-07-18 11:39:21.000000000 +0200 +++ new/openSUSE-release-tools-20180720.7920baa/dist/ci/Dockerfile 2018-07-20 09:59:59.000000000 +0200 @@ -12,7 +12,6 @@ obs-service-* \ osc \ python-packaging \ - python2-pyOpenSSL \ sudo # `osc build` directories that are effective to cache: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180718.2f192f1/osclib/stagingapi.py new/openSUSE-release-tools-20180720.7920baa/osclib/stagingapi.py --- old/openSUSE-release-tools-20180718.2f192f1/osclib/stagingapi.py 2018-07-18 11:39:21.000000000 +0200 +++ new/openSUSE-release-tools-20180720.7920baa/osclib/stagingapi.py 2018-07-20 09:59:59.000000000 +0200 @@ -1826,3 +1826,16 @@ return meta.find(xpath) is not None return False + + # recursively detect underlying projects + def expand_project_repo(self, project, repo, repos): + repos.append([project, repo]) + url = self.makeurl(['source', project, '_meta']) + meta = ET.parse(self.retried_GET(url)).getroot() + for path in meta.findall('.//repository[@name="{}"]/path'.format(repo)): + self.expand_project_repo(path.get('project', project), path.get('repository'), repos) + return repos + + def expanded_repos(self, repo): + return self.expand_project_repo(self.project, repo, []) + diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180718.2f192f1/pkglistgen.py new/openSUSE-release-tools-20180720.7920baa/pkglistgen.py --- old/openSUSE-release-tools-20180718.2f192f1/pkglistgen.py 2018-07-18 11:39:21.000000000 +0200 +++ new/openSUSE-release-tools-20180720.7920baa/pkglistgen.py 2018-07-20 09:59:59.000000000 +0200 @@ -33,7 +33,7 @@ import urllib2 import filecmp from osc.core import checkout_package -from osc.core import http_GET +from osc.core import http_GET, http_PUT from osc.core import makeurl from osc.core import Package from osc.core import show_results_meta @@ -89,6 +89,7 @@ self.solved = False self.not_found = dict() self.unresolvable = dict() + self.default_support_status = None for a in ARCHITECTURES: self.packages[a] = [] self.unresolvable[a] = dict() @@ -388,7 +389,7 @@ else: logger.error(msg) name = msg - status = self.pkglist.supportstatus(name) + status = self.pkglist.supportstatus(name) or self.default_support_status attrs = {'name': name} if status is not None: attrs['supportstatus'] = status @@ -399,6 +400,13 @@ return root + # just list all packages in it as an array - to be output as one yml + def summary(self): + ret = set() + for arch in ['*'] + self.architectures: + ret |= set(self.solved_packages[arch].keys()) + return ret + def dump(self): pprint({'name': self.name, 'missing': self.missing, 'packages': self.packages, 'solved': self.solved_packages, 'silents': self.silents}) @@ -415,7 +423,6 @@ ToolBase.ToolBase.__init__(self) # package -> supportatus self.packages = dict() - self.default_support_status = 'l3' self.groups = dict() self._supportstatus = None self.input_dir = '.' @@ -428,14 +435,6 @@ self.output = None self.locales = set() - def _dump_supportstatus(self): - for name in self.packages.keys(): - for status in self.packages[name]: - if status == self.default_support_status: - continue - for group in self.packages[name][status]: - print(name, status) - def _load_supportstatus(self): # XXX fn = os.path.join(self.input_dir, 'supportstatus.txt') @@ -448,15 +447,11 @@ if len(a) > 1: self._supportstatus[a[0]] = a[1] - # TODO: make per product def supportstatus(self, package): if self._supportstatus is None: self._load_supportstatus() - if package in self._supportstatus: - return self._supportstatus[package] - else: - return self.default_support_status + return self._supportstatus.get(package) def _load_group_file(self, fn): output = None @@ -486,11 +481,13 @@ def _write_all_groups(self): self._check_supplements() + summary = dict() archs = ['*'] + self.architectures for name in self.groups: group = self.groups[name] if not group.solved: continue + summary[name] = group.summary() fn = '{}.group'.format(group.name) with open(os.path.join(self.output_dir, fn), 'w') as fh: comment = group.comment @@ -502,6 +499,7 @@ x = re.sub('\s*<!-- reason:', ' <!-- reason:', x) # fh.write(ET.tostring(x, pretty_print = True, doctype = '<?xml version="1.0" encoding="UTF-8"?>')) fh.write(x) + return summary def _parse_product(self, root): print(root.find('.//products/product/name').text) @@ -526,16 +524,8 @@ for e in excludes: g.ignore(self.groups[e]) - def expand_project_repo(self, project, repo, repos): - repos.append([project, repo]) - url = makeurl(self.apiurl, ['source', project, '_meta']) - meta = ET.parse(http_GET(url)).getroot() - for path in meta.findall('.//repository[@name="{}"]/path'.format(repo)): - self.expand_project_repo(path.get('project', project), path.get('repository'), repos) - return repos - def expand_repos(self, project, repo): - return self.expand_project_repo(project, repo, []) + return StagingAPI(self.apiurl, project).expanded_repos('standard') def _check_supplements(self): tocheck = set() @@ -696,7 +686,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): - SCOPES = ['all', 'arm', 'target', 'rings', 'staging', 'ports'] + SCOPES = ['all', 'arm', 'target', 'rings', 'ports', 'staging'] def __init__(self, *args, **kwargs): ToolBase.CommandLineInterface.__init__(self, args, kwargs) @@ -710,15 +700,12 @@ help='input directory', default='.') parser.add_option('-a', '--architecture', dest='architectures', metavar='ARCH', help='architecure', action='append') - parser.add_option('--default-support-status', dest='default_support_status', metavar='STATUS', - help='default support status', default=None) return parser def setup_tool(self): tool = PkgListGen() tool.input_dir = self.options.input_dir tool.output_dir = self.options.output_dir - tool.default_support_status = self.options.default_support_status tool.repos = self.repos if self.options.architectures: tool.architectures = self.options.architectures @@ -738,17 +725,6 @@ for name in sorted(self.tool.groups.keys()): print(name) - # to be called only once to bootstrap - def do_dump_supportstatus(self, subcmd, opts): - """${cmd_name}: dump supportstatus of input files - - ${cmd_usage} - ${cmd_option_list} - """ - - self.tool.load_all_groups() - self.tool._dump_supportstatus() - def do_list_products(self, subcmd, opts): """${cmd_name}: list all products @@ -1119,6 +1095,7 @@ self.tool.solve_module(groupname, includes, excludes) g = self.tool.groups[groupname] g.conflicts = settings.get('conflicts', []) + g.default_support_status = settings.get('default-support', 'unsupported') modules.append(g) # not defined for openSUSE @@ -1142,11 +1119,11 @@ module.solved_packages[arch].pop(p, None) self.tool._collect_unsorted_packages(modules, self.tool.groups.get('unsorted')) - self.tool._write_all_groups() + return self.tool._write_all_groups() @cmdln.option('-f', '--force', action='store_true', help='continue even if build is in progress') @cmdln.option('-p', '--project', help='target project') - @cmdln.option('-s', '--scope', default='all', help='scope on which to operate ({})'.format(', '.join(SCOPES))) + @cmdln.option('-s', '--scope', default='all', help='scope on which to operate ({}, staging:$letter)'.format(', '.join(SCOPES))) @cmdln.option('--no-checkout', action='store_true', help='reuse checkout in cache') @cmdln.option('--stop-after-solve', action='store_true', help='only create group files') def do_update_and_solve(self, subcmd, opts): @@ -1160,6 +1137,12 @@ if not opts.project: raise ValueError('project is required') + opts.staging_project = None + if opts.scope.startswith('staging:'): + opts.staging_project = re.match('staging:(.*)', opts.scope).group(1) + opts.staging_project = opts.staging_project.upper() + opts.scope = 'staging' + if opts.scope not in self.SCOPES: raise ValueError('scope must be one of: {}'.format(', '.join(self.SCOPES))) @@ -1191,7 +1174,7 @@ if opts.scope == 'target': self.repos = self.tool.expand_repos(target_project, main_repo) - self.update_and_solve_target_wrapper(apiurl, target_project, target_config, main_repo, opts, drop_list=True) + self.update_and_solve_target_wrapper(api, target_project, target_config, main_repo, opts, drop_list=True) return self.error_occured elif opts.scope == 'arm': main_repo = 'ports' @@ -1204,19 +1187,21 @@ main_repo = 'ports' opts.project += ':Ports' self.repos = self.tool.expand_repos(opts.project, main_repo) - self.update_and_solve_target_wrapper(apiurl, target_project, target_config, main_repo, opts, drop_list=True) + self.update_and_solve_target_wrapper(api, target_project, target_config, main_repo, opts, drop_list=True) return self.error_occured elif opts.scope == 'rings': opts.project = api.rings[1] self.repos = self.tool.expand_repos(api.rings[1], main_repo) - self.update_and_solve_target_wrapper(apiurl, target_project, target_config, main_repo, opts) + self.update_and_solve_target_wrapper(api, target_project, target_config, main_repo, opts) return self.error_occured elif opts.scope == 'staging': letters = api.get_staging_projects_short() for letter in letters: + if opts.staging_project and letter != opts.staging_project: + continue opts.project = api.prj_from_short(letter) self.repos = self.tool.expand_repos(opts.project, main_repo) - self.update_and_solve_target_wrapper(apiurl, target_project, target_config, main_repo, opts) + self.update_and_solve_target_wrapper(api, target_project, target_config, main_repo, opts) return self.error_occured def update_and_solve_target_wrapper(self, *args, **kwargs): @@ -1231,7 +1216,7 @@ traceback.print_exc() self.error_occured = True - def update_and_solve_target(self, apiurl, target_project, target_config, main_repo, opts, + def update_and_solve_target(self, api, target_project, target_config, main_repo, opts, skip_release=False, drop_list=False): print('[{}] {}/{}: update and solve'.format(opts.scope, opts.project, main_repo)) @@ -1239,16 +1224,16 @@ product = target_config.get('pkglistgen-product', '000product') release = target_config.get('pkglistgen-release', '000release-packages') - url = makeurl(apiurl, ['source', opts.project]) + url = api.makeurl(['source', opts.project]) packages = ET.parse(http_GET(url)).getroot() if packages.find('entry[@name="{}"]'.format(product)) is None: if not self.options.dry: - undelete_package(apiurl, opts.project, product, 'revive') + undelete_package(api.apiurl, opts.project, product, 'revive') # TODO disable build. print('{} undeleted, skip dvd until next cycle'.format(product)) return elif not opts.force: - root = ET.fromstringlist(show_results_meta(apiurl, opts.project, product, + root = ET.fromstringlist(show_results_meta(api.apiurl, opts.project, product, repository=[main_repo], multibuild=True)) if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')): print('{}/{} build in progress'.format(opts.project, product)) @@ -1260,12 +1245,12 @@ if packages.find('entry[@name="{}"]'.format(release)) is None: if not self.options.dry: - undelete_package(apiurl, opts.project, product, 'revive') + undelete_package(api.apiurl, opts.project, release, 'revive') print('{} undeleted, skip dvd until next cycle'.format(release)) return # Cache dir specific to hostname and project. - host = urlparse.urlparse(apiurl).hostname + host = urlparse.urlparse(api.apiurl).hostname cache_dir = save_cache_path('opensuse-packagelists', host, opts.project) if not opts.no_checkout: @@ -1281,7 +1266,7 @@ if opts.no_checkout: print("Skipping checkout of {}/{}".format(opts.project, package)) continue - checkout_package(apiurl, opts.project, package, expand_link=True, prj_dir=cache_dir) + checkout_package(api.apiurl, opts.project, package, expand_link=True, prj_dir=cache_dir) if not skip_release: self.unlink_all_except(release_dir) @@ -1321,7 +1306,8 @@ opts.include_suggested = str2bool(target_config.get('pkglistgen-include-suggested')) opts.locale = target_config.get('pkglistgen-local') opts.locales_from = target_config.get('pkglistgen-locales-from') - self.do_solve('solve', opts) + summary = self.do_solve('solve', opts) + if opts.stop_after_solve: return @@ -1332,7 +1318,7 @@ family_last = target_config.get('pkglistgen-product-family-last') family_include = target_config.get('pkglistgen-product-family-include') solv_prior = self.solv_cache_update( - apiurl, cache_dir_solv, target_project, family_last, family_include, opts) + api.apiurl, cache_dir_solv, target_project, family_last, family_include, opts) # Include pre-final release solv files for target project. These # files will only exist from previous runs. @@ -1376,6 +1362,16 @@ self.build_stub(release_dir, 'spec') self.commit_package(release_dir) + if api.item_exists(opts.project, '000product-summary'): + summary_str = "# Summary of packages in groups" + for group in sorted(summary.keys()): + summary_str += "\n" + group + ":\n" + for package in sorted(summary[group]): + summary_str += " - " + package + "\n" + + url = api.makeurl(['source', opts.project, '000product-summary', 'summary.yml']) + http_PUT(url, data=summary_str) + def solv_cache_update(self, apiurl, cache_dir_solv, target_project, family_last, family_include, opts): """Dump solv files (do_dump_solv) for all products in family.""" prior = set() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20180718.2f192f1/repo_checker.py new/openSUSE-release-tools-20180720.7920baa/repo_checker.py --- old/openSUSE-release-tools-20180718.2f192f1/repo_checker.py 2018-07-18 11:39:21.000000000 +0200 +++ new/openSUSE-release-tools-20180720.7920baa/repo_checker.py 2018-07-20 09:59:59.000000000 +0200 @@ -164,12 +164,9 @@ # Not in a "ready" state. openQA_only = False # Not relevant so set to False. if status and str(status['overall_state']) == 'failed': - # Exception to the rule is openQA only in failed state. - openQA_only = True - if len(status['broken_packages']): - # Broken packages so not just openQA. - openQA_only = False - break + # Exception to the rule is openQA only in failed state, + # Broken packages so not just openQA. + openQA_only = (len(status['broken_packages']) == 0) if not self.force and not openQA_only: self.logger.debug('{}: {} not ready'.format(request.reqid, group)) @@ -257,7 +254,11 @@ continue # Only bother if staging can match arch, but layered first. - directories.insert(0, self.mirror(project, arch)) + repos = self.staging_api(project).expanded_repos('standard') + for layered_project, repo in reversed(repos): + if repo != 'standard': + raise "We assume all is standard" + directories.insert(0, self.mirror(layered_project, arch)) whitelist = self.binary_whitelist(project, arch, group) ++++++ openSUSE-release-tools.obsinfo ++++++ --- /var/tmp/diff_new_pack.Ndad1c/_old 2018-07-21 10:25:22.058955341 +0200 +++ /var/tmp/diff_new_pack.Ndad1c/_new 2018-07-21 10:25:22.058955341 +0200 @@ -1,5 +1,5 @@ name: openSUSE-release-tools -version: 20180718.2f192f1 -mtime: 1531906761 -commit: 2f192f1240ef413f200b620210e0ce80eb9d3af0 +version: 20180720.7920baa +mtime: 1532073599 +commit: 7920baacc7bdfbfeb9f3c6b40910166e5a1980fd