Hello community, here is the log from the commit of package openSUSE-release-tools for openSUSE:Factory checked in at 2018-01-02 16:35:44 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old) and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openSUSE-release-tools" Tue Jan 2 16:35:44 2018 rev:36 rq:561094 version:20180102.98a659c Changes: -------- --- /work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes 2017-12-29 18:51:04.442062391 +0100 +++ /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new/openSUSE-release-tools.changes 2018-01-02 16:35:50.539375711 +0100 @@ -1,0 +2,17 @@ +Tue Jan 02 10:50:33 UTC 2018 - opensuse-releaset...@opensuse.org + +- Update to version 20180102.98a659c: + * manager42: ignore meta packages + * manager42: refactor + * manager42: don't turn dropped packages into fork + * manager42: warn about inconsistent packages + * reochecker: avoid accessing empty status + * repo_checker: only update comment if published or message changed. + * ReviewBot: comment_write(): provide info_extra_identical parameter. + * fcc_submitter: handle multibuild package in build_succeeded package list + * check_duplicate_binaries: better yaml + * pkglistgen: migrate bash scripts to python. + * osclib/stagingapi: provide is_staging_bootstrapped() from request_splitter. + * ToolBase: provide -d alias for --debug for consistency. + +------------------------------------------------------------------- Old: ---- openSUSE-release-tools-20171226.96a3eec.obscpio New: ---- openSUSE-release-tools-20180102.98a659c.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openSUSE-release-tools.spec ++++++ --- /var/tmp/diff_new_pack.qsXQKW/_old 2018-01-02 16:35:51.599022346 +0100 +++ /var/tmp/diff_new_pack.qsXQKW/_new 2018-01-02 16:35:51.603021012 +0100 @@ -1,7 +1,7 @@ # # spec file for package openSUSE-release-tools # -# Copyright (c) 2017 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2018 SUSE LINUX GmbH, Nuernberg, Germany. # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -20,7 +20,7 @@ %define source_dir osc-plugin-factory %define announcer_filename factory-package-news Name: openSUSE-release-tools -Version: 20171226.96a3eec +Version: 20180102.98a659c Release: 0 Summary: Tools to aid in staging and release work for openSUSE/SUSE License: GPL-2.0+ and MIT @@ -483,11 +483,6 @@ %files pkglistgen %defattr(-,root,root,-) %{_bindir}/osrt-pkglistgen -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0 -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0-all -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Rings -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Staging -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh %{_unitdir}/osrt-pkglistgen@.service %{_unitdir}/osrt-pkglistgen@.timer ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.qsXQKW/_old 2018-01-02 16:35:51.651005011 +0100 +++ /var/tmp/diff_new_pack.qsXQKW/_new 2018-01-02 16:35:51.651005011 +0100 @@ -1,6 +1,6 @@ <servicedata> <service name="tar_scm"> <param name="url">https://github.com/openSUSE/osc-plugin-factory.git</param> - <param name="changesrevision">96a3eec425d49f995a85e000da4a8799b5a6a461</param> + <param name="changesrevision">98a659c3fab8dc036f9945854eae93265d695bed</param> </service> </servicedata> ++++++ openSUSE-release-tools-20171226.96a3eec.obscpio -> openSUSE-release-tools-20180102.98a659c.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/Makefile new/openSUSE-release-tools-20180102.98a659c/Makefile --- old/openSUSE-release-tools-20171226.96a3eec/Makefile 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/Makefile 2018-01-02 11:39:25.000000000 +0100 @@ -9,7 +9,7 @@ leaper.py \ manager_42.py \ metrics.py \ - pkglistgen.sh \ + pkglistgen.py \ repo_checker.py \ suppkg_rebuild.py \ totest-manager.py \ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/ReviewBot.py new/openSUSE-release-tools-20180102.98a659c/ReviewBot.py --- old/openSUSE-release-tools-20171226.96a3eec/ReviewBot.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/ReviewBot.py 2018-01-02 11:39:25.000000000 +0100 @@ -436,13 +436,15 @@ def comment_write(self, state='done', result=None, project=None, package=None, request=None, message=None, identical=False, only_replace=False, - info_extra=None): + info_extra=None, info_extra_identical=True): """Write comment if not similar to previous comment and replace old one. The state, result, and info_extra (dict) are combined to create the info that is passed to CommentAPI methods for creating a marker and finding previous comments. self.bot_name, which defaults to class, will be used - as the primary matching key. + as the primary matching key. When info_extra_identical is set to False + info_extra will not be included when finding previous comments to + compare message against. A comment from the same bot will be replaced when a new comment is written. The only_replace flag will restrict to only writing a comment @@ -482,15 +484,24 @@ message = '\n\n'.join(self.comment_handler.lines) info = {'state': state, 'result': result} - if info_extra: + if info_extra and info_extra_identical: info.update(info_extra) - message = self.comment_api.add_marker(message, self.bot_name, info) - message = self.comment_api.truncate(message.strip()) comments = self.comment_api.get_comments(**kwargs) comment, _ = self.comment_api.comment_find(comments, self.bot_name, info) + + if info_extra and not info_extra_identical: + # Add info_extra once comment has already been matched. + info.update(info_extra) + + message = self.comment_api.add_marker(message, self.bot_name, info) + message = self.comment_api.truncate(message.strip()) + if (comment is not None and - ((identical and comment['comment'] == message) or + ((identical and + # Remove marker from comments since handled during comment_find(). + self.comment_api.remove_marker(comment['comment']) == + self.comment_api.remove_marker(message)) or (not identical and comment['comment'].count('\n') == message.count('\n'))) ): # Assume same state/result and number of lines in message is duplicate. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/ToolBase.py new/openSUSE-release-tools-20180102.98a659c/ToolBase.py --- old/openSUSE-release-tools-20171226.96a3eec/ToolBase.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/ToolBase.py 2018-01-02 11:39:25.000000000 +0100 @@ -140,7 +140,7 @@ parser = cmdln.Cmdln.get_optparser(self) parser.add_option("--apiurl", '-A', metavar="URL", help="api url") parser.add_option("--dry", action="store_true", help="dry run") - parser.add_option("--debug", action="store_true", help="debug output") + parser.add_option("-d", "--debug", action="store_true", help="debug output") parser.add_option("--osc-debug", action="store_true", help="osc debug output") parser.add_option("--verbose", action="store_true", help="verbose") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/dist/package/openSUSE-release-tools.spec new/openSUSE-release-tools-20180102.98a659c/dist/package/openSUSE-release-tools.spec --- old/openSUSE-release-tools-20171226.96a3eec/dist/package/openSUSE-release-tools.spec 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/dist/package/openSUSE-release-tools.spec 2018-01-02 11:39:25.000000000 +0100 @@ -483,11 +483,6 @@ %files pkglistgen %defattr(-,root,root,-) %{_bindir}/osrt-pkglistgen -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0 -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0-all -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Rings -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Staging -%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh %{_unitdir}/osrt-pkglistgen@.service %{_unitdir}/osrt-pkglistgen@.timer diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/fcc_submitter.py new/openSUSE-release-tools-20180102.98a659c/fcc_submitter.py --- old/openSUSE-release-tools-20171226.96a3eec/fcc_submitter.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/fcc_submitter.py 2018-01-02 11:39:25.000000000 +0100 @@ -184,10 +184,25 @@ root = ET.fromstring(''.join(f)) #print ET.dump(root) + failed_multibuild_pacs = [] pacs = [] for node in root.findall('result'): if node.get('repository') == 'pure_42' and node.get('arch') == 'x86_64': for pacnode in node.findall('status'): + if ':' in pacnode.get('package'): + mainpac = pacnode.get('package').split(':')[0] + if pacnode.get('code') not in ['succeeded', 'excluded']: + failed_multibuild_pacs.append(pacnode.get('package')) + if mainpac not in failed_multibuild_pacs: + failed_multibuild_pacs.append(mainpac) + if mainpac in pacs: + pacs.remove(mainpac) + else: + if mainpac in failed_multibuild_pacs: + failed_multibuild_pacs.append(pacnode.get('package')) + elif mainpac not in pacs: + pacs.append(mainpac) + continue if pacnode.get('code') == 'succeeded': pacs.append(pacnode.get('package')) else: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/manager_42.py new/openSUSE-release-tools-20180102.98a659c/manager_42.py --- old/openSUSE-release-tools-20171226.96a3eec/manager_42.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/manager_42.py 2018-01-02 11:39:25.000000000 +0100 @@ -46,7 +46,15 @@ class Manager42(object): config_defaults = { + 'ignored_packages' : [ + '00Meta', + '00aggregates', + '000product', + '000package-groups', + '000release-packages', + ], 'project_preference_order' : [], + 'drop_if_vanished_from' : [], 'from_prj' : 'openSUSE:Leap:42.3', 'factory' : 'openSUSE:Factory', } @@ -61,7 +69,7 @@ self.fill_package_meta() self.packages = dict() for project in [self.config.from_prj] + self.config.project_preference_order: - self.packages[project] = self.get_source_packages(project) + self._fill_package_list(project) # FIXME: add to ToolBase and rebase Manager42 on that def _load_config(self, handle = None): @@ -81,6 +89,9 @@ packages.add(title[3:].split(' ')[0]) return sorted(packages) + def all_packages(self): + return self.packages[self.config.from_prj] + def parse_lookup(self, project): self.lookup_changes = 0 self.lookup = {} @@ -152,11 +163,9 @@ ['source', project, package], opts)) - def crawl(self, given_packages = None): + def crawl(self, packages): """Main method of the class that runs the crawler.""" - packages = given_packages or self.packages[self.config.from_prj] - for package in sorted(packages): try: self.check_one_package(package) @@ -171,6 +180,18 @@ if self.lookup_changes: self.store_lookup() + def get_inconsistent(self): + known = set(self.lookup.keys()) + stale = known - set(self.packages[self.config.from_prj]) + unknown = set(self.packages[self.config.from_prj]) - known + + if (stale): + logger.info("stale packages: %s", ', '.join(stale)) + if (unknown): + logger.info("unknown packages: %s", ', '.join(unknown)) + + return (stale|unknown) + def get_package_history(self, project, package, deleted = False): try: query = {} @@ -183,9 +204,20 @@ return None raise - def check_source_in_project(self, project, package, verifymd5, deleted=False): + + def _is_ignored(self, project, package): + if package in self.config.ignored_packages: + logger.debug("%s in ignore list", package) + return True + return False + + def _fill_package_list(self, project): if project not in self.packages: - self.packages[project] = self.get_source_packages(project) + self.packages[project] = [ p for p in self.get_source_packages(project) if not self._is_ignored(project, p) ] + + def check_source_in_project(self, project, package, verifymd5, deleted=False): + + self._fill_package_list(project) if not deleted and not package in self.packages[project]: return None, None @@ -289,6 +321,8 @@ if not foundit: if lproject == 'FORK': logger.debug("{}: lookup is correctly marked as fork".format(package)) + elif lproject in self.config.drop_if_vanished_from: + logger.info('{} dropped from {}'.format(package, lproject)) else: logger.info('{} is a fork (was {})'.format(package, lproject)) self.lookup[package] = 'FORK' @@ -318,9 +352,15 @@ osc.conf.config['debug'] = args.debug uc = Manager42(caching = args.cache_requests, configfh = args.config ) - given_packages = args.packages - if not args.all and not given_packages: - given_packages = uc.latest_packages() + given_packages = set(args.packages) + if args.all: + given_packages = set(uc.all_packages()) + elif not given_packages: + given_packages = set(uc.latest_packages()) + + if args.check_inconsistent: + given_packages |= uc.get_inconsistent() + if args.force: uc.force = True uc.crawl(given_packages) @@ -342,6 +382,8 @@ help='don\'t take previous lookup information into consideration') parser.add_argument('--cache-requests', action='store_true', default=False, help='cache GET requests. Not recommended for daily use.') + parser.add_argument('--check-inconsistent', action='store_true', default=False, + help='also check insonsistent lookup entries') parser.add_argument("packages", nargs='*', help="packages to check") args = parser.parse_args() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/osclib/check_duplicate_binaries_command.py new/openSUSE-release-tools-20180102.98a659c/osclib/check_duplicate_binaries_command.py --- old/openSUSE-release-tools-20171226.96a3eec/osclib/check_duplicate_binaries_command.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/osclib/check_duplicate_binaries_command.py 2018-01-02 11:39:25.000000000 +0100 @@ -60,14 +60,19 @@ binaries[arch][name] = package + # convert sets to lists for readable yaml + for arch in duplicates.keys(): + for name in duplicates[arch].keys(): + duplicates[arch][name] = list(duplicates[arch][name]) + + current = yaml.dump(duplicates, default_flow_style=False) if save: args = ['{}:Staging'.format(self.api.project), 'dashboard', 'duplicate_binaries'] previous = self.api.load_file_content(*args) - current = yaml.dump(duplicates, default_flow_style=False) if current != previous: args.append(current) self.api.save_file_content(*args) else: - pprint(duplicates) + print(current) # vim: sw=4 et diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/osclib/comments.py new/openSUSE-release-tools-20180102.98a659c/osclib/comments.py --- old/openSUSE-release-tools-20171226.96a3eec/osclib/comments.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/osclib/comments.py 2018-01-02 11:39:25.000000000 +0100 @@ -123,6 +123,12 @@ marker = '<!-- {}{} -->'.format(bot, ' ' + ' '.join(infos) if info else '') return marker + '\n\n' + comment + def remove_marker(self, comment): + if comment.startswith('<!--'): + comment = ''.join(comment.splitlines(True)[1:]).strip() + + return comment + def add_comment(self, request_id=None, project_name=None, package_name=None, comment=None, parent_id=None): """Add a comment in an object in OBS. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/osclib/conf.py new/openSUSE-release-tools-20180102.98a659c/osclib/conf.py --- old/openSUSE-release-tools-20171226.96a3eec/osclib/conf.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/osclib/conf.py 2018-01-02 11:39:25.000000000 +0100 @@ -69,6 +69,11 @@ # check_source.py # review-team optionally added by leaper.py. 'repo-checker': 'repo-checker', + 'pkglistgen-archs': 'x86_64', + 'pkglistgen-archs-ports': 'aarch64', + 'pkglistgen-locals-from': 'openSUSE-product', + 'pkglistgen-include-suggested': '1', + 'pkglistgen-delete-kiwis': 'openSUSE-ftp-ftp-x86_64.kiwi openSUSE-cd-mini-x86_64.kiwi', }, r'SUSE:(?P<project>SLE-15.*$)': { 'staging': 'SUSE:%(project)s:Staging', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/osclib/request_splitter.py new/openSUSE-release-tools-20180102.98a659c/osclib/request_splitter.py --- old/openSUSE-release-tools-20171226.96a3eec/osclib/request_splitter.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/osclib/request_splitter.py 2018-01-02 11:39:25.000000000 +0100 @@ -176,15 +176,6 @@ return '00' return '__'.join(key) - def is_staging_bootstrapped(self, project): - if self.api.rings: - # Determine if staging is bootstrapped. - meta = self.api.get_prj_meta(project) - xpath = 'link[@project="{}"]'.format(self.api.rings[0]) - return meta.find(xpath) is not None - - return False - def is_staging_mergeable(self, status, pseudometa): return len(pseudometa['requests']) > 0 and 'splitter_info' in pseudometa @@ -238,7 +229,7 @@ for staging in stagings: project = self.api.prj_from_short(staging) status, pseudometa = self.staging_status_load(project) - bootstrapped = self.is_staging_bootstrapped(project) + bootstrapped = self.api.is_staging_bootstrapped(project) # Store information about staging. self.stagings[staging] = { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/osclib/stagingapi.py new/openSUSE-release-tools-20180102.98a659c/osclib/stagingapi.py --- old/openSUSE-release-tools-20171226.96a3eec/osclib/stagingapi.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/osclib/stagingapi.py 2018-01-02 11:39:25.000000000 +0100 @@ -1811,3 +1811,12 @@ break_long_words=False) return None + + def is_staging_bootstrapped(self, project): + if self.rings: + # Determine if staging is bootstrapped. + meta = self.get_prj_meta(project) + xpath = 'link[@project="{}"]'.format(self.rings[0]) + return meta.find(xpath) is not None + + return False diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/pkglistgen.py new/openSUSE-release-tools-20180102.98a659c/pkglistgen.py --- old/openSUSE-release-tools-20171226.96a3eec/pkglistgen.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/pkglistgen.py 2018-01-02 11:39:25.000000000 +0100 @@ -24,13 +24,22 @@ # TODO: solve all devel packages to include from __future__ import print_function +import copy from lxml import etree as ET from collections import namedtuple import sys import cmdln import logging import urllib2 -import osc.core +from osc.core import checkout_package +from osc.core import http_GET +from osc.core import makeurl +from osc.core import Package +from osc.core import show_results_meta +from osc.core import undelete_package +from osc import conf +from osclib.conf import Config +from osclib.stagingapi import StagingAPI import glob import solv from pprint import pprint, pformat @@ -44,6 +53,7 @@ import gzip import tempfile import random +import shutil import string import ToolBase @@ -55,6 +65,7 @@ ARCHITECTURES = ['x86_64', 'ppc64le', 's390x', 'aarch64'] DEFAULT_REPOS = ("openSUSE:Factory/standard") +PRODUCT_SERVICE = '/usr/lib/obs/service/create_single_product' class Group(object): @@ -627,6 +638,7 @@ fh.write(" \n") class CommandLineInterface(ToolBase.CommandLineInterface): + SCOPES = ['all', 'target', 'rings', 'staging', 'ports'] def __init__(self, *args, **kwargs): ToolBase.CommandLineInterface.__init__(self, args, kwargs) @@ -929,6 +941,233 @@ self.tool._collect_unsorted_packages(modules) self.tool._write_all_groups() + @cmdln.option('-f', '--force', action='store_true', help='continue even if build is in progress') + @cmdln.option('-p', '--project', help='target project') + @cmdln.option('-s', '--scope', default='all', help='scope on which to operate ({})'.format(', '.join(SCOPES))) + def do_update_and_solve(self, subcmd, opts): + """${cmd_name}: update and solve for given scope + + ${cmd_usage} + ${cmd_option_list} + """ + + if not opts.project: + raise ValueError('project is required') + if opts.scope not in self.SCOPES: + raise ValueError('scope must be one of: {}'.format(', '.join(self.SCOPES))) + + if opts.scope == 'all': + for scope in self.SCOPES[1:]: + opts.scope = scope + self.do_update_and_solve(subcmd, copy.deepcopy(opts)) + return + + # Store target project as opts.project will contain subprojects. + target_project = opts.project + + config = Config(target_project) + apiurl = conf.config['apiurl'] + api = StagingAPI(apiurl, target_project) + config.apply_remote(api) + + target_config = conf.config[target_project] + archs_key = 'pkglistgen-archs' if opts.scope != 'ports' else 'pkglistgen-archs-ports' + if archs_key in target_config: + self.options.architectures = target_config.get(archs_key).split(' ') + main_repo = target_config['main-repo'] + + if opts.scope == 'target': + self.options.repos = ['/'.join([target_project, main_repo])] + self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts) + return + elif opts.scope == 'ports': + # TODO Continue supporting #1297, but should be abstracted. + main_repo = 'ports' + opts.project += ':Ports' + self.options.repos = ['/'.join([opts.project, main_repo])] + self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts) + return + elif opts.scope == 'rings': + opts.project = api.rings[1] + self.options.repos = [ + '/'.join([api.rings[1], main_repo]), + '/'.join([api.rings[0], main_repo]), + ] + self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts) + + opts.project = api.rings[2] + self.options.repos.insert(0, '/'.join([api.rings[2], main_repo])) + self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts, skip_release=True) + return + elif opts.scope == 'staging': + letters = api.get_staging_projects_short() + for letter in letters: + opts.project = api.prj_from_short(letter) + self.options.repos = ['/'.join([opts.project, main_repo])] + + if not api.is_staging_bootstrapped(opts.project): + self.options.repos.append('/'.join([opts.project, 'bootstrap_copy'])) + + # DVD project first since it depends on main. + if api.rings: + opts_dvd = copy.deepcopy(opts) + opts.project += ':DVD' + self.options.repos.insert(0, '/'.join([opts.project, main_repo])) + self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts_dvd, skip_release=True) + + self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts) + return + + def update_and_solve_target(self, apiurl, target_project, target_config, main_repo, opts, + skip_release=False): + group = target_config.get('pkglistgen-group', '000package-groups') + product = target_config.get('pkglistgen-product', '000product') + release = target_config.get('pkglistgen-release', '000release-packages') + + url = makeurl(apiurl, ['source', opts.project]) + packages = ET.parse(http_GET(url)).getroot() + if packages.find('entry[@name="{}"]'.format(product)) is None: + undelete_package(apiurl, opts.project, product, 'revive') + # TODO disable build. + print('{} undeleted, skip dvd until next cycle'.format(product)) + return + elif not opts.force: + root = ET.fromstringlist(show_results_meta(apiurl, opts.project, product, + repository=[main_repo], multibuild=True)) + if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')): + print('{}/{} build in progress'.format(opts.project, product)) + return + + checkout_list = [group, product] + if not skip_release: + checkout_list.append(release) + + if packages.find('entry[@name="{}"]'.format(release)) is None: + undelete_package(apiurl, opts.project, product, 'revive') + print('{} undeleted, skip dvd until next cycle'.format(release)) + return + + # Cache dir specific to hostname and project. + host = urlparse.urlparse(apiurl).hostname + cache_dir = os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache')) + cache_dir = os.path.join(cache_dir, 'opensuse-packagelists', host, opts.project) + + if os.path.exists(cache_dir): + shutil.rmtree(cache_dir) + os.makedirs(cache_dir) + + group_dir = os.path.join(cache_dir, group) + product_dir = os.path.join(cache_dir, product) + release_dir = os.path.join(cache_dir, release) + + for package in checkout_list: + checkout_package(apiurl, opts.project, package, expand_link=True, prj_dir=cache_dir) + + if not skip_release: + self.unlink_all_except(release_dir) + self.unlink_all_except(product_dir) + self.copy_directory_contents(group_dir, product_dir, + ['supportstatus.txt', 'groups.yml', 'package-groups.changes']) + self.change_extension(product_dir, '.spec.in', '.spec') + + self.options.output_dir = product_dir + self.postoptparse() + + self.do_update('update', opts) + + opts.ignore_recommended = bool(target_config.get('pkglistgen-include-recommended')) + opts.include_suggested = bool(target_config.get('pkglistgen-include-suggested')) + opts.locales_from = target_config.get('pkglistgen-locals-from') + self.do_solve('solve', opts) + + delete_products = target_config.get('pkglistgen-delete-products', '').split(' ') + self.unlink_list(product_dir, delete_products) + + for product_file in glob.glob(os.path.join(product_dir, '*.product')): + print(subprocess.check_output( + [PRODUCT_SERVICE, product_file, product_dir, opts.project])) + + delete_kiwis = target_config.get('pkglistgen-delete-kiwis', '').split(' ') + self.unlink_list(product_dir, delete_kiwis) + + spec_files = glob.glob(os.path.join(product_dir, '*.spec')) + if skip_release: + self.unlink_list(None, spec_files) + else: + self.move_list(spec_files, release_dir) + + self.multibuild_from_glob(product_dir, '*.kiwi') + self.build_stub(product_dir, 'kiwi') + self.commit_package(product_dir) + + if not skip_release: + self.multibuild_from_glob(release_dir, '*.spec') + self.build_stub(release_dir, 'spec') + self.commit_package(release_dir) + + def move_list(self, file_list, destination): + for name in file_list: + os.rename(name, os.path.join(destination, os.path.basename(name))) + + def unlink_list(self, path, names): + for name in names: + if path is None: + name_path = name + else: + name_path = os.path.join(path, name) + + if os.path.isfile(name_path): + os.unlink(name_path) + + def unlink_all_except(self, path, ignore_list=['_service'], ignore_hidden=True): + for name in os.listdir(path): + if name in ignore_list or (ignore_hidden and name.startswith('.')): + continue + + name_path = os.path.join(path, name) + if os.path.isfile(name_path): + os.unlink(name_path) + + def copy_directory_contents(self, source, destination, ignore_list=[]): + for name in os.listdir(source): + name_path = os.path.join(source, name) + if name in ignore_list or not os.path.isfile(name_path): + continue + + shutil.copy(name_path, os.path.join(destination, name)) + + def change_extension(self, path, original, final): + for name in glob.glob(os.path.join(path, '*{}'.format(original))): + # Assumes the extension is only found at the end. + os.rename(name, name.replace(original, final)) + + def multibuild_from_glob(self, destination, pathname): + root = ET.Element('multibuild') + for name in glob.glob(os.path.join(destination, pathname)): + package = ET.SubElement(root, 'package') + package.text = os.path.splitext(os.path.basename(name))[0] + + with open(os.path.join(destination, '_multibuild'), 'w+b') as f: + f.write(ET.tostring(root, pretty_print=True)) + + def build_stub(self, destination, extension): + f = file(os.path.join(destination, '.'.join(['stub', extension])), 'w+') + f.write('# prevent building single {} files twice\n'.format(extension)) + f.write('Name: stub\n') + f.write('Version: 0.0\n') + f.close() + + def commit_package(self, path): + package = Package(path) + if self.options.dry: + for i in package.get_diff(): + print(''.join(i)) + else: + # No proper API function to perform the same operation. + print(subprocess.check_output( + ' '.join(['cd', path, '&&', 'osc', 'addremove']), shell=True)) + package.commit(msg='Automatic update') + if __name__ == "__main__": app = CommandLineInterface() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/pkglistgen.sh new/openSUSE-release-tools-20180102.98a659c/pkglistgen.sh --- old/openSUSE-release-tools-20171226.96a3eec/pkglistgen.sh 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/pkglistgen.sh 1970-01-01 01:00:00.000000000 +0100 @@ -1,146 +0,0 @@ -#!/bin/bash -# FIXME: 000package-groups is not frozen, osc up doesn't do anything -# when updated in underlying project - -set -e -shopt -s nullglob - -self=$(readlink -e $(type -p "$0")) - -: ${project:=openSUSE:Factory} -: ${api:=api.opensuse.org} -: ${repos:=$project/standard} -: ${productrepo:=standard} -: ${arch:=x86_64} - -groups="000package-groups" -product="000product" -releases="000release-packages" - -cachedir=${XDG_CACHE_HOME:-~/.cache}/opensuse-packagelists/$api/$project -todo=("$product" "$groups") -solveargs=() - -if [ -n "$IGNORE_RECOMMENDED" ]; then - solveargs+=('--ignore-recommended') -fi -if [ -n "$INCLUDE_SUGGESTED" ]; then - solveargs+=('--include-suggested') -fi -if [ -n "$LOCALES_FROM" ]; then - solveargs+=('--locales-from', "$LOCALES_FROM") -fi - -_osc=`type -p osc` -osc() -{ - "$_osc" -A "https://$api" "$@" -} - -checkin() { - if [ -n "$dryrun" ]; then - osc diff - else - osc addremove - osc ci -m "Automatic update" - fi -} - -if ! osc api "/source/$project/" | grep -q "$product" ; then - osc undelete -m revive "$project/$product" - # FIXME: build disable it - echo "$product undeleted, skip dvd until next cycle" - exit 0 -elif [ -z "$FORCE" ]; then - bs_status=`osc api "/build/$project/_result?package=$product&repository=$productrepo"` - if echo "${bs_status}" | grep -q 'building\|dirty'; then - echo "$project build in progress, skipping." - exit 0 - fi -fi - -mkdir -p "$cachedir" -cd "$cachedir" - -if [ -z "$skip_releases" ]; then - todo+=("$releases") - if ! osc api "/source/$project/" | grep -q "$releases" ; then - osc undelete -m revive "$project/$releases" - echo "$releases undeleted, skip dvd until next cycle" - exit 0 - fi -fi -# update package checkouts -for i in "${todo[@]}"; do - if [ ! -e "$i" ]; then - osc co -c "$project/$i" - fi - pushd "$i" - if ! osc status; then - # merge conflict etc, try to check out new - popd - rm -rf "$i" - osc co -c "$project/$i" - else - osc up - popd - fi -done - -[ -z "$releases" ] || rm -f "$cachedir/$releases"/* -cd "$cachedir/$product" -rm -f -- * -cp .osc/_service . -cp "$cachedir/$groups"/* . -rm -f supportstatus.txt groups.yml package-groups.changes -for i in *.spec.in; do - mv -v $i "${i%.in}" -done -if ! ${self%.sh}.py -i "$cachedir/$groups" -r $repos -o . -a $arch update; then - echo "no change in packages" -fi -${self%.sh}.py -i "$cachedir/$groups" -r $repos -o . -a $arch solve "${solveargs[@]}" -for i in $delete_products; do - rm -vf -- "$i" -done -for i in *.product; do - /usr/lib/obs/service/create_single_product $PWD/$i $PWD $(cat .osc/_project) -done -for i in $delete_kiwis; do - rm -vf -- "$i" -done -if [ -z "$skip_releases" ]; then - mv -v *.spec "$cachedir/$releases" -else - rm -vf *.spec -fi -echo '<multibuild>' > _multibuild -for file in *.kiwi; do - container="${file##*/}" - container="${container%.kiwi}" - echo " <package>${container}</package>" >> _multibuild -done -echo '</multibuild>' >> _multibuild -cat << EOF > stub.kiwi -# prevent building single kiwi files twice -Name: stub -Version: 0.0 -EOF -checkin - -if [ -z "$skip_releases" ]; then - cd "$cachedir/$releases" - echo '<multibuild>' > _multibuild - for file in *.spec; do - container="${file##*/}" - container="${container%.spec}" - echo " <package>${container}</package>" >> _multibuild - done - echo '</multibuild>' >> _multibuild - cat <<-EOF > stub.spec - # prevent building single spec files twice - Name: stub - Version: 0.0 - EOF - checkin -fi diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/repo_checker.py new/openSUSE-release-tools-20180102.98a659c/repo_checker.py --- old/openSUSE-release-tools-20171226.96a3eec/repo_checker.py 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/repo_checker.py 2018-01-02 11:39:25.000000000 +0100 @@ -42,13 +42,16 @@ self.force = False self.limit_group = None + def repository_published(self, project): + root = ET.fromstringlist(show_results_meta( + self.apiurl, project, multibuild=True, repository=['standard'])) + return not len(root.xpath('result[@state!="published"]')) + def project_only(self, project, post_comments=False): # self.staging_config needed by target_archs(). api = self.staging_api(project) - root = ET.fromstringlist(show_results_meta( - self.apiurl, project, multibuild=True, repository=['standard'])) - if not self.force and len(root.xpath('result[@state!="published"]')): + if not self.force and self.repository_published(project): self.logger.info('{}/standard not published'.format(project)) return @@ -132,7 +135,7 @@ if not status or str(status['overall_state']) not in ('testing', 'review', 'acceptable'): # Not in a "ready" state. openQA_only = False # Not relevant so set to False. - if str(status['overall_state']) == 'failed': + if status and str(status['overall_state']) == 'failed': # Exception to the rule is openQA only in failed state. openQA_only = True for project in api.project_status_walk(status): @@ -241,9 +244,12 @@ info_extra = {'build': self.groups_build[group]} if not self.group_pass: # Some checks in group did not pass, post comment. + # Avoid identical comments with different build hash during target + # project build phase. Once published update regardless. + published = self.repository_published(project) self.comment_write(state='seen', result='failed', project=group, message='\n'.join(comment).strip(), identical=True, - info_extra=info_extra) + info_extra=info_extra, info_extra_identical=published) else: # Post passed comment only if previous failed comment. text = 'Previously reported problems have been resolved.' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0 new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0 --- old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0 1970-01-01 01:00:00.000000000 +0100 @@ -1,3 +0,0 @@ -#!/bin/bash -export project=openSUSE:Leap:15.0 -osrt-pkglistgen diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0-all new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0-all --- old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0-all 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0-all 1970-01-01 01:00:00.000000000 +0100 @@ -1,6 +0,0 @@ -#!/bin/bash -export LOCALES_FROM=openSUSE-product -export INCLUDE_SUGGESTED=1 -osrt-pkglistgen-openSUSE:Leap:15.0 -osrt-pkglistgen-openSUSE:Leap:15.0:Rings -osrt-pkglistgen-openSUSE:Leap:15.0:Staging diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh --- old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh 1970-01-01 01:00:00.000000000 +0100 @@ -1,7 +0,0 @@ -#!/bin/bash -self=$(readlink $(type -p "$0")) -export project=openSUSE:Leap:15.0:Ports -export repos=$project/ports -export arch=aarch64 -export productrepo=ports -osrt-pkglistgen diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0:Rings new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0:Rings --- old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0:Rings 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0:Rings 1970-01-01 01:00:00.000000000 +0100 @@ -1,12 +0,0 @@ -#!/bin/bash -main=openSUSE:Leap:15.0 -export delete_kiwis="openSUSE-ftp-ftp-x86_64.kiwi openSUSE-cd-mini-x86_64.kiwi" - -export project=$main:Rings:1-MinimalX -export repos=$project/standard,$main:Rings:0-Bootstrap/standard -osrt-pkglistgen - -export project=$main:Rings:2-TestDVD -export repos=$main:Rings:2-TestDVD/standard,$repos -export skip_releases=1 -osrt-pkglistgen diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0:Staging new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0:Staging --- old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0:Staging 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0:Staging 1970-01-01 01:00:00.000000000 +0100 @@ -1,20 +0,0 @@ -#!/bin/bash -self=$(readlink $(type -p "$0")) -main=openSUSE:Leap:15.0 -: ${letter:=A B C D E} - -export delete_kiwis="openSUSE-ftp-ftp-x86_64.kiwi openSUSE-cd-mini-x86_64.kiwi" -for l in $letter; do - export project=$main:Staging:$l - echo "checking $project..." - export repos=$project/standard - if [ "$l" != A -a "$l" != B ]; then - repos="$repos,$project/bootstrap_copy" - fi - - # DVD project first as it depends on the project below, so might look - # busy if we update the other one first - project=$project:DVD repos=$project/standard,$repos skip_releases=1 osrt-pkglistgen - - osrt-pkglistgen -done diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/systemd/osrt-pkglistgen@.service new/openSUSE-release-tools-20180102.98a659c/systemd/osrt-pkglistgen@.service --- old/openSUSE-release-tools-20171226.96a3eec/systemd/osrt-pkglistgen@.service 2017-12-26 12:36:26.000000000 +0100 +++ new/openSUSE-release-tools-20180102.98a659c/systemd/osrt-pkglistgen@.service 2018-01-02 11:39:25.000000000 +0100 @@ -3,7 +3,7 @@ [Service] User=osrt-repo-checker -ExecStart=/usr/bin/osrt-pkglistgen-%i-all +ExecStart=/usr/bin/osrt-pkglistgen -p "%i" [Install] WantedBy=multi-user.target ++++++ openSUSE-release-tools.obsinfo ++++++ --- /var/tmp/diff_new_pack.qsXQKW/_old 2018-01-02 16:35:52.334776989 +0100 +++ /var/tmp/diff_new_pack.qsXQKW/_new 2018-01-02 16:35:52.338775656 +0100 @@ -1,5 +1,5 @@ name: openSUSE-release-tools -version: 20171226.96a3eec -mtime: 1514288186 -commit: 96a3eec425d49f995a85e000da4a8799b5a6a461 +version: 20180102.98a659c +mtime: 1514889565 +commit: 98a659c3fab8dc036f9945854eae93265d695bed