Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package openSUSE-release-tools for
openSUSE:Factory checked in at 2021-08-31 19:55:38
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old)
and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.1899 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openSUSE-release-tools"
Tue Aug 31 19:55:38 2021 rev:318 rq:915210 version:20210831.0de63caa
Changes:
--------
---
/work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes
2021-08-23 10:09:10.456176120 +0200
+++
/work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.1899/openSUSE-release-tools.changes
2021-08-31 19:56:32.054025267 +0200
@@ -1,0 +2,8 @@
+Tue Aug 31 08:53:00 UTC 2021 - opensuse-releaset...@opensuse.org
+
+- Update to version 20210831.0de63caa:
+ * Update specfile for skippkg-finder
+ * Run skippkg-finder.py before pkglistgen
+ * Add skippkg-finder.py to overwrite NON_FTP_PACKAGES for obsoleted and
unneeded package
+
+-------------------------------------------------------------------
Old:
----
openSUSE-release-tools-20210820.43b07171.obscpio
New:
----
openSUSE-release-tools-20210831.0de63caa.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ openSUSE-release-tools.spec ++++++
--- /var/tmp/diff_new_pack.NbnALe/_old 2021-08-31 19:56:32.726026196 +0200
+++ /var/tmp/diff_new_pack.NbnALe/_new 2021-08-31 19:56:32.730026202 +0200
@@ -20,7 +20,7 @@
%define source_dir openSUSE-release-tools
%define announcer_filename factory-package-news
Name: openSUSE-release-tools
-Version: 20210820.43b07171
+Version: 20210831.0de63caa
Release: 0
Summary: Tools to aid in staging and release work for openSUSE/SUSE
License: GPL-2.0-or-later AND MIT
@@ -438,6 +438,7 @@
%exclude %{_datadir}/%{source_dir}/maintenance-installcheck.py
%exclude %{_datadir}/%{source_dir}/project-installcheck.py
%exclude %{_datadir}/%{source_dir}/suppkg_rebuild.py
+%exclude %{_datadir}/%{source_dir}/skippkg-finder.py
%exclude %{_datadir}/%{source_dir}/osclib
%exclude %{_datadir}/%{source_dir}/osc-cycle.py
%exclude %{_datadir}/%{source_dir}/osc-origin.py
@@ -551,8 +552,10 @@
%files pkglistgen
%defattr(-,root,root,-)
%{_bindir}/osrt-pkglistgen
+%{_bindir}/osrt-skippkg-finder
%{_datadir}/%{source_dir}/pkglistgen
%{_datadir}/%{source_dir}/pkglistgen.py
+%{_datadir}/%{source_dir}/skippkg-finder.py
%files -n osclib
%defattr(-,root,root,-)
++++++ _servicedata ++++++
--- /var/tmp/diff_new_pack.NbnALe/_old 2021-08-31 19:56:32.782026274 +0200
+++ /var/tmp/diff_new_pack.NbnALe/_new 2021-08-31 19:56:32.786026280 +0200
@@ -1,6 +1,6 @@
<servicedata>
<service name="tar_scm">
<param
name="url">https://github.com/openSUSE/openSUSE-release-tools.git</param>
- <param
name="changesrevision">33d44e218e9251d5dd3cc30d285c03a5a20b5b4c</param>
+ <param
name="changesrevision">2a3ee0c246daccbd3b500896fe6057c8dc04777b</param>
</service>
</servicedata>
++++++ openSUSE-release-tools-20210820.43b07171.obscpio ->
openSUSE-release-tools-20210831.0de63caa.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20210820.43b07171/dist/package/openSUSE-release-tools.spec
new/openSUSE-release-tools-20210831.0de63caa/dist/package/openSUSE-release-tools.spec
---
old/openSUSE-release-tools-20210820.43b07171/dist/package/openSUSE-release-tools.spec
2021-08-20 09:49:06.000000000 +0200
+++
new/openSUSE-release-tools-20210831.0de63caa/dist/package/openSUSE-release-tools.spec
2021-08-31 10:51:24.000000000 +0200
@@ -439,6 +439,7 @@
%exclude %{_datadir}/%{source_dir}/maintenance-installcheck.py
%exclude %{_datadir}/%{source_dir}/project-installcheck.py
%exclude %{_datadir}/%{source_dir}/suppkg_rebuild.py
+%exclude %{_datadir}/%{source_dir}/skippkg-finder.py
%exclude %{_datadir}/%{source_dir}/osclib
%exclude %{_datadir}/%{source_dir}/osc-cycle.py
%exclude %{_datadir}/%{source_dir}/osc-origin.py
@@ -552,8 +553,10 @@
%files pkglistgen
%defattr(-,root,root,-)
%{_bindir}/osrt-pkglistgen
+%{_bindir}/osrt-skippkg-finder
%{_datadir}/%{source_dir}/pkglistgen
%{_datadir}/%{source_dir}/pkglistgen.py
+%{_datadir}/%{source_dir}/skippkg-finder.py
%files -n osclib
%defattr(-,root,root,-)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20210820.43b07171/gocd/pkglistgen.opensuse.gocd.yaml
new/openSUSE-release-tools-20210831.0de63caa/gocd/pkglistgen.opensuse.gocd.yaml
---
old/openSUSE-release-tools-20210820.43b07171/gocd/pkglistgen.opensuse.gocd.yaml
2021-08-20 09:49:06.000000000 +0200
+++
new/openSUSE-release-tools-20210831.0de63caa/gocd/pkglistgen.opensuse.gocd.yaml
2021-08-31 10:51:24.000000000 +0200
@@ -107,6 +107,7 @@
resources:
- repo-checker
tasks:
+ - script: python3 ./skippkg-finder.py-A https://api.opensuse.org
-o openSUSE:Leap:15.4 -s SUSE:SLE-15-SP4:GA
- script: python3 ./pkglistgen.py -d -A https://api.opensuse.org
update_and_solve -p openSUSE:Leap:15.4 -s target
Update.Repos.Leap:
group: Leap.pkglistgen
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20210820.43b07171/gocd/pkglistgen.opensuse.gocd.yaml.erb
new/openSUSE-release-tools-20210831.0de63caa/gocd/pkglistgen.opensuse.gocd.yaml.erb
---
old/openSUSE-release-tools-20210820.43b07171/gocd/pkglistgen.opensuse.gocd.yaml.erb
2021-08-20 09:49:06.000000000 +0200
+++
new/openSUSE-release-tools-20210831.0de63caa/gocd/pkglistgen.opensuse.gocd.yaml.erb
2021-08-31 10:51:24.000000000 +0200
@@ -81,6 +81,7 @@
resources:
- repo-checker
tasks:
+ - script: python3 ./skippkg-finder.py-A https://api.opensuse.org
-o openSUSE:Leap:15.4 -s SUSE:SLE-15-SP4:GA
- script: python3 ./pkglistgen.py -d -A https://api.opensuse.org
update_and_solve -p <%= project[0] %><%= options %>
<% end -%>
Update.Repos.Leap:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20210820.43b07171/skippkg-finder.py
new/openSUSE-release-tools-20210831.0de63caa/skippkg-finder.py
--- old/openSUSE-release-tools-20210820.43b07171/skippkg-finder.py
1970-01-01 01:00:00.000000000 +0100
+++ new/openSUSE-release-tools-20210831.0de63caa/skippkg-finder.py
2021-08-31 10:51:24.000000000 +0200
@@ -0,0 +1,372 @@
+#!/usr/bin/python3
+
+import argparse
+import logging
+import sys
+
+from urllib.error import HTTPError
+
+import re
+from lxml import etree as ET
+from collections import namedtuple
+
+import osc.conf
+import osc.core
+from osc.core import http_GET
+from osc.core import makeurl
+from osc import oscerr
+import osclib
+from osclib.core import source_file_ensure
+
+SUPPORTED_ARCHS = ['x86_64', 'i586', 'aarch64', 'ppc64le', 's390x']
+DEFAULT_REPOSITORY = 'standard'
+
+META_PACKAGE = '000package-groups'
+
+class SkippkgFinder(object):
+ def __init__(self, opensuse_project, sle_project, print_only, verbose):
+ self.opensuse_project = opensuse_project
+ self.sle_project = sle_project
+ self.print_only = print_only
+ self.verbose = verbose
+ self.apiurl = osc.conf.config['apiurl']
+ self.debug = osc.conf.config['debug']
+
+ def is_sle_specific(self, package):
+ """
+ Return True if package is provided for SLE only or a SLE forking.
+ Add new condition here if you do not want package being added to
+ selected_binarylist[].
+ """
+ pkg = package.lower()
+ prefixes = (
+ 'desktop-data',
+ 'libyui-bindings',
+ 'libyui-doc',
+ 'libyui-ncurses',
+ 'libyui-qt',
+ 'libyui-rest',
+ 'lifecycle-data-sle',
+ 'kernel-livepatch',
+ 'kiwi-template',
+ 'mgr-',
+ 'migrate',
+ 'patterns',
+ 'release-notes',
+ 'sap',
+ 'sca-',
+ 'skelcd',
+ 'sle-',
+ 'sle_',
+ 'sle15',
+ 'sles15',
+ 'spacewalk',
+ 'supportutils-plugin',
+ 'suse-migration',
+ 'susemanager-',
+ 'yast2-hana'
+ )
+ suffixes = (
+ '-caasp',
+ '-sle',
+ 'bootstrap'
+ )
+ matches = (
+ 'gtk-vnc2',
+ 'ibus-googlepinyin',
+ 'infiniband-diags',
+ 'llvm',
+ 'lua51-luajit',
+ 'lvm2-clvm',
+ 'osad',
+ 'rhncfg',
+ 'python-ibus',
+ 'python-pymemcache',
+ 'suse-build-key',
+ 'suse-hpc',
+ 'txt2tags',
+ 'zypp-plugin-spacewalk',
+ 'zypper-search-packages-plugin'
+ )
+ if pkg.startswith(prefixes) or pkg.endswith(suffixes) or pkg in
matches:
+ return True
+ if 'sles' in pkg or\
+ 'sled' in pkg or\
+ 'sap-' in pkg or\
+ '-sap' in pkg or\
+ 'eula' in pkg or\
+ 'branding' in pkg:
+ return True
+ return False
+
+ def get_packagelist(self, project, by_project=True):
+ """
+ Return the list of package's info of a project.
+ If the latest package is from an incident then returns incident
+ package.
+ """
+
+ pkglist = {}
+ packageinfo = {}
+ query = {'expand': 1}
+ root = ET.parse(http_GET(makeurl(self.apiurl, ['source', project],
+ query=query))).getroot()
+ for i in root.findall('entry'):
+ pkgname = i.get('name')
+ orig_project = i.get('originproject')
+ is_incidentpkg = False
+ # Metapackage should not be selected
+ if pkgname.startswith('000') or\
+ pkgname.startswith('_') or\
+ pkgname.startswith('patchinfo.') or\
+ pkgname.startswith('skelcd-') or\
+ pkgname.startswith('installation-images') or\
+ pkgname.endswith('-mini'):
+ continue
+ # Ugly hack for package has dot in source package name
+ # eg. go1.x incidents as the name would be go1.x.xxx
+ if '.' in pkgname and re.match(r'[0-9]+$', pkgname.split('.')[-1])
and \
+ orig_project.startswith('SUSE:') and
orig_project.endswith(':Update'):
+ is_incidentpkg = True
+ if pkgname.startswith('go1') or\
+ pkgname.startswith('bazel0') or\
+ pkgname.startswith('dotnet') or\
+ pkgname.startswith('rust1') or\
+ pkgname.startswith('ruby2'):
+ if not (pkgname.count('.') > 1):
+ is_incidentpkg = False
+
+ # If an incident found then update the package origin info
+ if is_incidentpkg:
+ orig_name = re.sub(r'\.[0-9]+$', '', pkgname)
+ incident_number = int(pkgname.split('.')[-1])
+ if orig_name in pkglist and pkglist[orig_name]['Project'] ==
orig_project:
+ if re.match(r'[0-9]+$',
pkglist[orig_name]['Package'].split('.')[-1]):
+ old_incident_number =
int(pkglist[orig_name]['Package'].split('.')[-1])
+ if incident_number > old_incident_number:
+ pkglist[orig_name]['Package'] = pkgname
+ else:
+ pkglist[orig_name]['Package'] = pkgname
+ else:
+ pkglist[pkgname] = {'Project': orig_project, 'Package':
pkgname}
+
+ if by_project:
+ for pkg in pkglist.keys():
+ if pkglist[pkg]['Project'].startswith('SUSE:') and
self.is_sle_specific(pkg):
+ continue
+ if pkglist[pkg]['Project'] not in packageinfo:
+ packageinfo[pkglist[pkg]['Project']] = []
+ if pkglist[pkg]['Package'] not in
packageinfo[pkglist[pkg]['Project']]:
+
packageinfo[pkglist[pkg]['Project']].append(pkglist[pkg]['Package'])
+ return packageinfo
+
+ return pkglist
+
+ def get_project_binary_list(self, project, repository, arch,
package_binaries={}):
+ """
+ Returns binarylist of a project
+ """
+
+ # Use pool repository for SUSE namespace project.
+ # Because RPMs were injected to pool repository on OBS rather than
+ # standard repository.
+ if project.startswith('SUSE:'):
+ repository = 'pool'
+
+ path = ['build', project, repository, arch]
+ url = makeurl(self.apiurl, path, {'view': 'binaryversions'})
+ root = ET.parse(http_GET(url)).getroot()
+
+ for binary_list in root:
+ package = binary_list.get('package')
+ package = package.split(':', 1)[0]
+ index = project + "_" + package
+
+ if index not in package_binaries:
+ package_binaries[index] = []
+ for binary in binary_list:
+ filename = binary.get('name')
+ result = re.match(osclib.core.RPM_REGEX, filename)
+ if not result:
+ continue
+
+ if result.group('arch') == 'src' or result.group('arch') ==
'nosrc':
+ continue
+ if result.group('name').endswith('-debuginfo') or
result.group('name').endswith('-debuginfo-32bit'):
+ continue
+ if result.group('name').endswith('-debugsource'):
+ continue
+
+ if result.group('name') not in package_binaries[index]:
+ package_binaries[index].append(result.group('name'))
+
+ return package_binaries
+
+ def exception_package(self, package):
+ """
+ Do not skip the package if matches the condition.
+ package parameter is source package name.
+ """
+
+ if '-bootstrap' in package or\
+ 'Tumbleweed' in package or\
+ 'metis' in package:
+ return True
+ # These packages must have a good reason not to be single-speced
+ # from one source.
+ if package.startswith('python2-') or\
+ package.startswith('python3'):
+ return True
+ return False
+
+ def exception_binary(self, package):
+ """
+ Do not skip the binary if matches the condition
+ package parameter is RPM filename.
+ """
+
+ if package == 'openSUSE-release' or\
+ package == 'openSUSE-release-ftp' or\
+ package == 'openSUSE-Addon-NonOss-release':
+ return True
+ return False
+
+ def crawl(self):
+ """Main method"""
+
+ leap_pkglist = self.get_packagelist(self.opensuse_project)
+ sle_pkglist = self.get_packagelist(self.sle_project, by_project=False)
+ # The selected_binarylist[] includes the latest sourcepackage list
+ # binary RPMs from the latest sources need to be presented in ftp
eventually
+ selected_binarylist = []
+ # Any existed binary RPMs from any SPx/Leap/Backports
+ fullbinarylist = []
+ # package_binaries[] is a pre-formated binarylist per each package
+ # access to the conotent uses
package_binaries['SUSE:SLE-15:Update_libcdio.12032']
+ package_binaries = {}
+
+ # Inject binarylist to a list per package name no matter what
archtectures was
+ for arch in SUPPORTED_ARCHS:
+ for prj in leap_pkglist.keys():
+ package_binaries = self.get_project_binary_list(prj,
DEFAULT_REPOSITORY, arch, package_binaries)
+
+ for pkg in package_binaries.keys():
+ if not self.exception_package(pkg):
+ fullbinarylist += package_binaries[pkg]
+
+ for prj in leap_pkglist.keys():
+ for pkg in leap_pkglist[prj]:
+ cands = [prj + "_" + pkg]
+ # Handling for SLE forks, or package has different multibuild
bits
+ # enablility between SLE and openSUSE
+ if prj.startswith('openSUSE:') and pkg in sle_pkglist and\
+ not self.is_sle_specific(pkg):
+ cands.append(sle_pkglist[pkg]['Project'] + "_" +
sle_pkglist[pkg]['Package'])
+ logging.debug(cands)
+ for index in cands:
+ if index in package_binaries:
+ selected_binarylist += package_binaries[index]
+ else:
+ logging.info("Can not find binary of %s" % index)
+
+ # Some packages has been obsoleted by new updated package, however
+ # there are application still depend on old library when it builds
+ # eg. SUSE:SLE-15-SP3:GA has qpdf/libqpdf28 but cups-filter was build
+ # in/when SLE15 SP2 which requiring qpdf/libqpdf6, therefore old
+ # qpdf/libqpdf6 from SLE15 SP2 should not to be missed.
+ extra_packagelist = [
+ # gnome-software requirement
+ 'SUSE:SLE-15-SP2:Update_libxmlb.15999',
+ # cups-filter requirement
+ 'SUSE:SLE-15-SP2:GA_qpdf',
+ # libcdio_paranoia2 requirement
+ 'SUSE:SLE-15:Update_libcdio.12032',
+ # libstoken1 requirement
+ 'SUSE:SLE-15:Update_libnettle.19992',
+ # python2-Pillow requirement
+ 'SUSE:SLE-15:Update_libwebp.19719',
+ # amarok requirement
+ 'SUSE:SLE-15:Update_mariadb.20531',
+ # bogofilter requirement
+ 'SUSE:SLE-15:GA_gsl',
+ # gnome-builder requirement
+ 'SUSE:SLE-15-SP2:GA_vala',
+ # hfst-ospell requirement
+ 'SUSE:SLE-15:Update_icu.14528'
+ ]
+ for pkg in extra_packagelist:
+ selected_binarylist += package_binaries[pkg]
+
+ # Preparing a packagelist for the skipping candidate
+ obsoleted = []
+ for pkg in fullbinarylist:
+ if pkg not in selected_binarylist and pkg not in obsoleted:
+ if not self.exception_binary(pkg):
+ obsoleted.append(pkg)
+
+ # Post processing of obsoleted packagelist
+ tmp_obsoleted = obsoleted.copy()
+ for pkg in tmp_obsoleted:
+ # Respect to single-speced python package, when a python2 RPM is
+ # considered then a python3 flavor should also be selected to be
+ # skipped, if not, don't add it.
+ if pkg.startswith('python2-') and re.sub(r'^python2', 'python3',
pkg) not in obsoleted:
+ obsoleted.remove(pkg)
+ # Main RPM must to be skipped if -32 bit RPM or -64bit RPM is
+ # considered.
+ if pkg.endswith('-32bit') or pkg.endswith('-64bit'):
+ main_filename = re.sub('-[36][24]bit', '', pkg)
+ if main_filename not in obsoleted:
+ obsoleted.remove(pkg)
+
+ skip_list = ET.Element('group', {'name': 'NON_FTP_PACKAGES'})
+ ET.SubElement(skip_list, 'conditional', {'name': 'drop_from_ftp'})
+ packagelist = ET.SubElement(skip_list, 'packagelist', {'relationship':
'requires'})
+ for pkg in sorted(obsoleted):
+ if not self.print_only and self.verbose:
+ print(pkg)
+ attr = {'name': pkg}
+ ET.SubElement(packagelist, 'package', attr)
+ if not self.print_only:
+ source_file_ensure(self.apiurl, self.opensuse_project,
META_PACKAGE, 'NON_FTP_PACKAGES.group',
+ ET.tostring(skip_list, pretty_print=True,
encoding='unicode'),
+ 'Update the skip list')
+ else:
+ print(ET.tostring(skip_list, pretty_print=True,
+ encoding='unicode'))
+
+
+def main(args):
+ osc.conf.get_config(override_apiurl=args.apiurl)
+ osc.conf.config['debug'] = args.debug
+
+ if args.opensuse_project is None or args.sle_project is None:
+ print("Please pass --opensuse-project and --sle-project argument. See
usage with --help.")
+ quit()
+
+ uc = SkippkgFinder(args.opensuse_project, args.sle_project,
args.print_only, args.verbose)
+ uc.crawl()
+
+
+if __name__ == '__main__':
+ description = 'Overwrites NON_FTP_PACKAGES.group according to the latest
sources. '\
+ 'This tool only works for Leap after CtLG implemented.'
+ parser = argparse.ArgumentParser(description=description)
+ parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL')
+ parser.add_argument('-d', '--debug', action='store_true',
+ help='print info useful for debuging')
+ parser.add_argument('-o', '--opensuse-project', dest='opensuse_project',
metavar='OPENSUSE_PROJECT',
+ help='openSUSE project on buildservice')
+ parser.add_argument('-s', '--sle-project', dest='sle_project',
metavar='SLE_PROJECT',
+ help='SLE project on buildservice')
+ parser.add_argument('-p', '--print-only', action='store_true',
+ help='show the result instead of the uploading')
+ parser.add_argument('-v', '--verbose', action='store_true',
+ help='show the diff')
+
+ args = parser.parse_args()
+
+ logging.basicConfig(level=logging.DEBUG if args.debug
+ else logging.INFO)
+
+ sys.exit(main(args))
++++++ openSUSE-release-tools.obsinfo ++++++
--- /var/tmp/diff_new_pack.NbnALe/_old 2021-08-31 19:56:33.630027448 +0200
+++ /var/tmp/diff_new_pack.NbnALe/_new 2021-08-31 19:56:33.630027448 +0200
@@ -1,5 +1,5 @@
name: openSUSE-release-tools
-version: 20210820.43b07171
-mtime: 1629445746
-commit: 43b07171ad85c0edbf2f11ec0853a32672e378d3
+version: 20210831.0de63caa
+mtime: 1630399884
+commit: 0de63caa85db778ad9f6909093f87bedb11cad02
