Add --jobs and --load-average options which allow dependency checks for multiple profiles to run in parallel. The increase in performance is non-linear for the number of jobs, but it can be worthwhile (I measured a 35% decrease in time when running 'repoman -j8 full' on sys-apps/portage). For the -j1 case (default), all dependency checks run in the main process as usual, so there is no significant performance penalty for the default case.
Bug: https://bugs.gentoo.org/448462 Signed-off-by: Zac Medico <zmed...@gentoo.org> --- repoman/lib/repoman/argparser.py | 9 ++ .../repoman/modules/scan/depend/profile.py | 117 +++++++++++++++--- repoman/man/repoman.1 | 9 +- 3 files changed, 116 insertions(+), 19 deletions(-) diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py index 670a0e91d..6d545ccca 100644 --- a/repoman/lib/repoman/argparser.py +++ b/repoman/lib/repoman/argparser.py @@ -199,6 +199,15 @@ def parse_args(argv, repoman_default_opts): '--output-style', dest='output_style', choices=output_keys, help='select output type', default='default') + parser.add_argument( + '-j', '--jobs', dest='jobs', action='store', type=int, default=1, + help='Specifies the number of jobs (processes) to run simultaneously.') + + parser.add_argument( + '-l', '--load-average', dest='load_average', action='store', type=float, default=None, + help='Specifies that no new jobs (processes) should be started if there are others ' + 'jobs running and the load average is at least load (a floating-point number).') + parser.add_argument( '--mode', dest='mode', choices=mode_keys, help='specify which mode repoman will run in (default=full)') diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py index 39d8b550c..1eb69422a 100644 --- a/repoman/lib/repoman/modules/scan/depend/profile.py +++ b/repoman/lib/repoman/modules/scan/depend/profile.py @@ -2,7 +2,9 @@ import copy +import functools import os +import types from pprint import pformat from _emerge.Package import Package @@ -15,6 +17,10 @@ from repoman.modules.scan.depend._gen_arches import _gen_arches from portage.dep import Atom from portage.package.ebuild.profile_iuse import iter_iuse_vars from portage.util import getconfig +from portage.util.futures import asyncio +from portage.util.futures.compat_coroutine import coroutine, coroutine_return +from portage.util.futures.executor.fork import ForkExecutor +from portage.util.futures.iter_completed import async_iter_completed def sort_key(item): @@ -58,16 +64,14 @@ class ProfileDependsChecks(ScanBase): def check(self, **kwargs): '''Perform profile dependant dependency checks - @param arches: @param pkg: Package in which we check (object). @param ebuild: Ebuild which we check (object). - @param baddepsyntax: boolean - @param unknown_pkgs: set of tuples (type, atom.unevaluated_atom) @returns: dictionary ''' ebuild = kwargs.get('ebuild').get() pkg = kwargs.get('pkg').get() - unknown_pkgs, baddepsyntax = _depend_checks( + + ebuild.unknown_pkgs, ebuild.baddepsyntax = _depend_checks( ebuild, pkg, self.portdb, self.qatracker, self.repo_metadata, self.repo_settings.qadata) @@ -90,8 +94,64 @@ class ProfileDependsChecks(ScanBase): relevant_profiles.append((keyword, groups, prof)) relevant_profiles.sort(key=sort_key) + ebuild.relevant_profiles = relevant_profiles + + if self.options.jobs <= 1: + for task in self._iter_tasks(None, None, ebuild, pkg): + task, results = task + for result in results: + self._check_result(task, result) + + loop = asyncio._wrap_loop() + loop.run_until_complete(self._async_check(loop=loop, **kwargs)) + + return False + + @coroutine + def _async_check(self, loop=None, **kwargs): + '''Perform async profile dependant dependency checks + + @param arches: + @param pkg: Package in which we check (object). + @param ebuild: Ebuild which we check (object). + @param baddepsyntax: boolean + @param unknown_pkgs: set of tuples (type, atom.unevaluated_atom) + @returns: dictionary + ''' + loop = asyncio._wrap_loop(loop) + ebuild = kwargs.get('ebuild').get() + pkg = kwargs.get('pkg').get() + unknown_pkgs = ebuild.unknown_pkgs + baddepsyntax = ebuild.baddepsyntax + + # Use max_workers=True to ensure immediate fork, since _iter_tasks + # needs the fork to create a snapshot of current state. + executor = ForkExecutor(max_workers=self.options.jobs) + + if self.options.jobs > 1: + for future_done_set in async_iter_completed(self._iter_tasks(loop, executor, ebuild, pkg), + max_jobs=self.options.jobs, max_load=self.options.load_average, loop=loop): + for task in (yield future_done_set): + task, results = task.result() + for result in results: + self._check_result(task, result) + + if not baddepsyntax and unknown_pkgs: + type_map = {} + for mytype, atom in unknown_pkgs: + type_map.setdefault(mytype, set()).add(atom) + for mytype, atoms in type_map.items(): + self.qatracker.add_error( + "dependency.unknown", "%s: %s: %s" + % (ebuild.relative_path, mytype, ", ".join(sorted(atoms)))) - for keyword, groups, prof in relevant_profiles: + @coroutine + def _task(self, task): + yield task.future + coroutine_return((task, task.future.result())) + + def _iter_tasks(self, loop, executor, ebuild, pkg): + for keyword, groups, prof in ebuild.relevant_profiles: is_stable_profile = prof.status == "stable" is_dev_profile = prof.status == "dev" and \ @@ -154,6 +214,22 @@ class ProfileDependsChecks(ScanBase): dep_settings.usemask = dep_settings._use_manager.getUseMask( pkg, stable=dep_settings._parent_stable) + task = types.SimpleNamespace(ebuild=ebuild, prof=prof, keyword=keyword) + + target = functools.partial(self._task_subprocess, task, pkg, dep_settings) + + if self.options.jobs <= 1: + yield (task, target()) + else: + task.future = asyncio.ensure_future(loop.run_in_executor(executor, target), loop=loop) + yield self._task(task) + + + def _task_subprocess(self, task, pkg, dep_settings): + ebuild = task.ebuild + baddepsyntax = ebuild.baddepsyntax + results = [] + prof = task.prof if not baddepsyntax: ismasked = not ebuild.archs or \ pkg.cpv not in self.portdb.xmatch("match-visible", @@ -163,7 +239,7 @@ class ProfileDependsChecks(ScanBase): self.have['pmasked'] = bool(dep_settings._getMaskAtom( pkg.cpv, ebuild.metadata)) if self.options.ignore_masked: - continue + return results # we are testing deps for a masked package; give it some lee-way suffix = "masked" matchmode = "minimum-all-ignore-profile" @@ -191,6 +267,22 @@ class ProfileDependsChecks(ScanBase): myvalue, self.portdb, dep_settings, use="all", mode=matchmode, trees=self.repo_settings.trees) + results.append(types.SimpleNamespace(atoms=atoms, success=success, mykey=mykey, mytype=mytype)) + + return results + + + def _check_result(self, task, result): + prof = task.prof + keyword = task.keyword + ebuild = task.ebuild + unknown_pkgs = ebuild.unknown_pkgs + + success = result.success + atoms = result.atoms + mykey = result.mykey + mytype = result.mytype + if success: if atoms: @@ -223,7 +315,7 @@ class ProfileDependsChecks(ScanBase): # if we emptied out our list, continue: if not all_atoms: - continue + return # Filter out duplicates. We do this by hand (rather # than use a set) so the order is stable and better @@ -255,17 +347,6 @@ class ProfileDependsChecks(ScanBase): % (ebuild.relative_path, mytype, keyword, prof, pformat(atoms, indent=6))) - if not baddepsyntax and unknown_pkgs: - type_map = {} - for mytype, atom in unknown_pkgs: - type_map.setdefault(mytype, set()).add(atom) - for mytype, atoms in type_map.items(): - self.qatracker.add_error( - "dependency.unknown", "%s: %s: %s" - % (ebuild.relative_path, mytype, ", ".join(sorted(atoms)))) - - return False - @property def runInEbuilds(self): '''Ebuild level scans''' diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1 index a6a9937e5..6f9a24544 100644 --- a/repoman/man/repoman.1 +++ b/repoman/man/repoman.1 @@ -1,4 +1,4 @@ -.TH "REPOMAN" "1" "Mar 2018" "Repoman VERSION" "Repoman" +.TH "REPOMAN" "1" "Aug 2020" "Repoman VERSION" "Repoman" .SH NAME repoman \- Gentoo's program to enforce a minimal level of quality assurance in packages added to the ebuild repository @@ -83,6 +83,13 @@ Be less verbose about extraneous info \fB-p\fR, \fB--pretend\fR Don't commit or fix anything; just show what would be done .TP +\fB\-j\fR, \fB\-\-jobs\fR +Specifies the number of jobs (processes) to run simultaneously. +.TP +\fB\-l\fR, \fB\-\-load-average\fR +Specifies that no new jobs (processes) should be started if there are others +jobs running and the load average is at least load (a floating\-point number). +.TP \fB-x\fR, \fB--xmlparse\fR Forces the metadata.xml parse check to be carried out .TP -- 2.25.3
