Support /etc/portage/binrepos.conf as a replacement for the PORTAGE_BINHOST variable. Behavior is similar to repos.conf, initially supporting just the sync-uri attribute. Both binrepos.conf and PORTAGE_BINHOST can be used simultaneously, in the same way that repos.conf and PORTDIR_OVERLAY can be used simultaneously.
The emerge --info output for binrepos.conf looks like this: Binary Repositories: example-binhost sync-uri: https://example.com/packages Bug: https://bugs.gentoo.org/668334 Signed-off-by: Zac Medico <zmed...@gentoo.org> --- lib/_emerge/actions.py | 13 ++- lib/portage/binrepo/__init__.py | 0 lib/portage/binrepo/config.py | 131 ++++++++++++++++++++++++ lib/portage/const.py | 1 + lib/portage/dbapi/bintree.py | 14 ++- lib/portage/tests/emerge/test_simple.py | 14 ++- man/make.conf.5 | 3 +- man/portage.5 | 38 +++++++ 8 files changed, 206 insertions(+), 8 deletions(-) create mode 100644 lib/portage/binrepo/__init__.py create mode 100644 lib/portage/binrepo/config.py diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index a4ecfe43d..34344f046 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -32,7 +32,8 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage import os from portage import shutil from portage import _encodings, _unicode_decode -from portage.const import _DEPCLEAN_LIB_CHECK_DEFAULT +from portage.binrepo.config import BinRepoConfigLoader +from portage.const import BINREPOS_CONF_FILE, _DEPCLEAN_LIB_CHECK_DEFAULT from portage.dbapi.dep_expand import dep_expand from portage.dbapi._expand_new_virt import expand_new_virt from portage.dbapi.IndexedPortdb import IndexedPortdb @@ -1836,6 +1837,16 @@ def action_info(settings, trees, myopts, myfiles): for repo in repos: append(repo.info_string()) + binrepos_conf_path = os.path.join(settings['PORTAGE_CONFIGROOT'], BINREPOS_CONF_FILE) + binrepos_conf = BinRepoConfigLoader((binrepos_conf_path,), settings) + if binrepos_conf and any(repo.name for repo in binrepos_conf.values()): + append("Binary Repositories:\n") + for repo in reversed(list(binrepos_conf.values())): + # Omit repos from the PORTAGE_BINHOST variable, since they + # do not have a name to label them with. + if repo.name: + append(repo.info_string()) + installed_sets = sorted(s for s in root_config.sets['selected'].getNonAtoms() if s.startswith(SETPREFIX)) if installed_sets: diff --git a/lib/portage/binrepo/__init__.py b/lib/portage/binrepo/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py new file mode 100644 index 000000000..aa3ff7a77 --- /dev/null +++ b/lib/portage/binrepo/config.py @@ -0,0 +1,131 @@ +# Copyright 2020 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +from collections import OrderedDict +from collections.abc import Mapping +from hashlib import md5 + +from portage.localization import _ +from portage.util import _recursive_file_list, writemsg +from portage.util.configparser import (SafeConfigParser, ConfigParserError, + read_configs) + + +class BinRepoConfig: + __slots__ = ( + 'name', + 'name_fallback', + 'priority', + 'sync_uri', + ) + def __init__(self, opts): + """ + Create a BinRepoConfig with options in opts. + """ + for k in self.__slots__: + setattr(self, k, opts.get(k.replace('_', '-'))) + + def info_string(self): + """ + Returns a formatted string containing informations about the repository. + Used by emerge --info. + """ + indent = " " * 4 + repo_msg = [] + repo_msg.append(self.name or self.name_fallback) + if self.priority is not None: + repo_msg.append(indent + "priority: " + str(self.priority)) + repo_msg.append(indent + "sync-uri: " + self.sync_uri) + repo_msg.append("") + return "\n".join(repo_msg) + + +class BinRepoConfigLoader(Mapping): + def __init__(self, paths, settings): + """Load config from files in paths""" + + # Defaults for value interpolation. + parser_defaults = { + "EPREFIX" : settings["EPREFIX"], + "EROOT" : settings["EROOT"], + "PORTAGE_CONFIGROOT" : settings["PORTAGE_CONFIGROOT"], + "ROOT" : settings["ROOT"], + } + + try: + parser = self._parse(paths, parser_defaults) + except ConfigParserError as e: + writemsg( + _("!!! Error while reading binrepo config file: %s\n") % e, + noiselevel=-1) + parser = SafeConfigParser(defaults=parser_defaults) + + repos = [] + sync_uris = [] + for section_name in parser.sections(): + repo_data = dict(parser[section_name].items()) + repo_data['name'] = section_name + repo = BinRepoConfig(repo_data) + if repo.sync_uri is None: + writemsg(_("!!! Missing sync-uri setting for binrepo %s\n") % (repo.name,), noiselevel=-1) + continue + + sync_uri = self._normalize_uri(repo.sync_uri) + sync_uris.append(sync_uri) + repo.sync_uri = sync_uri + if repo.priority is not None: + try: + repo.priority = int(repo.priority) + except ValueError: + repo.priority = None + repos.append(repo) + + sync_uris = set(sync_uris) + current_priority = 0 + for sync_uri in reversed(settings.get("PORTAGE_BINHOST", "").split()): + sync_uri = self._normalize_uri(sync_uri) + if sync_uri not in sync_uris: + current_priority += 1 + sync_uris.add(sync_uri) + repos.append(BinRepoConfig({ + 'name-fallback': self._digest_uri(sync_uri), + 'name': None, + 'priority': current_priority, + 'sync-uri': sync_uri, + })) + + self._data = OrderedDict((repo.name, repo) for repo in + sorted(repos, key=lambda repo: (repo.priority or 0, repo.name or repo.name_fallback))) + + @staticmethod + def _digest_uri(uri): + return md5(uri.encode('utf_8')).hexdigest() + + @staticmethod + def _normalize_uri(uri): + return uri.rstrip('/') + + @staticmethod + def _parse(paths, defaults): + parser = SafeConfigParser(defaults=defaults) + recursive_paths = [] + for p in paths: + if isinstance(p, str): + recursive_paths.extend(_recursive_file_list(p)) + else: + recursive_paths.append(p) + + read_configs(parser, recursive_paths) + return parser + + def __iter__(self): + return iter(self._data) + + def __contains__(self, key): + return key in self._data + + def __getitem__(self, key): + return self._data[key] + + def __len__(self): + return len(self._data) diff --git a/lib/portage/const.py b/lib/portage/const.py index 9a7ea23bd..b895f0fa9 100644 --- a/lib/portage/const.py +++ b/lib/portage/const.py @@ -28,6 +28,7 @@ import os # variables used with config_root (these need to be relative) USER_CONFIG_PATH = "etc/portage" +BINREPOS_CONF_FILE = USER_CONFIG_PATH + "/binrepos.conf" MAKE_CONF_FILE = USER_CONFIG_PATH + "/make.conf" MODULES_FILE_PATH = USER_CONFIG_PATH + "/modules" CUSTOM_PROFILE_PATH = USER_CONFIG_PATH + "/profile" diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 620865a79..a96183561 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -22,8 +22,9 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.versions:best,catpkgsplit,catsplit,_pkg_str', ) +from portage.binrepo.config import BinRepoConfigLoader from portage.cache.mappings import slot_dict_class -from portage.const import CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS +from portage.const import BINREPOS_CONF_FILE, CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS from portage.dbapi.virtual import fakedbapi from portage.dep import Atom, use_reduce, paren_enclose from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \ @@ -364,6 +365,7 @@ class binarytree: self.move_slot_ent = self.dbapi.move_slot_ent self.populated = 0 self.tree = {} + self._binrepos_conf = None self._remote_has_index = False self._remotepkgs = None # remote metadata indexed by cpv self._additional_pkgs = {} @@ -628,8 +630,10 @@ class binarytree: self._populate_additional(add_repos) if getbinpkgs: - if not self.settings.get("PORTAGE_BINHOST"): - writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"), + config_path = os.path.join(self.settings['PORTAGE_CONFIGROOT'], BINREPOS_CONF_FILE) + self._binrepos_conf = BinRepoConfigLoader((config_path,), self.settings) + if not self._binrepos_conf: + writemsg(_("!!! %s is missing (or PORTAGE_BINHOST is unset), but use is requested.\n") % (config_path,), noiselevel=-1) else: self._populate_remote(getbinpkg_refresh=getbinpkg_refresh) @@ -903,7 +907,9 @@ class binarytree: self._remote_has_index = False self._remotepkgs = {} - for base_url in self.settings["PORTAGE_BINHOST"].split(): + # Order by descending priority. + for repo in reversed(list(self._binrepos_conf.values())): + base_url = repo.sync_uri parsed_url = urlparse(base_url) host = parsed_url.netloc port = parsed_url.port diff --git a/lib/portage/tests/emerge/test_simple.py b/lib/portage/tests/emerge/test_simple.py index c24f5c603..8635b70e4 100644 --- a/lib/portage/tests/emerge/test_simple.py +++ b/lib/portage/tests/emerge/test_simple.py @@ -7,7 +7,7 @@ import sys import portage from portage import shutil, os from portage import _unicode_decode -from portage.const import (BASH_BINARY, PORTAGE_PYM_PATH, USER_CONFIG_PATH) +from portage.const import (BASH_BINARY, BINREPOS_CONF_FILE, PORTAGE_PYM_PATH, USER_CONFIG_PATH) from portage.cache.mappings import Mapping from portage.process import find_binary from portage.tests import TestCase @@ -419,13 +419,23 @@ call_has_and_best_version() { ) # Test binhost support if FETCHCOMMAND is available. + binrepos_conf_file = os.path.join(os.sep, eprefix, BINREPOS_CONF_FILE) + with open(binrepos_conf_file, 'wt') as f: + f.write('[test-binhost]\n') + f.write('sync-uri = {}\n'.format(binhost_uri)) fetchcommand = portage.util.shlex_split(playground.settings['FETCHCOMMAND']) fetch_bin = portage.process.find_binary(fetchcommand[0]) if fetch_bin is not None: test_commands = test_commands + ( + lambda: os.rename(pkgdir, binhost_dir), + emerge_cmd + ("-e", "--getbinpkgonly", "dev-libs/A"), + lambda: shutil.rmtree(pkgdir), + lambda: os.rename(binhost_dir, pkgdir), + # Remove binrepos.conf and test PORTAGE_BINHOST. + lambda: os.unlink(binrepos_conf_file), lambda: os.rename(pkgdir, binhost_dir), ({"PORTAGE_BINHOST": binhost_uri},) + \ - emerge_cmd + ("-e", "--getbinpkgonly", "dev-libs/A"), + emerge_cmd + ("-fe", "--getbinpkgonly", "dev-libs/A"), lambda: shutil.rmtree(pkgdir), lambda: os.rename(binhost_dir, pkgdir), ) diff --git a/man/make.conf.5 b/man/make.conf.5 index eb812150f..f7b2460e5 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -853,7 +853,8 @@ Each entry in the list must specify the full address of a directory serving tbz2's for your system (this directory must contain a 'Packages' index file). This is only used when running with the get binary pkg options are given to \fBemerge\fR. Review \fBemerge\fR(1) -for more information. +for more information. The \fBPORTAGE_BINHOST\fR variable is deprecated in +favor of the \fBbinrepos.conf\fR configuration file (see \fBportage\fR(5)). .TP \fBPORTAGE_BINHOST_HEADER_URI\fR = \ \fI"ftp://login:p...@grp.mirror.site/pub/grp/i686/athlon\-xp/"\fR diff --git a/man/portage.5 b/man/portage.5 index 7472972cc..890a22adb 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -47,6 +47,7 @@ virtuals .BR /etc/portage/ .nf bashrc +binrepos.conf categories color.map license_groups @@ -620,6 +621,43 @@ any other bash script. Additional package-specific bashrc files can be created in /etc/portage/env. .TP +.BR binrepos.conf +Specifies remote binary package repository configuration information. This +is intended to be used as a replacement for the \fBmake.conf\fR(5) +\fBPORTAGE_BINHOST\fR variable. + +.I Format: +.nf +\- comments begin with # (no inline comments) +\- configuration of each repository is specified in a section starting with \ +"[${repository_name}]" +\- attributes are specified in "${attribute} = ${value}" format +.fi + +.RS +.I Attributes supported in sections of repositories: +.RS +.TP +.B priority +Specifies priority of given repository. When a package exists in multiple +repositories, those with higher priority are preferred. +.TP +.B sync\-uri +Specifies URI of repository used for `emerge \-\-getbinpkg`. +.RE +.RE + +.I Example: +.nf +[example-binhost] +# repos with higher priorities are preferred when packages with equal +# versions are found in multiple repos +priority = 9999 +# packages are fetched from here +sync-uri = https://example.com/binhost + +.fi +.TP .BR categories A simple list of valid categories that may be used in repositories and PKGDIR (see \fBmake.conf\fR(5)). This allows for custom categories to be created. -- 2.25.3