This is an automated email from the ASF dual-hosted git repository. akitouni pushed a commit to branch abderrahim/buildstream-mirrors-merge in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit a07ce04a0b3e4ab334dfaf83cb9cdffdde4a5bed Author: Tristan van Berkom <[email protected]> AuthorDate: Mon Oct 2 14:16:04 2023 +0900 _project.py: Use SourceMirror objects --- src/buildstream/_pluginfactory/pluginfactory.py | 2 + src/buildstream/_project.py | 77 ++++++++++++++------ src/buildstream/_site.py | 3 + src/buildstream/source.py | 93 +++++++++++++++++++------ 4 files changed, 130 insertions(+), 45 deletions(-) diff --git a/src/buildstream/_pluginfactory/pluginfactory.py b/src/buildstream/_pluginfactory/pluginfactory.py index de3223085..6052684bd 100644 --- a/src/buildstream/_pluginfactory/pluginfactory.py +++ b/src/buildstream/_pluginfactory/pluginfactory.py @@ -84,6 +84,8 @@ class PluginFactory: self._site_plugins_path = _site.source_plugins elif self._plugin_type == PluginType.ELEMENT: self._site_plugins_path = _site.element_plugins + elif self._plugin_type == PluginType.SOURCE_MIRROR: + self._site_plugins_path = _site.source_mirror_plugins self._site_source = self._plugin_base.make_plugin_source( searchpath=[self._site_plugins_path], diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index 1a1054898..8e114c828 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -15,7 +15,7 @@ # Tristan Van Berkom <[email protected]> # Tiago Gomes <[email protected]> -from typing import TYPE_CHECKING, Optional, Dict, Union, List +from typing import TYPE_CHECKING, Optional, Dict, Union, List, Tuple import os import urllib.parse @@ -31,14 +31,15 @@ from ._exceptions import LoadError from .exceptions import LoadErrorReason from ._options import OptionPool from .node import ScalarNode, MappingNode, ProvenanceInformation, _assert_symbol_name -from ._pluginfactory import ElementFactory, SourceFactory, load_plugin_origin -from .types import CoreWarnings, _HostMount, _SourceMirror, _SourceUriPolicy +from ._pluginfactory import ElementFactory, SourceFactory, SourceMirrorFactory, load_plugin_origin +from .types import CoreWarnings, _HostMount, _SourceUriPolicy from ._projectrefs import ProjectRefs, ProjectRefStorage from ._loader import Loader, LoadContext from .element import Element from ._includes import Includes from ._workspaces import WORKSPACE_PROJECT_FILE from ._remotespec import RemoteSpec +from .sourcemirror import SourceMirror if TYPE_CHECKING: @@ -56,7 +57,7 @@ class ProjectConfig: self.base_variables = {} # The base set of variables self.element_overrides = {} # Element specific configurations self.source_overrides = {} # Source specific configurations - self.mirrors = {} # Dictionary of _SourceAlias objects + self.mirrors = {} # Dictionary of SourceMirror objects self.default_mirror = None # The name of the preferred mirror. self._aliases = None # Aliases dictionary @@ -115,6 +116,7 @@ class Project: self.element_factory: Optional[ElementFactory] = None # ElementFactory for loading elements self.source_factory: Optional[SourceFactory] = None # SourceFactory for loading sources + self.source_mirror_factory: Optional[SourceMirrorFactory] = None # SourceMirrorFactory self.sandbox: Optional[MappingNode] = None self.splits: Optional[MappingNode] = None @@ -195,6 +197,28 @@ class Project: # Public Methods # ######################################################## + # get_alias_url(): + # + # Fetch the value of a URL alias + # + # Args: + # alias: The alias + # first_pass: Whether to use first pass configuration (for junctions) + # + # Returns: + # The alias substitution + # + def get_alias_url(self, alias: str, *, first_pass: bool = False) -> Optional[str]: + if first_pass: + config = self.first_pass_config + else: + config = self.config + + if config._aliases: + return config._aliases.get_str(alias, default=None) + + return None + # translate_url(): # # Translates the given url which may be specified with an alias @@ -211,14 +235,10 @@ class Project: # fully qualified urls based on the shorthand which is allowed # to be specified in the YAML def translate_url(self, url, *, first_pass=False): - if first_pass: - config = self.first_pass_config - else: - config = self.config if url and utils._ALIAS_SEPARATOR in url: url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1) - alias_url = config._aliases.get_str(url_alias, default=None) + alias_url = self.get_alias_url(url_alias) if alias_url: url = alias_url + url_body @@ -382,35 +402,45 @@ class Project: # get_alias_uris() # # Args: - # alias (str): The alias. - # first_pass (bool): Whether to use first pass configuration (for junctions) - # tracking (bool): Whether we want the aliases for tracking (otherwise assume fetching) + # alias: The alias. + # first_pass: Whether to use first pass configuration (for junctions) + # tracking: Whether we want the aliases for tracking (otherwise assume fetching) + # + # Returns: + # A list of (SourceMirror, string) tuples with each alias substitution found along + # with the SourceMirror object which produced it. # - # Returns a list of every URI to replace an alias with - def get_alias_uris(self, alias, *, first_pass=False, tracking=False): + def get_alias_uris( + self, alias: str, *, first_pass: bool = False, tracking: bool = False + ) -> List[Tuple[Optional[SourceMirror], Optional[str]]]: + if first_pass: config = self.first_pass_config else: config = self.config if not alias or alias not in config._aliases: # pylint: disable=unsupported-membership-test - return [None] + return [(None, None)] - uri_list = [] + uri_list: List[Tuple[Optional[SourceMirror], Optional[str]]] = [] policy = self._context.track_source if tracking else self._context.fetch_source if policy in (_SourceUriPolicy.ALL, _SourceUriPolicy.MIRRORS) or ( policy == _SourceUriPolicy.USER and self._mirror_override ): for mirror_name, mirror in config.mirrors.items(): - if alias in mirror.aliases: + mirror_uri_list = mirror._get_alias_uris(alias) + if mirror_uri_list: + + list_to_add = [(mirror, uri) for uri in mirror_uri_list] + if mirror_name == config.default_mirror: - uri_list = mirror.aliases[alias] + uri_list + uri_list = list_to_add + uri_list else: - uri_list += mirror.aliases[alias] + uri_list += list_to_add if policy in (_SourceUriPolicy.ALL, _SourceUriPolicy.ALIASES): - uri_list.append(config._aliases.get_str(alias)) + uri_list.append((None, config._aliases.get_str(alias))) return uri_list @@ -1025,9 +1055,9 @@ class Project: # even if the mirrors are specified in user configuration. variables.expand(mirrors_node) - # Collect _SourceMirror objects + # Collect SourceMirror objects for mirror_node in mirrors_node: - mirror = _SourceMirror.new_from_node(mirror_node) + mirror = self.source_mirror_factory.create(self._context, self, mirror_node) output.mirrors[mirror.name] = mirror if not output.default_mirror: output.default_mirror = mirror.name @@ -1087,6 +1117,7 @@ class Project: pluginbase = PluginBase(package="buildstream.plugins") self.element_factory = ElementFactory(pluginbase) self.source_factory = SourceFactory(pluginbase) + self.source_mirror_factory = SourceMirrorFactory(pluginbase) # Load the plugin origins and register them to their factories origins = config.get_sequence("plugins", default=[]) @@ -1096,6 +1127,8 @@ class Project: self.element_factory.register_plugin_origin(kind, origin, conf.allow_deprecated) for kind, conf in origin.sources.items(): self.source_factory.register_plugin_origin(kind, origin, conf.allow_deprecated) + for kind, conf in origin.source_mirrors.items(): + self.source_mirror_factory.register_plugin_origin(kind, origin, conf.allow_deprecated) # _warning_is_fatal(): # diff --git a/src/buildstream/_site.py b/src/buildstream/_site.py index a75faaa7d..c548d7ad9 100644 --- a/src/buildstream/_site.py +++ b/src/buildstream/_site.py @@ -30,6 +30,9 @@ element_plugins = os.path.join(root, "plugins", "elements") # The Source plugin directory source_plugins = os.path.join(root, "plugins", "sources") +# The SourceMirror plugin directory +source_mirror_plugins = os.path.join(root, "plugins", "sourcemirrors") + # Default user configuration default_user_config = os.path.join(root, "data", "userconfig.yaml") diff --git a/src/buildstream/source.py b/src/buildstream/source.py index e842318d0..b9911bba8 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -232,6 +232,7 @@ from .storage import CasBasedDirectory from .storage import FileBasedDirectory from .storage.directory import Directory from ._variables import Variables +from .sourcemirror import SourceMirror if TYPE_CHECKING: from typing import Any, Dict, Set @@ -388,8 +389,9 @@ class Source(Plugin): meta: MetaSource, variables: Variables, *, + active_mirror: Optional[SourceMirror] = None, alias_override: Optional[Tuple[str, str]] = None, - unique_id: Optional[int] = None + unique_id: Optional[int] = None, ): # Set element_name member before parent init, as needed for debug messaging self.__element_name = meta.element_name # The name of the element owning this source @@ -415,6 +417,9 @@ class Source(Plugin): # Set of marked download URLs self.__marked_urls = set() # type: Set[str] + # The active SourceMirror in context of fetch/track + self.__active_mirror: Optional[SourceMirror] = active_mirror + # Collect the composited element configuration and # ask the element to configure itself. self.__init_defaults(project, meta) @@ -721,31 +726,58 @@ class Source(Plugin): the URL was previously marked with :func:`Source.mark_download_url() <buildstream.source.Source.mark_download_url>` at :func:`Plugin.configure() <buildstream.plugin.Plugin.configure>` time. """ + project = self._get_project() + # Ensure that the download URL is also marked self.mark_download_url(url, primary=primary) if suffix: url = url + suffix + # Here we are called either: + # * From a source fetcher with "alias_override" specified + # * From a cloned source with self.__alias_override specified + # * From the default source, with a possibly aliased source + # * In this last case, mirrors are ignored + # + # In both relevant cases, we need to have the SourceMirror in context, and + # then we can delegate the translation to the SourceMirror while providing + # the original "url" to the SourceMirror method. + # + # Use the `self.__active_mirror` object + # Alias overriding can happen explicitly (by command-line) or # implicitly (the Source being constructed with an __alias_override). - if alias_override or self.__alias_override: + # + if self.__active_mirror is not None: + + assert alias_override or self.__alias_override + url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1) - if url_alias: - if alias_override: - url = alias_override + url_body - else: - # Implicit alias overrides may only be done for one - # specific alias, so that sources that fetch from multiple - # URLs and use different aliases default to only overriding - # one alias, rather than getting confused. - override_alias = self.__alias_override[0] # type: ignore - override_url = self.__alias_override[1] # type: ignore - if url_alias == override_alias: - url = override_url + url_body - return url + project_alias_url = project.get_alias_url(url_alias, first_pass=self.__first_pass) + + if self.__alias_override is not None: + override_alias = self.__alias_override[0] # type: ignore + override_url = self.__alias_override[1] # type: ignore + + # Implicit alias overrides may only be done for one + # specific alias, so that sources that fetch from multiple + # URLs and use different aliases default to only overriding + # one alias, rather than getting confused. + # + if url_alias != override_alias: + return url + + alias_override = override_url + + # + # Delegate the URL translation to the SourceMirror plugin + # + return self.__active_mirror.translate_url( + project.name, url_alias, project_alias_url, alias_override, url_body + ) + else: - project = self._get_project() return project.translate_url(url, first_pass=self.__first_pass) def mark_download_url(self, url: str, *, primary: bool = True) -> None: @@ -1345,7 +1377,7 @@ class Source(Plugin): # primary with either mark_download_url() or # translate_url(). # - def __clone_for_uri(self, uri): + def __clone_for_uri(self, mirror, uri): project = self._get_project() context = self._get_context() alias = self._get_alias() @@ -1363,7 +1395,13 @@ class Source(Plugin): ) clone = source_kind( - context, project, meta, self.__variables, alias_override=(alias, uri), unique_id=self._unique_id + context, + project, + meta, + self.__variables, + active_mirror=mirror, + alias_override=(alias, uri), + unique_id=self._unique_id, ) # Do the necessary post instantiation routines here @@ -1406,7 +1444,10 @@ class Source(Plugin): alias = fetcher._get_alias() last_error = None - for uri in project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False): + for mirror, uri in project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False): + + self.__active_mirror = mirror + try: fetcher.fetch(uri) # FIXME: Need to consider temporary vs. permanent failures, @@ -1416,10 +1457,12 @@ class Source(Plugin): continue # No error, we're done with this fetcher + self.__active_mirror = None break else: # No break occurred, raise the last detected error + self.__active_mirror = None raise last_error # Default codepath is to reinstantiate the Source @@ -1435,8 +1478,9 @@ class Source(Plugin): return last_error = None - for uri in project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False): - new_source = self.__clone_for_uri(uri) + for mirror, uri in project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False): + + new_source = self.__clone_for_uri(mirror, uri) try: new_source.fetch(**kwargs) # FIXME: Need to consider temporary vs. permanent failures, @@ -1466,8 +1510,9 @@ class Source(Plugin): # NOTE: We are assuming here that tracking only requires substituting the # first alias used last_error = None - for uri in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=True)): - new_source = self.__clone_for_uri(uri) + for mirror, uri in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=True)): + + new_source = self.__clone_for_uri(mirror, uri) try: ref = new_source.track(**kwargs) # pylint: disable=assignment-from-none # FIXME: Need to consider temporary vs. permanent failures, @@ -1475,7 +1520,9 @@ class Source(Plugin): except BstError as e: last_error = e continue + return ref + raise last_error @classmethod
