This is an automated email from the ASF dual-hosted git repository. juergbi pushed a commit to branch juerg/junction-aliases in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit fb8435b2f1090bd3c3960210338cf6f657e49571 Author: Jürg Billeter <[email protected]> AuthorDate: Fri Mar 22 15:17:06 2024 +0100 Support mapping subproject aliases to aliases of the parent project --- doc/source/format_project.rst | 35 ++++++++++++++ src/buildstream/_project.py | 71 ++++++++++++++++++++++++++-- src/buildstream/plugins/elements/junction.py | 10 +++- src/buildstream/source.py | 6 +-- 4 files changed, 113 insertions(+), 9 deletions(-) diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst index 83eac5ea0..cc56e4b35 100644 --- a/doc/source/format_project.rst +++ b/doc/source/format_project.rst @@ -204,6 +204,9 @@ URLs which are to be used in the individual ``.bst`` files. foo: git://git.foo.org/ bar: http://bar.com/downloads/ +If you want this project's alias definitions to also be used for subprojects, +see :ref:`Mapping source aliases of subprojects <project_junctions_source_aliases>`. + Sandbox options ~~~~~~~~~~~~~~~ @@ -319,6 +322,9 @@ The mirrors can be overridden on a per project basis using be used first in the :ref:`user configuration <config_default_mirror>`, or using the :ref:`--default-mirror <invoking_bst>` command-line argument. +If you want this project's mirrors to also be used for subprojects, +see :ref:`Mapping source aliases of subprojects <project_junctions_source_aliases>`. + .. _project_plugins: @@ -989,6 +995,35 @@ subproject. will not accrue runtime dependencies on elements in your *internal* subproject. +.. _project_junctions_source_aliases: + +Mapping source aliases of subprojects +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:mod:`junction <elements.junction>` elements allow source aliases of subprojects +to be mapped to aliases of the parent project. This makes it possible to control +the translation of aliases to URLs including mirror configuration across multiple +project levels. + +To ensure that there are mappings for all aliases of all subprojects, you can set the +``disallow-subproject-uris`` flag in the ``junctions`` group here in ``project.conf``. + +top-level + +.. code:: yaml + + junctions: + disallow-subproject-uris: True + +This will raise an error if an alias without a mapping is encountered. This flag +is applied recursively across all junctions. + +It also configures ``unaliased-url`` as a fatal warning in all subprojects to +ensure that the current project is in full control over all source URLs. +As the fatal warning configuration contributes to the cache key, this flag will +affect the cache key of subprojects that haven't already configured +``unaliased-url`` as a fatal warning. + + .. _project_defaults: Element default configuration diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index ac57a2523..5bdcdd383 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -40,10 +40,12 @@ from ._includes import Includes from ._workspaces import WORKSPACE_PROJECT_FILE from ._remotespec import RemoteSpec from .sourcemirror import SourceMirror +from .source import SourceError if TYPE_CHECKING: from ._context import Context + from .plugins.elements.junction import JunctionElement # Project Configuration file @@ -82,7 +84,7 @@ class Project: directory: Optional[str], context: "Context", *, - junction: Optional[object] = None, + junction: Optional["JunctionElement"] = None, cli_options: Optional[Dict[str, str]] = None, default_mirror: Optional[str] = None, parent_loader: Optional[Loader] = None, @@ -98,11 +100,12 @@ class Project: self.load_context: LoadContext # The LoadContext self.loader: Optional[Loader] = None # The loader associated to this project - self.junction: Optional[object] = junction # The junction Element object, if this is a subproject + self.junction: Optional["JunctionElement"] = junction # The junction Element object, if this is a subproject self.ref_storage: Optional[ProjectRefStorage] = None # Where to store source refs self.refs: Optional[ProjectRefs] = None self.junction_refs: Optional[ProjectRefs] = None + self.disallow_subproject_uris: bool = False self.config: ProjectConfig = ProjectConfig() self.first_pass_config: ProjectConfig = ProjectConfig() @@ -234,12 +237,28 @@ class Project: # This method is provided for :class:`.Source` objects to resolve # fully qualified urls based on the shorthand which is allowed # to be specified in the YAML - def translate_url(self, url, *, first_pass=False): + def translate_url(self, url, *, source, first_pass=False): if url and utils._ALIAS_SEPARATOR in url: url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1) alias_url = self.get_alias_url(url_alias, first_pass=first_pass) if alias_url: + if self.junction: + parent_project = self.junction._get_project() + parent_alias = self.junction.aliases.get_str(url_alias, default=None) + if parent_alias: + # Delegate translation to parent project + return parent_project.translate_url( + parent_alias + utils._ALIAS_SEPARATOR + url_body, source=source, first_pass=first_pass + ) + elif parent_project.disallow_subproject_uris: + raise SourceError( + "{}: Parent project did not provide a mapping for alias '{}' and disallowed usage of unmapped aliases".format( + source, url_alias + ), + reason="missing-alias-mapping", + ) + url = alias_url + url_body return url @@ -391,12 +410,33 @@ class Project: # Returns: # bool: Whether the alias is declared in the scope of this project # - def alias_exists(self, alias, *, first_pass=False): + def alias_exists(self, alias, *, source, first_pass=False): if first_pass: config = self.first_pass_config else: config = self.config + if self.junction: + parent_project = self.junction._get_project() + parent_alias = self.junction.aliases.get_str(alias, default=None) + if parent_alias: + if parent_project.alias_exists(parent_alias, source=source, first_pass=first_pass): + return True + else: + raise SourceError( + "{}: Mapped alias '{}' for subproject alias '{}' is invalid in the parent project".format( + self.junction, parent_alias, alias + ), + reason="invalid-source-alias", + ) + elif parent_project.disallow_subproject_uris: + raise SourceError( + "{}: Parent project did not provide a mapping for alias '{}' and disallowed usage of unmapped aliases".format( + source, alias + ), + reason="missing-alias-mapping", + ) + return config._aliases.get_str(alias, default=None) is not None # get_alias_uris() @@ -423,6 +463,15 @@ class Project: if not alias or alias not in config._aliases: # pylint: disable=unsupported-membership-test return [None] + if self.junction: + parent_project = self.junction._get_project() + parent_alias = self.junction.aliases.get_str(alias, default=None) + if parent_alias: + # Delegate translation to parent project + return parent_project.get_alias_uris(parent_alias, first_pass=first_pass, tracking=tracking) + elif parent_project.disallow_subproject_uris: + return [None] + uri_list: List[Union[SourceMirror, str]] = [] policy = self._context.track_source if tracking else self._context.fetch_source @@ -814,7 +863,19 @@ class Project: # Junction configuration junctions_node = pre_config_node.get_mapping("junctions", default={}) - junctions_node.validate_keys(["duplicates", "internal"]) + junctions_node.validate_keys(["duplicates", "internal", "disallow-subproject-uris"]) + + if self.junction and self.junction._get_project().disallow_subproject_uris: + # If the parent project doesn't allow subproject URIs, this must + # be enforced for nested subprojects as well. + self.disallow_subproject_uris = True + + # The `disallow-subproject-uris` flag also implies fatal `unaliased-url` in subprojects + # to ensure no subproject URIs escape the parent project's control. + if CoreWarnings.UNALIASED_URL not in self._fatal_warnings: + self._fatal_warnings.append(CoreWarnings.UNALIASED_URL) + else: + self.disallow_subproject_uris = junctions_node.get_bool("disallow-subproject-uris", default=False) # Parse duplicates junction_duplicates = junctions_node.get_mapping("duplicates", default={}) diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py index d35ed6317..5d457143f 100644 --- a/src/buildstream/plugins/elements/junction.py +++ b/src/buildstream/plugins/elements/junction.py @@ -50,6 +50,11 @@ Overview overrides: subproject-junction.bst: local-junction.bst + # Optionally override aliases in subprojects, to allow using mirrors + # defined in the parent project. + aliases: + subproject-alias: local-alias + With a junction element in place, local elements can depend on elements in the other BuildStream project using :ref:`element paths <format_element_names>`. For example, if you have a ``toolchain.bst`` junction element referring to @@ -338,7 +343,7 @@ class JunctionElement(Element): def configure(self, node): - node.validate_keys(["path", "options", "overrides"]) + node.validate_keys(["path", "options", "overrides", "aliases"]) self.path = node.get_str("path", default="") self.options = node.get_mapping("options", default={}) @@ -361,6 +366,9 @@ class JunctionElement(Element): ) self.overrides[key] = junction_name + # Map from subproject alias to local alias + self.aliases = node.get_mapping("aliases", default={}) + def preflight(self): pass diff --git a/src/buildstream/source.py b/src/buildstream/source.py index 96873a7a2..ef193cd37 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -768,7 +768,7 @@ class Source(Plugin): alias=url_alias, alias_url=project_alias_url, source_url=url_body, extra_data=extra_data ) else: - return project.translate_url(url, first_pass=self.__first_pass) + return project.translate_url(url, source=self, first_pass=self.__first_pass) def mark_download_url(self, url: str, *, primary: bool = True) -> None: """Identifies the URL that this Source uses to download @@ -830,7 +830,7 @@ class Source(Plugin): # If there is an alias in use, ensure that it exists in the project if alias: project = self._get_project() - if not project.alias_exists(alias, first_pass=self.__first_pass): + if not project.alias_exists(alias, first_pass=self.__first_pass, source=self): raise SourceError( "{}: Invalid alias '{}' specified in URL: {}".format(self, alias, url), reason="invalid-source-alias", @@ -1287,7 +1287,7 @@ class Source(Plugin): def _get_alias(self): alias = self.__expected_alias project = self._get_project() - if project.alias_exists(alias, first_pass=self.__first_pass): + if project.alias_exists(alias, first_pass=self.__first_pass, source=self): # The alias must already be defined in the project's aliases # otherwise http://foo gets treated like it contains an alias return alias
