This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch tristan/junction-source-mirrors in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f76e79d122a8f1a0fa3de7e2b097dc42919df4d0 Author: Tristan van Berkom <[email protected]> AuthorDate: Sat Jul 6 18:59:07 2024 +0900 Support loading source mirror plugins from pip and junction origins. This patch does the following: * Handles the new PluginType.SOURCE_MIRROR in both junction and pip origins. * Makes error reporting regarding unfound plugins a bit more consistent between junction and pip origins. * Passes Stream.fetch_subprojects() to the Project() constructor, instead of setting this in Stream.set_project() This last bit was necessary since we now encounter a situation where the Project will want to fetch subprojects within it's initial load phase, before the toplevel Project constructor returns. --- src/buildstream/_frontend/app.py | 1 + .../_pluginfactory/pluginoriginjunction.py | 4 ++- src/buildstream/_pluginfactory/pluginoriginpip.py | 6 ++-- src/buildstream/_project.py | 2 ++ src/buildstream/_stream.py | 32 +++++++++++----------- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py index a6b08cfdd..577d80d4d 100644 --- a/src/buildstream/_frontend/app.py +++ b/src/buildstream/_frontend/app.py @@ -282,6 +282,7 @@ class App: self.context, cli_options=self._main_options["option"], default_mirror=self._main_options.get("default_mirror"), + fetch_subprojects=self.stream.fetch_subprojects, ) except LoadError as e: diff --git a/src/buildstream/_pluginfactory/pluginoriginjunction.py b/src/buildstream/_pluginfactory/pluginoriginjunction.py index 1f719df01..4c5485cbb 100644 --- a/src/buildstream/_pluginfactory/pluginoriginjunction.py +++ b/src/buildstream/_pluginfactory/pluginoriginjunction.py @@ -41,6 +41,8 @@ class PluginOriginJunction(PluginOrigin): factory = project.source_factory elif plugin_type == PluginType.ELEMENT: factory = project.element_factory + elif plugin_type == PluginType.SOURCE_MIRROR: + factory = project.source_mirror_factory # Now ask for the paths from the subproject PluginFactory try: @@ -64,7 +66,7 @@ class PluginOriginJunction(PluginOrigin): # subproject. # raise PluginError( - "{}: project '{}' referred to by junction '{}' does not declare any {} plugin kind: '{}'".format( + "{}: project '{}' referred to by junction '{}' does not declare a {} plugin named: '{}'".format( self.provenance_node.get_provenance(), project.name, self._junction, plugin_type, kind ), reason="junction-plugin-not-found", diff --git a/src/buildstream/_pluginfactory/pluginoriginpip.py b/src/buildstream/_pluginfactory/pluginoriginpip.py index 00ac58e96..511e43f36 100644 --- a/src/buildstream/_pluginfactory/pluginoriginpip.py +++ b/src/buildstream/_pluginfactory/pluginoriginpip.py @@ -41,6 +41,8 @@ class PluginOriginPip(PluginOrigin): entrypoint_group = "buildstream.plugins.sources" elif plugin_type == PluginType.ELEMENT: entrypoint_group = "buildstream.plugins.elements" + elif plugin_type == PluginType.SOURCE_MIRROR: + entrypoint_group = "buildstream.plugins.sourcemirrors" # key by a tuple to avoid collision try: @@ -75,8 +77,8 @@ class PluginOriginPip(PluginOrigin): if package is None: raise PluginError( - "{}: Pip package {} does not contain a plugin named '{}'".format( - self.provenance_node.get_provenance(), self._package_name, kind + "{}: Pip package {} does not contain a {} plugin named '{}'".format( + self.provenance_node.get_provenance(), self._package_name, plugin_type, kind ), reason="plugin-not-found", ) diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index c3d8aa9ff..984f2f648 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -90,6 +90,7 @@ class Project: parent_loader: Optional[Loader] = None, provenance_node: Optional[ProvenanceInformation] = None, search_for_project: bool = True, + fetch_subprojects=None ): # # Public members @@ -161,6 +162,7 @@ class Project: self.load_context = parent_loader.load_context else: self.load_context = LoadContext(self._context) + self.load_context.set_fetch_subprojects(fetch_subprojects) if search_for_project: self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 973e58e0a..2266b27f0 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -127,8 +127,6 @@ class Stream: def set_project(self, project): assert self._project is None self._project = project - if self._project: - self._project.load_context.set_fetch_subprojects(self._fetch_subprojects) # load_selection() # @@ -1323,6 +1321,22 @@ class Stream: assert queue queue.enqueue([element]) + # fetch_subprojects() + # + # Fetch subprojects as part of the project and element loading process. + # + # This is passed to the Project in order to handle loading of subprojects + # + # Args: + # junctions (list of Element): The junctions to fetch + # + def fetch_subprojects(self, junctions): + self._reset() + queue = FetchQueue(self._scheduler) + queue.enqueue(junctions) + self.queues = [queue] + self._run() + ############################################################# # Private Methods # ############################################################# @@ -1343,20 +1357,6 @@ class Stream: message, detail="No project.conf or active workspace was located", reason="project-not-loaded" ) - # _fetch_subprojects() - # - # Fetch subprojects as part of the project and element loading process. - # - # Args: - # junctions (list of Element): The junctions to fetch - # - def _fetch_subprojects(self, junctions): - self._reset() - queue = FetchQueue(self._scheduler) - queue.enqueue(junctions) - self.queues = [queue] - self._run() - # _load_artifacts() # # Loads artifacts from target artifact refs
