This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch tristan/relative-search-paths in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 926a9186a226a5c2eb19d24f47eb2592c2edb484 Author: Tristan van Berkom <[email protected]> AuthorDate: Fri Jul 24 23:37:51 2020 +0900 element.py & loader: Support searching elements using full paths This makes the following changes: * The LoadElements now keep a pointer to the resolved Elements, this allows quick resolution of Elements via the regular loader search resolution paths. * The Loader now removes the _clear_caches() code, the caches of loaded LoadElements are now persisted after the data model is loaded so that quick Element.search() can be performed. * The loader now exposes Loader.search_element() which returns a LoadElement * Element now implements Element.search() using the loader to resolve a project relative element path to a LoadElement, and returns the Element associated with the found LoadElement. This fixes #931. --- src/buildstream/_loader/loadelement.pyx | 15 +++++++++++++ src/buildstream/_loader/loader.py | 38 ++++++++++++++++----------------- src/buildstream/_loader/metaelement.py | 3 +++ src/buildstream/element.py | 22 ++++++++++++------- 4 files changed, 51 insertions(+), 27 deletions(-) diff --git a/src/buildstream/_loader/loadelement.pyx b/src/buildstream/_loader/loadelement.pyx index de2f96b..2bf1260 100644 --- a/src/buildstream/_loader/loadelement.pyx +++ b/src/buildstream/_loader/loadelement.pyx @@ -84,6 +84,7 @@ cdef class LoadElement: # TODO: if/when pyroaring exports symbols, we could type this statically cdef object _dep_cache cdef readonly list dependencies + cdef readonly object _element def __cinit__(self, MappingNode node, str filename, object loader): @@ -103,6 +104,7 @@ cdef class LoadElement: # self._loader = loader # The Loader object self._dep_cache = None # The dependency cache, to speed up depends() + self._element = None # The instantiated Element, if any # # Initialization @@ -150,6 +152,19 @@ cdef class LoadElement: def junction(self): return self._loader.project.junction + # element() + # + # The Element property stores the instantiated Element, if an + # Element has been instantiated for this LoadElement + # + @property + def element(self): + return self._element + + @element.setter + def element(self, value): + self._element = value + # depends(): # # Checks if this element depends on another element, directly diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 9f88174..d2d14c3 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -170,8 +170,6 @@ class Loader: # ret.append(loader._collect_element(element)) - self._clean_caches() - # Cache how many Elements have just been loaded if self.load_context.task: # Workaround for task potentially being None (because no State object) @@ -224,6 +222,24 @@ class Loader: return loader + # search_element() + # + # Search the project for a given LoadElement + # + # Args: + # path (str): The relative element path + # + # Returns: + # (LoadElement): The LoadElement + # + # Raises: + # (LoadError): In case an element was not found, or would have + # required loading a project that is not yet loaded. + # + def search_element(self, path): + _, filename, loader = self._parse_name(path, None, load_subprojects=False) + return loader._load_file(filename, None, load_subprojects=False) + # ancestors() # # This will traverse all active loaders in the ancestry for which this @@ -319,6 +335,7 @@ class Loader: node.get_mapping(Symbol.PUBLIC, default={}), node.get_mapping(Symbol.SANDBOX, default={}), element_kind in ("junction", "link"), + element, ) # Cache it now, make sure it's already there before recursing @@ -905,20 +922,3 @@ class Loader: ), warning_token=CoreWarnings.BAD_CHARACTERS_IN_NAME, ) - - # _clean_caches() - # - # Clean internal loader caches, recursively - # - # When loading the elements, the loaders use caches in order to not load the - # same element twice. These are kept after loading and prevent garbage - # collection. Cleaning them explicitely is required. - # - def _clean_caches(self): - for loader in self._loaders.values(): - # value may be None with nested junctions without overrides - if loader is not None: - loader._clean_caches() - - self._meta_elements = {} - self._elements = {} diff --git a/src/buildstream/_loader/metaelement.py b/src/buildstream/_loader/metaelement.py index 1c1f6fe..50d1bc5 100644 --- a/src/buildstream/_loader/metaelement.py +++ b/src/buildstream/_loader/metaelement.py @@ -39,6 +39,7 @@ class MetaElement: # public: Public domain data dictionary # sandbox: Configuration specific to the sandbox environment # first_pass: The element is to be loaded with first pass configuration (junction) + # load_element (LoadElement): A reference back to the load_element # def __init__( self, @@ -54,6 +55,7 @@ class MetaElement: public=None, sandbox=None, first_pass=False, + load_element=None, ): self.project = project self.name = name @@ -71,3 +73,4 @@ class MetaElement: self.strict_dependencies = [] self.first_pass = first_pass self.is_junction = kind in ("junction", "link") + self.load_element = load_element diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 547397e..78f80bc 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -499,21 +499,23 @@ class Element(Plugin): yield from visit(self, scope, visited) - def search(self, scope: Scope, name: str) -> Optional["Element"]: - """Search for a dependency by name + def search(self, scope: Scope, path: str) -> Optional["Element"]: + """Search for a dependency by it's element relative search path + + The search path as a relative path to the element, including + any junction names leading up to the element which might + be in a subproject. Args: scope: The scope to search - name: The dependency to search for + path: An element relative search path Returns: The dependency element, or None if not found. """ - for dep in self.dependencies(scope): - if dep.name == name: - return dep - - return None + project = self._get_project() + load_element = project.loader.search_element(path) + return load_element.element def node_subst_vars(self, node: "ScalarNode") -> str: """Replace any variables in the string contained in the node and returns it. @@ -908,6 +910,10 @@ class Element(Plugin): element = meta.project.create_element(meta, first_pass=meta.first_pass) cls.__instantiated_elements[meta] = element + # Store a reference on the originating LoadElement, allowing us + # to perform optimal Element.search(). + meta.load_element.element = element + # Instantiate sources and generate their keys for meta_source in meta.sources: meta_source.first_pass = meta.is_junction
