This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch bschubert/optimize-element-init in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit efd9d179916540c1aa10c5458126d90caf882d5a Author: Benjamin Schubert <[email protected]> AuthorDate: Thu Oct 10 15:19:37 2019 +0100 element.py: Extract __expand_splits to a cython module __expand_splits takes roughly 4% of our runtime in our benchmarks. Moving it to cython allows us to regain part of this time and speeds up all buildstream's invocations. - export _variables symbols for cython - introduce _element.pyx for cython extension of element.py --- setup.py | 1 + src/buildstream/_element.pyx | 37 +++++++++++++++++++++++++++++++++++++ src/buildstream/_variables.pxd | 26 ++++++++++++++++++++++++++ src/buildstream/_variables.pyx | 6 +----- src/buildstream/element.py | 18 ++---------------- 5 files changed, 67 insertions(+), 21 deletions(-) diff --git a/setup.py b/setup.py index 19779fb..e0d3dcd 100755 --- a/setup.py +++ b/setup.py @@ -403,6 +403,7 @@ def register_cython_module(module_name, dependencies=None): BUILD_EXTENSIONS = [] register_cython_module("buildstream.node") +register_cython_module("buildstream._element", dependencies=["buildstream.node", "buildstream._variables"]) register_cython_module("buildstream._loader._loader") register_cython_module("buildstream._loader.loadelement") register_cython_module("buildstream._loader.types", dependencies=["buildstream.node"]) diff --git a/src/buildstream/_element.pyx b/src/buildstream/_element.pyx new file mode 100644 index 0000000..f4e2011 --- /dev/null +++ b/src/buildstream/_element.pyx @@ -0,0 +1,37 @@ +# +# Copyright (C) 2018 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +from .node cimport MappingNode, SequenceNode +from ._variables cimport Variables + + +# Expand the splits in the public data using the Variables in the element +def expand_splits(MappingNode element_public not None, Variables variables not None): + cdef MappingNode element_bst = element_public.get_mapping('bst', default={}) + cdef MappingNode element_splits = element_bst.get_mapping('split-rules', default={}) + cdef str domain + cdef list new_splits + cdef SequenceNode splits + + # Resolve any variables in the public split rules directly + for domain, splits in element_splits.items(): + new_splits = [ + variables.subst(split.strip()) + for split in splits.as_str_list() + ] + element_splits[domain] = new_splits + + return element_public diff --git a/src/buildstream/_variables.pxd b/src/buildstream/_variables.pxd new file mode 100644 index 0000000..28f9507 --- /dev/null +++ b/src/buildstream/_variables.pxd @@ -0,0 +1,26 @@ +# +# Copyright (C) 2018 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +from .node cimport MappingNode + +cdef class Variables: + cdef MappingNode original + cdef dict _expstr_map + cdef public dict flat + + cpdef subst(self, str string) + cdef dict _resolve(self, MappingNode node) + cdef dict _flatten(self) diff --git a/src/buildstream/_variables.pyx b/src/buildstream/_variables.pyx index f8662ac..d5db8cd 100644 --- a/src/buildstream/_variables.pyx +++ b/src/buildstream/_variables.pyx @@ -65,10 +65,6 @@ PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}") # cdef class Variables: - cdef MappingNode original - cdef dict _expstr_map - cdef public dict flat - def __init__(self, MappingNode node): self.original = node self._expstr_map = self._resolve(node) @@ -87,7 +83,7 @@ cdef class Variables: # Raises: # LoadError, if the string contains unresolved variable references. # - def subst(self, str string): + cpdef subst(self, str string): expstr = _parse_expstr(string) try: diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 966f0f7..e22495f 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -95,6 +95,7 @@ from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, \ from .utils import FileListResult from . import utils from . import _cachekey +from . import _element # type: any from . import _signals from . import _site from ._platform import Platform @@ -318,7 +319,7 @@ class Element(Plugin): # Grab public domain data declared for this instance unexpanded_public = self.__extract_public(meta) - self.__public = self.__expand_splits(unexpanded_public) + self.__public = _element.expand_splits(unexpanded_public, self.__variables) self.__dynamic_public = None # Collect the composited element configuration and @@ -2846,21 +2847,6 @@ class Element(Plugin): return element_public - # Expand the splits in the public data using the Variables in the element - def __expand_splits(self, element_public): - element_bst = element_public.get_mapping('bst', default={}) - element_splits = element_bst.get_mapping('split-rules', default={}) - - # Resolve any variables in the public split rules directly - for domain, splits in element_splits.items(): - splits = [ - self.__variables.subst(split.strip()) - for split in splits.as_str_list() - ] - element_splits[domain] = splits - - return element_public - def __init_splits(self): bstdata = self.get_public_data('bst') splits = bstdata.get_mapping('split-rules')
