This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch aevri/check_spawn_ci_working in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ac39de3ca9e4a8acce4c1ceedf04a7fcb380e0f5 Author: Angelos Evripiotis <[email protected]> AuthorDate: Thu Oct 17 10:13:41 2019 +0100 WIP: pickle first_pass_config too --- src/buildstream/_scheduler/jobs/jobpickler.py | 39 ++++++++++++++++++++++++++- src/buildstream/element.py | 4 +-- src/buildstream/source.py | 3 ++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/src/buildstream/_scheduler/jobs/jobpickler.py b/src/buildstream/_scheduler/jobs/jobpickler.py index c67353e..0b15feb 100644 --- a/src/buildstream/_scheduler/jobs/jobpickler.py +++ b/src/buildstream/_scheduler/jobs/jobpickler.py @@ -66,17 +66,29 @@ from ..._messenger import Messenger # def pickle_child_job(child_job, projects): + p = child_job._element._Plugin__project + element_classes = [ cls for p in projects if p.config.element_factory is not None for cls, _ in p.config.element_factory.all_loaded_plugins() + ] + [ + cls + for p in projects + if p.first_pass_config.element_factory is not None + for cls, _ in p.first_pass_config.element_factory.all_loaded_plugins() ] source_classes = [ cls for p in projects if p.config.source_factory is not None for cls, _ in p.config.source_factory.all_loaded_plugins() + ] + [ + cls + for p in projects + if p.first_pass_config.source_factory is not None + for cls, _ in p.first_pass_config.source_factory.all_loaded_plugins() ] data = io.BytesIO() @@ -121,7 +133,32 @@ def _new_artifact_proto_from_reduction_args(data): def _reduce_plugin(plugin): - factory, meta_kind, state = plugin._get_args_for_child_job_pickling() + project, meta_kind, state = plugin._get_args_for_child_job_pickling() + assert project + assert meta_kind + + factories = [ + project.config.element_factory, + project.first_pass_config.element_factory, + project.config.source_factory, + project.first_pass_config.source_factory, + ] + + print("plugin:", plugin) + print("factories:", factories) + + factory = None + for f in factories: + if f is None: + continue + for cls, _ in f.all_loaded_plugins(): + print(f, "comparing", type(plugin), cls) + if type(plugin) == cls: + factory = f + + if factory is None: + raise Exception("Couldn't find plugin in factories") + args = (factory, meta_kind) return (_new_plugin_from_reduction_args, args, state) diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 966f0f7..3be6b80 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -2373,8 +2373,8 @@ class Element(Plugin): # let us know, and we will need to update accordingly. del state["_Element__required_callback"] - factory = self._get_project().config.element_factory - return factory, self.__meta_kind, state + # factory = self._get_project().config.element_factory + return self._get_project(), self.__meta_kind, state def _walk_artifact_files(self): yield from self.__artifact.get_files().walk() diff --git a/src/buildstream/source.py b/src/buildstream/source.py index 1fb318b..76c5d57 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -1131,7 +1131,8 @@ class Source(Plugin): # o The code sketch of how pickling works also returns `self.__dict__`: # https://docs.python.org/3/library/pickle.html#pickling-class-instances # - return factory, self.__meta_kind, self.__dict__ + # return factory, self.__meta_kind, self.__dict__ + return self._get_project(), self.__meta_kind, self.__dict__ ############################################################# # Local Private Methods #
