This is an automated email from the ASF dual-hosted git repository. github-bot pushed a commit to branch valentindavid/flatpak-demo in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9df849dfa282e124b876e1ba869023f1eecd1397 Author: Valentin David <[email protected]> AuthorDate: Mon Jul 2 11:55:15 2018 +0200 Delay full load of project for tracking --- buildstream/_artifactcache/artifactcache.py | 1 + buildstream/_frontend/widget.py | 7 ++- buildstream/_loader/loader.py | 5 +- buildstream/_project.py | 80 +++++++++++++++++------------ buildstream/element.py | 12 ++--- tests/artifactcache/config.py | 1 + tests/format/include/recursive/element.bst | 1 + 7 files changed, 62 insertions(+), 45 deletions(-) diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py index 2d745f8..8e51a64 100644 --- a/buildstream/_artifactcache/artifactcache.py +++ b/buildstream/_artifactcache/artifactcache.py @@ -125,6 +125,7 @@ class ArtifactCache(): has_remote_caches = True if use_config: for project in self.context.get_projects(): + project.ensure_fully_loaded() artifact_caches = _configured_remote_artifact_cache_specs(self.context, project) if artifact_caches: # artifact_caches is a list of ArtifactCacheSpec instances self._set_remotes(artifact_caches, project=project) diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py index cad1a28..61fd6cd 100644 --- a/buildstream/_frontend/widget.py +++ b/buildstream/_frontend/widget.py @@ -480,8 +480,11 @@ class LogLine(Widget): text += '\n' # Plugins - text += self._format_plugins(project.plugins._element_factory.loaded_dependencies, - project.plugins._source_factory.loaded_dependencies) + text += self._format_plugins(project.first_pass_config.plugins._element_factory.loaded_dependencies, + project.first_pass_config.plugins._source_factory.loaded_dependencies) + if project.plugins: + text += self._format_plugins(project.plugins._element_factory.loaded_dependencies, + project.plugins._source_factory.loaded_dependencies) # Pipeline state text += self.content_profile.fmt("Pipeline\n", bold=True) diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py index bc43180..4a76b03 100644 --- a/buildstream/_loader/loader.py +++ b/buildstream/_loader/loader.py @@ -249,10 +249,7 @@ class Loader(): if kind == "junction": self._first_pass_options.process_node(node) else: - if not self.project.is_loaded(): - raise LoadError(LoadErrorReason.INVALID_DATA, - "{}: Cannot pre-load. Element depends on project defaults." - .format(filename)) + self.project.ensure_fully_loaded() self._includes.process(node) diff --git a/buildstream/_project.py b/buildstream/_project.py index 2b9d637..5156539 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -270,13 +270,16 @@ class Project(): self._context.add_project(self) - self._loaded = False + self._partially_loaded = False + self._fully_loaded = False + self._project_includes = None + self._config_node = None profile_start(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-')) self._load(parent_loader=parent_loader, tempdir=tempdir) profile_end(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-')) - self._loaded = True + self._partially_loaded = True @property def plugins(self): @@ -298,8 +301,6 @@ class Project(): def source_overrides(self): return self.config.source_overrides - def is_loaded(self): - return self._loaded # translate_url(): # @@ -415,7 +416,7 @@ class Project(): # Load builtin default projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE) - config = _yaml.load(_site.default_project_config) + self._config_node = _yaml.load(_site.default_project_config) # Load project local config and override the builtin try: @@ -424,10 +425,10 @@ class Project(): # Raise a more specific error here raise LoadError(LoadErrorReason.MISSING_PROJECT_CONF, str(e)) - _yaml.composite(config, project_conf) + _yaml.composite(self._config_node, project_conf) # Assert project's format version early, before validating toplevel keys - format_version = _yaml.node_get(config, int, 'format-version') + format_version = _yaml.node_get(self._config_node, int, 'format-version') if BST_FORMAT_VERSION < format_version: major, minor = utils.get_bst_version() raise LoadError( @@ -437,15 +438,15 @@ class Project(): # The project name, element path and option declarations # are constant and cannot be overridden by option conditional statements - self.name = _yaml.node_get(config, str, 'name') + self.name = _yaml.node_get(self._config_node, str, 'name') # Validate that project name is a valid symbol name - _yaml.assert_symbol_name(_yaml.node_get_provenance(config, 'name'), + _yaml.assert_symbol_name(_yaml.node_get_provenance(self._config_node, 'name'), self.name, "project name") self.element_path = os.path.join( self.directory, - _yaml.node_get(config, str, 'element-path') + _yaml.node_get(self._config_node, str, 'element-path') ) self.config.options = OptionPool(self.element_path) @@ -455,18 +456,38 @@ class Project(): parent=parent_loader, tempdir=tempdir) - project_includes = Includes(self.loader) + self._project_includes = Includes(self.loader) - config_no_include = _yaml.node_copy(config) - project_includes.ignore_includes(config_no_include) + config_no_include = _yaml.node_copy(self._config_node) + self._project_includes.ignore_includes(config_no_include) self._load_pass(config_no_include, self.first_pass_config, True) - project_includes.process(config) + # Use separate file for storing source references + self.ref_storage = _yaml.node_get(self._config_node, str, 'ref-storage') + if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]: + p = _yaml.node_get_provenance(self._config_node, 'ref-storage') + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Invalid value '{}' specified for ref-storage" + .format(p, self.ref_storage)) + + if self.ref_storage == ProjectRefStorage.PROJECT_REFS: + self.junction_refs.load(self.first_pass_config.options) + + def ensure_fully_loaded(self): + if self._fully_loaded: + return + assert self._partially_loaded + self._fully_loaded = True + + if self.junction: + self.junction._get_project().ensure_fully_loaded() + + self._project_includes.process(self._config_node) - self._load_pass(config, self.config, False) + self._load_pass(self._config_node, self.config, False) - _yaml.node_validate(config, self.INCLUDE_CONFIG_KEYS + self.MAIN_FILE_CONFIG_KEYS) + _yaml.node_validate(self._config_node, self.INCLUDE_CONFIG_KEYS + self.MAIN_FILE_CONFIG_KEYS) # # Now all YAML composition is done, from here on we just load @@ -474,39 +495,30 @@ class Project(): # # Load artifacts pull/push configuration for this project - self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config) + self.artifact_cache_specs = ArtifactCache.specs_from_config_node(self._config_node) # Source url aliases - self._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={}) + self._aliases = _yaml.node_get(self._config_node, Mapping, 'aliases', default_value={}) # Load sandbox environment variables - self.base_environment = _yaml.node_get(config, Mapping, 'environment') - self.base_env_nocache = _yaml.node_get(config, list, 'environment-nocache') + self.base_environment = _yaml.node_get(self._config_node, Mapping, 'environment') + self.base_env_nocache = _yaml.node_get(self._config_node, list, 'environment-nocache') # Load sandbox configuration - self._sandbox = _yaml.node_get(config, Mapping, 'sandbox') + self._sandbox = _yaml.node_get(self._config_node, Mapping, 'sandbox') # Load project split rules - self._splits = _yaml.node_get(config, Mapping, 'split-rules') + self._splits = _yaml.node_get(self._config_node, Mapping, 'split-rules') # Fail on overlap - self.fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap') - - # Use separate file for storing source references - self.ref_storage = _yaml.node_get(config, str, 'ref-storage') - if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]: - p = _yaml.node_get_provenance(config, 'ref-storage') - raise LoadError(LoadErrorReason.INVALID_DATA, - "{}: Invalid value '{}' specified for ref-storage" - .format(p, self.ref_storage)) + self.fail_on_overlap = _yaml.node_get(self._config_node, bool, 'fail-on-overlap') # Load project.refs if it exists, this may be ignored. if self.ref_storage == ProjectRefStorage.PROJECT_REFS: self.refs.load(self.options) - self.junction_refs.load(self.options) # Parse shell options - shell_options = _yaml.node_get(config, Mapping, 'shell') + shell_options = _yaml.node_get(self._config_node, Mapping, 'shell') _yaml.node_validate(shell_options, ['command', 'environment', 'host-files']) self._shell_command = _yaml.node_get(shell_options, list, 'command') @@ -535,6 +547,8 @@ class Project(): self._shell_host_files.append(mount) + self._config_node = None + # _load_pass(): # # Loads parts of the project configuration that are different diff --git a/buildstream/element.py b/buildstream/element.py index aa49484..62e6554 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -197,9 +197,8 @@ class Element(Plugin): self.__is_junction = meta.kind == "junction" - if not project.is_loaded() and not self.__is_junction: - raise ElementError("{}: Cannot load element before project" - .format(self), reason="project-not-loaded") + if not self.__is_junction: + project.ensure_fully_loaded() self.normal_name = os.path.splitext(self.name.replace(os.sep, '-'))[0] """A normalized element name @@ -896,6 +895,7 @@ class Element(Plugin): if meta.first_pass: plugins = meta.project.first_pass_config.plugins else: + meta.project.ensure_fully_loaded() plugins = meta.project.plugins if meta in cls.__instantiated_elements: @@ -2184,7 +2184,7 @@ class Element(Plugin): project_nocache = [] else: project = self._get_project() - assert project.is_loaded() + project.ensure_fully_loaded() project_nocache = project.base_env_nocache default_nocache = _yaml.node_get(self.__defaults, list, 'environment-nocache', default_value=[]) @@ -2207,7 +2207,7 @@ class Element(Plugin): if self.__is_junction: variables = _yaml.node_chain_copy(project.first_pass_config.base_variables) else: - assert project.is_loaded() + project.ensure_fully_loaded() variables = _yaml.node_chain_copy(project.base_variables) _yaml.composite(variables, default_vars) @@ -2238,7 +2238,7 @@ class Element(Plugin): 'build-gid': 0} else: project = self._get_project() - assert project.is_loaded() + project.ensure_fully_loaded() sandbox_config = _yaml.node_chain_copy(project._sandbox) # The default config is already composited with the project overrides diff --git a/tests/artifactcache/config.py b/tests/artifactcache/config.py index 079e511..f594747 100644 --- a/tests/artifactcache/config.py +++ b/tests/artifactcache/config.py @@ -98,6 +98,7 @@ def test_artifact_cache_precedence(tmpdir, override_caches, project_caches, user context = Context() context.load(config=user_config_file) project = Project(str(project_dir), context) + project.ensure_fully_loaded() # Use the helper from the artifactcache module to parse our configuration. parsed_cache_specs = _configured_remote_artifact_cache_specs(context, project) diff --git a/tests/format/include/recursive/element.bst b/tests/format/include/recursive/element.bst new file mode 100644 index 0000000..4d7f702 --- /dev/null +++ b/tests/format/include/recursive/element.bst @@ -0,0 +1 @@ +kind: manual
