This is an automated email from the ASF dual-hosted git repository. juergbi pushed a commit to branch jbilleter/digest-environment in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8963d037737d030739366a02bb80d1675fe65527 Author: Jürg Billeter <[email protected]> AuthorDate: Fri Aug 8 14:04:32 2025 +0200 wip: Store sandbox state in Artifact proto Use that to configure the sandbox state for `bst shell --use-buildtree`. --- src/buildstream/_artifact.py | 30 ++++++++++++++++++++++ src/buildstream/_artifactcache.py | 6 +++++ .../_protos/buildstream/v2/artifact.proto | 7 +++++ .../_protos/buildstream/v2/artifact_pb2.py | 14 +++++----- .../_protos/buildstream/v2/artifact_pb2.pyi | 15 +++++++++-- src/buildstream/element.py | 10 +++++--- 6 files changed, 71 insertions(+), 11 deletions(-) diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py index 7ccc61c6a..c06603559 100644 --- a/src/buildstream/_artifact.py +++ b/src/buildstream/_artifact.py @@ -200,6 +200,7 @@ class Artifact: # variables (Variables): The element's Variables # environment (dict): dict of the element's environment variables # sandboxconfig (SandboxConfig): The element's SandboxConfig + # buildsandbox (Sandbox): The element's configured build sandbox # def cache( self, @@ -213,6 +214,7 @@ class Artifact: variables, environment, sandboxconfig, + buildsandbox, ): context = self._context @@ -317,6 +319,19 @@ class Artifact: rootvdir._import_files_internal(buildrootvdir, properties=properties, collect_result=False) artifact.buildroot.CopyFrom(rootvdir._get_digest()) + if buildsandbox is not None: + sandbox_env = buildsandbox._get_configured_environment() + if sandbox_env: + for key, value in sorted(sandbox_env.items()): + artifact.buildsandbox.environment.add(name=key, value=value) + + artifact.buildsandbox.working_directory = buildsandbox._get_work_directory() + + for subsandbox in buildsandbox._get_subsandboxes(): + vdir = subsandbox.get_virtual_directory() + digest = artifact.buildsandbox.subsandbox_digests.add() + digest.CopyFrom(vdir._get_digest()) + os.makedirs(os.path.dirname(os.path.join(self._artifactdir, element.get_artifact_name())), exist_ok=True) keys = utils._deduplicate([self._cache_key, self._weak_cache_key]) for key in keys: @@ -681,6 +696,21 @@ class Artifact: return True + def configure_sandbox(self, sandbox): + artifact = self._get_proto() + + if artifact.buildsandbox and artifact.buildsandbox.environment: + env = {} + for env_var in artifact.buildsandbox.environment: + env[env_var.name] = env_var.value + else: + env = self.load_environment() + + sandbox.set_environment(env) + + if artifact.buildsandbox and artifact.buildsandbox.working_directory: + sandbox.set_work_directory(artifact.buildsandbox.working_directory) + # load_proto() # # Returns: diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py index a32d0ee62..d5dc27e95 100644 --- a/src/buildstream/_artifactcache.py +++ b/src/buildstream/_artifactcache.py @@ -361,6 +361,9 @@ class ArtifactCache(AssetCache): referenced_directories.append(artifact_proto.sources) if artifact_proto.buildroot: referenced_directories.append(artifact_proto.buildroot) + if artifact_proto.buildsandbox: + for subsandbox_digest in artifact_proto.buildsandbox.subsandbox_digests: + referenced_directories.append(subsandbox_digest) referenced_blobs = [artifact_proto.low_diversity_meta, artifact_proto.high_diversity_meta] + [ log_file.digest for log_file in artifact_proto.logs @@ -419,6 +422,9 @@ class ArtifactCache(AssetCache): self.cas.fetch_directory(remote, artifact.buildtree) if str(artifact.buildroot): self.cas.fetch_directory(remote, artifact.buildroot) + if artifact.buildsandbox: + for subsandbox_digest in artifact.buildsandbox.subsandbox_digests: + self.cas.fetch_directory(remote, subsandbox_digest) digests = [artifact.low_diversity_meta, artifact.high_diversity_meta] if str(artifact.public_data): diff --git a/src/buildstream/_protos/buildstream/v2/artifact.proto b/src/buildstream/_protos/buildstream/v2/artifact.proto index 86ac452c8..1024ce860 100644 --- a/src/buildstream/_protos/buildstream/v2/artifact.proto +++ b/src/buildstream/_protos/buildstream/v2/artifact.proto @@ -85,4 +85,11 @@ message Artifact { // digest of a directory build.bazel.remote.execution.v2.Digest buildroot = 17; // optional + + message SandboxState { + repeated build.bazel.remote.execution.v2.Command.EnvironmentVariable environment = 1; + string working_directory = 2; + repeated build.bazel.remote.execution.v2.Digest subsandbox_digests = 3; + }; + SandboxState buildsandbox = 18; // optional } diff --git a/src/buildstream/_protos/buildstream/v2/artifact_pb2.py b/src/buildstream/_protos/buildstream/v2/artifact_pb2.py index 21cc67f4f..7d86ea3b8 100644 --- a/src/buildstream/_protos/buildstream/v2/artifact_pb2.py +++ b/src/buildstream/_protos/buildstream/v2/artifact_pb2.py @@ -26,7 +26,7 @@ from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1d\x62uildstream/v2/artifact.proto\x12\x0e\x62uildstream.v2\x1a\x36\x62uild/bazel/remote/execution/v2/remote_execution.proto\x1a\x1cgoogle/api/annotations.proto\"\x89\x07\n\x08\x41rtifact\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x15\n\rbuild_success\x18\x02 \x01(\x08\x12\x13\n\x0b\x62uild_error\x18\x03 \x01(\t\x12\x1b\n\x13\x62uild_error_details\x18\x04 \x01(\t\x12\x12\n\nstrong_key\x18\x05 \x01(\t\x12\x10\n\x08weak_key\x1 [...] +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1d\x62uildstream/v2/artifact.proto\x12\x0e\x62uildstream.v2\x1a\x36\x62uild/bazel/remote/execution/v2/remote_execution.proto\x1a\x1cgoogle/api/annotations.proto\"\x8a\t\n\x08\x41rtifact\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x15\n\rbuild_success\x18\x02 \x01(\x08\x12\x13\n\x0b\x62uild_error\x18\x03 \x01(\t\x12\x1b\n\x13\x62uild_error_details\x18\x04 \x01(\t\x12\x12\n\nstrong_key\x18\x05 \x01(\t\x12\x10\n\x08weak_key\x18\ [...] _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -34,9 +34,11 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'buildstream.v2.artifact_pb2 if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None _globals['_ARTIFACT']._serialized_start=136 - _globals['_ARTIFACT']._serialized_end=1041 - _globals['_ARTIFACT_DEPENDENCY']._serialized_start=860 - _globals['_ARTIFACT_DEPENDENCY']._serialized_end=959 - _globals['_ARTIFACT_LOGFILE']._serialized_start=961 - _globals['_ARTIFACT_LOGFILE']._serialized_end=1041 + _globals['_ARTIFACT']._serialized_end=1298 + _globals['_ARTIFACT_DEPENDENCY']._serialized_start=921 + _globals['_ARTIFACT_DEPENDENCY']._serialized_end=1020 + _globals['_ARTIFACT_LOGFILE']._serialized_start=1022 + _globals['_ARTIFACT_LOGFILE']._serialized_end=1102 + _globals['_ARTIFACT_SANDBOXSTATE']._serialized_start=1105 + _globals['_ARTIFACT_SANDBOXSTATE']._serialized_end=1298 # @@protoc_insertion_point(module_scope) diff --git a/src/buildstream/_protos/buildstream/v2/artifact_pb2.pyi b/src/buildstream/_protos/buildstream/v2/artifact_pb2.pyi index 3f80ffa4b..08c6960c1 100644 --- a/src/buildstream/_protos/buildstream/v2/artifact_pb2.pyi +++ b/src/buildstream/_protos/buildstream/v2/artifact_pb2.pyi @@ -8,7 +8,7 @@ from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Map DESCRIPTOR: _descriptor.FileDescriptor class Artifact(_message.Message): - __slots__ = ("version", "build_success", "build_error", "build_error_details", "strong_key", "weak_key", "was_workspaced", "files", "build_deps", "public_data", "logs", "buildtree", "sources", "low_diversity_meta", "high_diversity_meta", "strict_key", "buildroot") + __slots__ = ("version", "build_success", "build_error", "build_error_details", "strong_key", "weak_key", "was_workspaced", "files", "build_deps", "public_data", "logs", "buildtree", "sources", "low_diversity_meta", "high_diversity_meta", "strict_key", "buildroot", "buildsandbox") class Dependency(_message.Message): __slots__ = ("project_name", "element_name", "cache_key", "was_workspaced") PROJECT_NAME_FIELD_NUMBER: _ClassVar[int] @@ -27,6 +27,15 @@ class Artifact(_message.Message): name: str digest: _remote_execution_pb2.Digest def __init__(self, name: _Optional[str] = ..., digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ...) -> None: ... + class SandboxState(_message.Message): + __slots__ = ("environment", "working_directory", "subsandbox_digests") + ENVIRONMENT_FIELD_NUMBER: _ClassVar[int] + WORKING_DIRECTORY_FIELD_NUMBER: _ClassVar[int] + SUBSANDBOX_DIGESTS_FIELD_NUMBER: _ClassVar[int] + environment: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Command.EnvironmentVariable] + working_directory: str + subsandbox_digests: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Digest] + def __init__(self, environment: _Optional[_Iterable[_Union[_remote_execution_pb2.Command.EnvironmentVariable, _Mapping]]] = ..., working_directory: _Optional[str] = ..., subsandbox_digests: _Optional[_Iterable[_Union[_remote_execution_pb2.Digest, _Mapping]]] = ...) -> None: ... VERSION_FIELD_NUMBER: _ClassVar[int] BUILD_SUCCESS_FIELD_NUMBER: _ClassVar[int] BUILD_ERROR_FIELD_NUMBER: _ClassVar[int] @@ -44,6 +53,7 @@ class Artifact(_message.Message): HIGH_DIVERSITY_META_FIELD_NUMBER: _ClassVar[int] STRICT_KEY_FIELD_NUMBER: _ClassVar[int] BUILDROOT_FIELD_NUMBER: _ClassVar[int] + BUILDSANDBOX_FIELD_NUMBER: _ClassVar[int] version: int build_success: bool build_error: str @@ -61,4 +71,5 @@ class Artifact(_message.Message): high_diversity_meta: _remote_execution_pb2.Digest strict_key: str buildroot: _remote_execution_pb2.Digest - def __init__(self, version: _Optional[int] = ..., build_success: bool = ..., build_error: _Optional[str] = ..., build_error_details: _Optional[str] = ..., strong_key: _Optional[str] = ..., weak_key: _Optional[str] = ..., was_workspaced: bool = ..., files: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., build_deps: _Optional[_Iterable[_Union[Artifact.Dependency, _Mapping]]] = ..., public_data: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., logs: _Opt [...] + buildsandbox: Artifact.SandboxState + def __init__(self, version: _Optional[int] = ..., build_success: bool = ..., build_error: _Optional[str] = ..., build_error_details: _Optional[str] = ..., strong_key: _Optional[str] = ..., weak_key: _Optional[str] = ..., was_workspaced: bool = ..., files: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., build_deps: _Optional[_Iterable[_Union[Artifact.Dependency, _Mapping]]] = ..., public_data: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., logs: _Opt [...] diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 85a0d36fb..c84daff79 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -1431,18 +1431,21 @@ class Element(Plugin): # pylint: disable-next=contextmanager-generator-missing-cleanup with self.__sandbox(config=self.__sandbox_config, allow_remote=False) as sandbox: - # Configure always comes first, and we need it. - self.__configure_sandbox(sandbox) - if usebuildtree: + # Configure the sandbox from artifact metadata + self.__artifact.configure_sandbox(sandbox) + # Use the cached buildroot directly buildrootvdir = self.__artifact.get_buildroot() sandbox_vroot = sandbox.get_virtual_directory() sandbox_vroot._import_files_internal(buildrootvdir, collect_result=False) elif shell and scope == _Scope.BUILD: + self.__configure_sandbox(sandbox) # Stage what we need self.__stage(sandbox) else: + self.__configure_sandbox(sandbox) + # Stage deps in the sandbox root with self.timed_activity("Staging dependencies", silent_nested=True), self.__collect_overlaps(sandbox): self._stage_dependency_artifacts(sandbox, scope) @@ -1778,6 +1781,7 @@ class Element(Plugin): variables=self.__variables, environment=self.__environment, sandboxconfig=self.__sandbox_config, + buildsandbox=sandbox if buildrootvdir else None, ) if collect is not None and collectvdir is None:
