This is an automated email from the ASF dual-hosted git repository. akitouni pushed a commit to branch abderrahim/re-tree-only in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7b0cfa8f386dbb16c0f8e15e954a0a7a2da7c548 Author: Abderrahim Kitouni <[email protected]> AuthorDate: Fri Oct 11 09:16:04 2024 +0100 sandbox: request remote execution output as a directory digest This gives us an output that is easier to manipulate given we're using buildbox-casd, and allows to avoid a whole class of bugs such as #1842 and #1961 The old code for handling the output as a Tree is kept for now as the remote execution spec recommends using it as a fallback --- src/buildstream/sandbox/_sandboxreapi.py | 23 +++++++++++++---------- src/buildstream/sandbox/_sandboxremote.py | 16 ++++++++++++---- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py index 35901103e..34b6c8892 100644 --- a/src/buildstream/sandbox/_sandboxreapi.py +++ b/src/buildstream/sandbox/_sandboxreapi.py @@ -157,6 +157,7 @@ class SandboxREAPI(Sandbox): environment_variables=environment_variables, output_paths=output_directories, output_node_properties=self._output_node_properties, + output_directory_format=remote_execution_pb2.Command.OutputDirectoryFormat.DIRECTORY_ONLY, platform=platform, ) @@ -174,16 +175,18 @@ class SandboxREAPI(Sandbox): vdir = self.get_virtual_directory() for output_directory in output_directories: - tree_digest = output_directory.tree_digest - if tree_digest is None or not tree_digest.hash: - raise SandboxError("Output directory structure had no digest attached.") - - # Get digest of output directory from tree digest - tree = remote_execution_pb2.Tree() - with open(cascache.objpath(tree_digest), "rb") as f: - tree.ParseFromString(f.read()) - root_directory = tree.root.SerializeToString() - dir_digest = utils._message_digest(root_directory) + dir_digest = output_directory.root_directory_digest + if dir_digest is None or not dir_digest.hash: + tree_digest = output_directory.tree_digest + if tree_digest is None or not tree_digest.hash: + raise SandboxError("Output directory structure had no digest attached.") + + # Get digest of output directory from tree digest + tree = remote_execution_pb2.Tree() + with open(cascache.objpath(tree_digest), "rb") as f: + tree.ParseFromString(f.read()) + root_directory = tree.root.SerializeToString() + dir_digest = utils._message_digest(root_directory) # Create a normalized absolute path (inside the input tree) path = os.path.normpath(os.path.join(working_directory, output_directory.path)).lstrip(os.path.sep) diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py index ef8ef7de8..be312fb68 100644 --- a/src/buildstream/sandbox/_sandboxremote.py +++ b/src/buildstream/sandbox/_sandboxremote.py @@ -294,12 +294,20 @@ class SandboxRemote(SandboxREAPI): # Fetch outputs for output_directory in action_result.output_directories: + # Now do a pull to ensure we have the full directory structure. + # We first try the root_directory_digest we requested, then fall back to tree_digest + + root_directory_digest = output_directory.root_directory_digest + if root_directory_digest and root_directory_digest.hash: + cascache._fetch_directory(casremote, root_directory_digest) + break + tree_digest = output_directory.tree_digest - if tree_digest is None or not tree_digest.hash: - raise SandboxError("Output directory structure had no digest attached.") + if tree_digest and tree_digest.hash: + cascache.pull_tree(casremote, tree_digest) + break - # Now do a pull to ensure we have the full directory structure. - cascache.pull_tree(casremote, tree_digest) + raise SandboxError("Output directory structure had no digest attached.") # Fetch stdout and stderr blobs cascache.fetch_blobs(casremote, [action_result.stdout_digest, action_result.stderr_digest])
