This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch jmac/virtual_directories_pre_rebase in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 753566f39ca4bae7eb62b93129c4288628e26445 Author: Jim MacArthur <[email protected]> AuthorDate: Tue May 8 16:19:22 2018 +0100 element.py: Conversion to use virtual directories --- buildstream/element.py | 105 ++++++++++++++++++++++++++----------------------- 1 file changed, 56 insertions(+), 49 deletions(-) diff --git a/buildstream/element.py b/buildstream/element.py index 7964db6..83ef2fb 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -80,7 +80,6 @@ from collections import Mapping, OrderedDict from contextlib import contextmanager from enum import Enum import tempfile -import time import shutil from . import _yaml @@ -97,6 +96,10 @@ from . import _site from ._platform import Platform from .sandbox._config import SandboxConfig +from .storage.directory import Directory +from .storage._filebaseddirectory import FileBasedDirectory, VirtualDirectoryError + + # _KeyStrength(): # # Strength of cache key @@ -194,7 +197,7 @@ class Element(Plugin): """Whether to raise exceptions if an element uses Sandbox.get_directory instead of Sandbox.get_virtual_directory. - *Since: 1.2* + *Since: 1.4* """ def __init__(self, context, project, artifacts, meta, plugin_conf): @@ -626,10 +629,10 @@ class Element(Plugin): # Hard link it into the staging area # - basedir = sandbox.get_directory() - stagedir = basedir \ + vbasedir = sandbox.get_virtual_directory() + vstagedir = vbasedir \ if path is None \ - else os.path.join(basedir, path.lstrip(os.sep)) + else vbasedir.descend(path.lstrip(os.sep).split(os.sep)) files = list(self.__compute_splits(include, exclude, orphans)) @@ -641,15 +644,8 @@ class Element(Plugin): link_files = files copy_files = [] - link_result = utils.link_files(artifact, stagedir, files=link_files, - report_written=True) - copy_result = utils.copy_files(artifact, stagedir, files=copy_files, - report_written=True) - - cur_time = time.time() - - for f in copy_result.files_written: - os.utime(os.path.join(stagedir, f), times=(cur_time, cur_time)) + link_result = vstagedir.import_files(artifact, files=link_files, report_written=True, can_link=True) + copy_result = vstagedir.import_files(artifact, files=copy_files, report_written=True, update_utimes=True) return link_result.combine(copy_result) @@ -1306,8 +1302,8 @@ class Element(Plugin): sandbox._set_mount_source(directory, workspace.get_absolute_path()) # Stage all sources that need to be copied - sandbox_root = sandbox.get_directory() - host_directory = os.path.join(sandbox_root, directory.lstrip(os.sep)) + sandbox_vroot = sandbox.get_virtual_directory() + host_directory = sandbox_vroot.descend(directory.lstrip(os.sep).split(os.sep), create=True) self._stage_sources_at(host_directory, mount_workspaces=mount_workspaces) # _stage_sources_at(): @@ -1315,31 +1311,36 @@ class Element(Plugin): # Stage this element's sources to a directory # # Args: - # directory (str): An absolute path to stage the sources at + # vdirectory (:class:`.storage.Directory`): A virtual directory object to stage sources into. # mount_workspaces (bool): mount workspaces if True, copy otherwise # - def _stage_sources_at(self, directory, mount_workspaces=True): + def _stage_sources_at(self, vdirectory, mount_workspaces=True): with self.timed_activity("Staging sources", silent_nested=True): - if os.path.isdir(directory) and os.listdir(directory): - raise ElementError("Staging directory '{}' is not empty".format(directory)) - - workspace = self._get_workspace() - if workspace: - # If mount_workspaces is set and we're doing incremental builds, - # the workspace is already mounted into the sandbox. - if not (mount_workspaces and self.__can_build_incrementally()): - with self.timed_activity("Staging local files at {}".format(workspace.path)): - workspace.stage(directory) - else: - # No workspace, stage directly - for source in self.sources(): - source._stage(directory) - + if not isinstance(vdirectory, Directory): + vdirectory = FileBasedDirectory(vdirectory) + if not vdirectory.is_empty(): + raise ElementError("Staging directory '{}' is not empty".format(vdirectory)) + + with tempfile.TemporaryDirectory() as temp_staging_directory: + + workspace = self._get_workspace() + if workspace: + # If mount_workspaces is set and we're doing incremental builds, + # the workspace is already mounted into the sandbox. + if not (mount_workspaces and self.__can_build_incrementally()): + with self.timed_activity("Staging local files at {}".format(workspace.path)): + workspace.stage(temp_staging_directory) + else: + # No workspace, stage directly + for source in self.sources(): + source._stage(temp_staging_directory) + + vdirectory.import_files(temp_staging_directory) # Ensure deterministic mtime of sources at build time - utils._set_deterministic_mtime(directory) + vdirectory.set_deterministic_mtime() # Ensure deterministic owners of sources at build time - utils._set_deterministic_user(directory) + vdirectory.set_deterministic_user() # _set_required(): # @@ -1455,7 +1456,7 @@ class Element(Plugin): with _signals.terminator(cleanup_rootdir), \ self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox: # nopep8 - sandbox_root = sandbox.get_directory() + sandbox_vroot = sandbox.get_virtual_directory() # By default, the dynamic public data is the same as the static public data. # The plugin's assemble() method may modify this, though. @@ -1485,23 +1486,24 @@ class Element(Plugin): # workspace = self._get_workspace() if workspace and self.__staged_sources_directory: - sandbox_root = sandbox.get_directory() - sandbox_path = os.path.join(sandbox_root, - self.__staged_sources_directory.lstrip(os.sep)) + sandbox_vroot = sandbox.get_virtual_directory() + path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep) + sandbox_vpath = sandbox_vroot.descend(path_components) try: - utils.copy_files(workspace.path, sandbox_path) + sandbox_vpath.import_files(workspace.path) except UtilError as e: self.warn("Failed to preserve workspace state for failed build sysroot: {}" .format(e)) raise - collectdir = os.path.join(sandbox_root, collect.lstrip(os.sep)) - if not os.path.exists(collectdir): + try: + collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep)) + except VirtualDirectoryError: raise ElementError( - "Directory '{}' was not found inside the sandbox, " + "Subdirectory '{}' of '{}' does not exist following assembly, " "unable to collect artifact contents" - .format(collect)) + .format(collect, sandbox_vroot)) # At this point, we expect an exception was raised leading to # an error message, or we have good output to collect. @@ -1519,12 +1521,17 @@ class Element(Plugin): os.mkdir(buildtreedir) # Hard link files from collect dir to files directory - utils.link_files(collectdir, filesdir) + collectvdir.export_files(filesdir, can_link=True) - sandbox_build_dir = os.path.join(sandbox_root, self.get_variable('build-root').lstrip(os.sep)) - # Hard link files from build-root dir to buildtreedir directory - if os.path.isdir(sandbox_build_dir): - utils.link_files(sandbox_build_dir, buildtreedir) + try: + # Attempt to hard link files from build-root dir to buildtreedir directory + build_root = self.get_variable('build-root').lstrip(os.sep) + sandbox_build_dir = sandbox_vroot.descend(build_root.split(os.sep)) + sandbox_build_dir.export_files(buildtreedir, can_link=True, can_destroy=True) + except VirtualDirectoryError: + # This replaces code which previously did nothing if + # buildtreedir was not a directory, so we do the same. + pass # Copy build log log_filename = context.get_log_filename()
