This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch tristan/virtual-directory-cleanup in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 042b2bd26d1e8fd905828c78b35a25b30267b9fb Author: Tristan van Berkom <[email protected]> AuthorDate: Fri Mar 4 18:39:50 2022 +0900 storage/directory.py: Added full mypy annotations to API surface Also enhanced documentation thoroughly and demarked public/internal sections of the API. --- src/buildstream/storage/directory.py | 183 ++++++++++++++++++++++------------- 1 file changed, 114 insertions(+), 69 deletions(-) diff --git a/src/buildstream/storage/directory.py b/src/buildstream/storage/directory.py index f49bcb8..f39b0b4 100644 --- a/src/buildstream/storage/directory.py +++ b/src/buildstream/storage/directory.py @@ -33,7 +33,9 @@ See also: :ref:`sandboxing`. import os import stat -from typing import Callable, Optional, Union, List +from contextlib import contextmanager +from tarfile import TarFile +from typing import Callable, Optional, Union, List, IO, Iterator from .._exceptions import BstError from ..exceptions import ErrorDomain @@ -50,8 +52,7 @@ class DirectoryError(BstError): If this is not handled, the BuildStream core will fail the current task where the error occurs and present the user with the error. """ - - def __init__(self, message, reason=None): + def __init__(self, message: str, reason: str = None): super().__init__(message, domain=ErrorDomain.VIRTUAL_FS, reason=reason) @@ -59,22 +60,25 @@ class Directory: def __init__(self, external_directory=None): raise NotImplementedError() - def descend(self, *paths: str, create: bool = False, follow_symlinks: bool = False): + ################################################################### + # Public API # + ################################################################### + + def descend(self, *paths: str, create: bool = False, follow_symlinks: bool = False) -> "Directory": """Descend one or more levels of directory hierarchy and return a new Directory object for that directory. Args: - *paths: A list of strings which are all directory names. - create: If this is true, the directories will be created if - they don't already exist. + *paths: A list of strings which are all directory names. + create: If this is true, the directories will be created if + they don't already exist. - Yields: - A Directory object representing the found directory. + Returns: + A Directory object representing the found directory. Raises: DirectoryError: if any of the components in subdirectory_spec - cannot be found, or are files, or symlinks to files. - + cannot be found, or are files, or symlinks to files. """ raise NotImplementedError() @@ -92,77 +96,88 @@ class Directory: """Imports some or all files from external_path into this directory. Args: - external_pathspec: Either a string containing a pathname, or a - Directory object, to use as the source. - filter_callback: Optional filter callback. Called with the - relative path as argument for every file in the source directory. - The file is imported only if the callable returns True. - If no filter callback is specified, all files will be imported. - report_written: Return the full list of files - written. Defaults to true. If false, only a list of - overwritten files is returned. - update_mtime: Update the access and modification time - of each file copied to the time specified in seconds. - can_link: Whether it's OK to create a hard link to the - original content, meaning the stored copy will change when the - original files change. Setting this doesn't guarantee hard - links will be made. - properties: Optional list of strings representing file properties - to capture when importing. + external_pathspec: Either a string containing a pathname, or a + Directory object, to use as the source. + filter_callback: Optional filter callback. Called with the + relative path as argument for every file in the source directory. + The file is imported only if the callable returns True. + If no filter callback is specified, all files will be imported. + report_written: Return the full list of files written. Defaults to true. + If false, only a list of overwritten files is returned. + update_mtime: Update the access and modification time of each file copied to the time specified in seconds. + can_link: Whether it's OK to create a hard link to the original content, meaning the stored copy will change + when the original files change. Setting this doesn't guarantee hard links will be made. + properties: Optional list of strings representing file properties to capture when importing. - Yields: - A report of files imported and overwritten. + Returns: + A :class:`.FileListResult` report of files imported and overwritten. + Raises: + DirectoryError: if any system error occurs. """ raise NotImplementedError() - def import_single_file(self, external_pathspec, properties=None): - """Imports a single file from an external path""" + def import_single_file(self, external_pathspec: str, properties: Optional[List[str]] = None) -> FileListResult: + """Imports a single file from an external path + + Args: + external_pathspec: A string containing a pathname. + properties: Optional list of strings representing file properties to capture when importing. + + Returns: + A :class:`.FileListResult` report of files imported and overwritten. + + Raises: + DirectoryError: if any system error occurs. + """ raise NotImplementedError() - def export_files(self, to_directory, *, can_link=False, can_destroy=False): + def export_files(self, to_directory: str, *, can_link: bool = False, can_destroy: bool = False) -> None: """Copies everything from this into to_directory. Args: - to_directory (string): a path outside this directory object - where the contents will be copied to. - can_link (bool): Whether we can create hard links in to_directory - instead of copying. Setting this does not guarantee hard links will be used. - can_destroy (bool): Can we destroy the data already in this - directory when exporting? If set, this may allow data to be - moved rather than copied which will be quicker. - """ + to_directory: a path outside this directory object where the contents will be copied to. + can_link: Whether we can create hard links in to_directory instead of copying. + Setting this does not guarantee hard links will be used. + can_destroy: Can we destroy the data already in this directory when exporting? If set, + this may allow data to be moved rather than copied which will be quicker. + Raises: + DirectoryError: if any system error occurs. + """ raise NotImplementedError() - def export_to_tar(self, tarfile, destination_dir, mtime=BST_ARBITRARY_TIMESTAMP): + def export_to_tar(self, tarfile: TarFile, destination_dir: str, mtime: int = BST_ARBITRARY_TIMESTAMP) -> None: """ Exports this directory into the given tar file. Args: - tarfile (TarFile): A Python TarFile object to export into. - destination_dir (str): The prefix for all filenames inside the archive. - mtime (int): mtimes of all files in the archive are set to this. + tarfile: A Python TarFile object to export into. + destination_dir: The prefix for all filenames inside the archive. + mtime: mtimes of all files in the archive are set to this. + + Raises: + DirectoryError: if any system error occurs. """ raise NotImplementedError() # Convenience functions - def is_empty(self): + def is_empty(self) -> bool: """ Return true if this directory has no files, subdirectories or links in it. """ raise NotImplementedError() - def list_relative_paths(self): - """Provide a list of all relative paths in this directory. Includes - directories only if they are empty. + def list_relative_paths(self) -> Iterator[str]: + """Provide a list of all relative paths in this directory. - Yields: - (List(str)) - list of all files with relative paths. + Includes directories only if they are empty. + Yields: + All files in this directory with relative paths. """ raise NotImplementedError() - def get_size(self): + def get_size(self) -> int: """ Get an approximation of the storage space in bytes used by this directory and all files and subdirectories in it. Storage space varies by implementation and effective space used may be lower than this number due to deduplication. """ @@ -176,7 +191,7 @@ class Directory: follow_symlinks: True to follow symlinks. Returns: - True if the path exists, False otherwise. + True if the path exists, False otherwise. """ raise NotImplementedError() @@ -185,10 +200,13 @@ class Directory: Args: *paths: A list of strings which are all path components. - follow_symlinks: True to follow symlinks. + follow_symlinks: True to follow symlinks. Returns: - A `os.stat_result` object. + A `os.stat_result` object. + + Raises: + DirectoryError: if any system error occurs. """ raise NotImplementedError() @@ -196,11 +214,11 @@ class Directory: """ Check whether the specified path is an existing regular file. Args: - *paths: A list of strings which are all path components. - follow_symlinks: True to follow symlinks. + *paths: A list of strings which are all path components. + follow_symlinks: True to follow symlinks. Returns: - True if the path is an existing regular file, False otherwise. + True if the path is an existing regular file, False otherwise. """ try: st = self.stat(*paths, follow_symlinks=follow_symlinks) @@ -213,10 +231,10 @@ class Directory: Args: *paths: A list of strings which are all path components. - follow_symlinks: True to follow symlinks. + follow_symlinks: True to follow symlinks. Returns: - True if the path is an existing directory, False otherwise. + True if the path is an existing directory, False otherwise. """ try: st = self.stat(*paths, follow_symlinks=follow_symlinks) @@ -228,11 +246,11 @@ class Directory: """ Check whether the specified path is an existing symlink. Args: - *paths: A list of strings which are all path components. - follow_symlinks: True to follow symlinks. + *paths: A list of strings which are all path components. + follow_symlinks: True to follow symlinks. Returns: - True if the path is an existing symlink, False otherwise. + True if the path is an existing symlink, False otherwise. """ try: st = self.stat(*paths, follow_symlinks=follow_symlinks) @@ -240,13 +258,20 @@ class Directory: except (DirectoryError, FileNotFoundError): return False - def open_file(self, *paths: str, mode: str = "r"): + @contextmanager + def open_file(self, *paths: str, mode: str = "r") -> Iterator[IO]: """ Open file and return a corresponding file object. In text mode, UTF-8 is used as encoding. Args: - *paths: A list of strings which are all path components. - mode (str): An optional string that specifies the mode in which the file is opened. + *paths: A list of strings which are all path components. + mode (str): An optional string that specifies the mode in which the file is opened. + + Yields: + The file object for the open file + + Raises: + DirectoryError: if any system error occurs. """ raise NotImplementedError() @@ -255,7 +280,10 @@ class Directory: defined. Args: - *paths: A list of strings which are all path components. + *paths: A list of strings which are all path components. + + Raises: + DirectoryError: if any system error occurs. """ raise NotImplementedError() @@ -263,7 +291,10 @@ class Directory: """ Return a string representing the path to which the symbolic link points. Args: - *paths: A list of strings which are all path components. + *paths: A list of strings which are all path components. + + Raises: + DirectoryError: if any system error occurs. """ raise NotImplementedError() @@ -273,6 +304,9 @@ class Directory: Args: *paths: A list of strings which are all path components. recursive: True to delete non-empty directories. + + Raises: + DirectoryError: if any system error occurs. """ raise NotImplementedError() @@ -283,16 +317,27 @@ class Directory: Args: *src: Source path components. *dest: Destination path components. + + Raises: + DirectoryError: if any system error occurs. """ raise NotImplementedError() + ################################################################### + # Internal API # + ################################################################### + # _set_deterministic_user(): # - # Sets all files in this directory to the current user's euid/egid. + # Abstract method to set all files in this directory to the current user's euid/egid. # def _set_deterministic_user(self): raise NotImplementedError() + # _create_empty_file() + # + # Utility function to create an empty file + # def _create_empty_file(self, *paths): with self.open_file(*paths, mode="w"): pass
