This is an automated email from the ASF dual-hosted git repository. juergbi pushed a commit to branch juerg/update-requirements in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0e32671a28825e1df905c1ba3a62ea750d651f96 Author: Jürg Billeter <[email protected]> AuthorDate: Fri Jul 19 15:12:49 2024 +0200 Update requirements This includes an update to Pylint 3, which requires various changes to pass the new linter checks. With this commit, pylint and mypy now also pass with Python 3.12. --- .pylintrc | 12 +++++-- requirements/cov-requirements.txt | 15 ++++----- requirements/dev-requirements.txt | 38 +++++++++------------- requirements/requirements.txt | 20 ++++++------ setup.py | 3 +- src/buildstream/_elementsources.py | 3 +- src/buildstream/_pipeline.py | 3 +- src/buildstream/_pluginfactory/__init__.py | 2 ++ .../_pluginfactory/pluginoriginjunction.py | 2 ++ src/buildstream/_pluginfactory/pluginoriginpip.py | 2 ++ src/buildstream/_project.py | 5 ++- src/buildstream/_scheduler/scheduler.py | 2 +- src/buildstream/_testing/runcli.py | 1 + src/buildstream/element.py | 2 ++ src/buildstream/source.py | 5 ++- src/buildstream/storage/_casbaseddirectory.py | 13 +++++--- src/buildstream/utils.py | 2 +- tests/frontend/artifact_list_contents.py | 4 +++ tests/frontend/completions.py | 2 ++ tests/remoteexecution/workspace.py | 3 ++ tox.ini | 2 +- 21 files changed, 78 insertions(+), 63 deletions(-) diff --git a/.pylintrc b/.pylintrc index 9e124870c..5a7a1b73a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -[MASTER] +[MAIN] # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may @@ -32,7 +32,7 @@ ignore=CVS,doc # Add files or directories matching the regex patterns to the blacklist. The # regex matches against base names, not paths. -ignore-patterns=.*_pb2.py,.*_pb2_grpc.py +ignore-patterns=.*_pb2.py,.*_pb2_grpc.py,.*.pyi # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). @@ -51,6 +51,12 @@ persistent=yes # Specify a configuration file. #rcfile= +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots=src + # When enabled, pylint would attempt to guess common misconfiguration and emit # user-friendly hints instead of false-positive error messages suggestion-mode=yes @@ -223,7 +229,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local,contextlib.closing, # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis. It # supports qualified module names, as well as Unix pattern matching. -ignored-modules=pkg_resources,gi.repository,grpc,buildstream._protos.* +ignored-modules=pkg_resources,gi.repository,grpc,buildstream._protos.*,google.protobuf.* # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. diff --git a/requirements/cov-requirements.txt b/requirements/cov-requirements.txt index 04df6cbaf..538b93813 100644 --- a/requirements/cov-requirements.txt +++ b/requirements/cov-requirements.txt @@ -1,11 +1,8 @@ -coverage==7.2.1 -pytest-cov==4.0.0 -pytest==7.2.1 -Cython==0.29.33 +coverage==7.6.0 +pytest-cov==5.0.0 +pytest==8.2.2 +Cython==3.0.10 ## The following requirements were added by pip freeze: -attrs==22.2.0 -exceptiongroup==1.1.0 iniconfig==2.0.0 -packaging==23.0 -pluggy==1.0.0 -tomli==2.0.1 +packaging==24.1 +pluggy==1.5.0 diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 3bb641466..6d3c8f94f 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -1,29 +1,21 @@ -pexpect==4.8.0 -pylint==2.16.3 +pexpect==4.9.0 +pylint==3.2.5 # Pytest 6.0.0 doesn't play well with pylint -pytest==7.2.1 +pytest==8.2.2 pytest-datafiles==3.0.0 -pytest-env==0.8.1 -pytest-xdist==3.2.0 -pytest-timeout==2.1.0 -pyftpdlib==1.5.9 -# isort is an indirect requirement, but it must be constrained -# as 5.12 is not available for python 3.7 -isort==5.11.5 +pytest-env==1.1.3 +pytest-xdist==3.6.1 +pytest-timeout==2.3.1 +pyftpdlib==1.5.10 ## The following requirements were added by pip freeze: -astroid==2.14.2 -attrs==22.2.0 -dill==0.3.6 -exceptiongroup==1.1.0 -execnet==1.9.0 +astroid==3.2.3 +dill==0.3.8 +execnet==2.1.1 iniconfig==2.0.0 -lazy-object-proxy==1.9.0 +isort==5.13.2 mccabe==0.7.0 -packaging==23.0 -platformdirs==3.0.0 -pluggy==1.0.0 +packaging==24.1 +platformdirs==4.2.2 +pluggy==1.5.0 ptyprocess==0.7.0 -tomli==2.0.1 -tomlkit==0.11.6 -typing-extensions==4.5.0 -wrapt==1.15.0 +tomlkit==0.13.0 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index a9a9e9545..71485fc8a 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,13 +1,13 @@ -click==8.1.3 -grpcio==1.59.3 -Jinja2==3.1.2 +click==8.1.7 +grpcio==1.65.1 +Jinja2==3.1.4 pluginbase==1.0.1 -protobuf==4.22.0 -psutil==5.9.4 -ruamel.yaml==0.17.21 +protobuf==5.27.2 +psutil==6.0.0 +ruamel.yaml==0.18.6 ruamel.yaml.clib==0.2.8 -setuptools==67.4.0 -pyroaring==0.4.4 -ujson==5.7.0 +setuptools==71.0.3 +pyroaring==0.4.5 +ujson==5.10.0 ## The following requirements were added by pip freeze: -MarkupSafe==2.1.2 +MarkupSafe==2.1.5 diff --git a/setup.py b/setup.py index b1a61887f..78b71a546 100755 --- a/setup.py +++ b/setup.py @@ -194,8 +194,7 @@ def get_args(cls, dist, header=None): cls._ensure_safe_name(name) script_text = TEMPLATE.format(ep.module_name, ep.attrs[0], ".".join(ep.attrs)) args = cls._get_script_args("console", name, header, script_text) - for res in args: - yield res + yield from args ScriptWriter.get_args = get_args diff --git a/src/buildstream/_elementsources.py b/src/buildstream/_elementsources.py index 549e12e48..c88acaef6 100644 --- a/src/buildstream/_elementsources.py +++ b/src/buildstream/_elementsources.py @@ -72,8 +72,7 @@ class ElementSources: # Source: The individual sources # def sources(self) -> Iterator["Source"]: - for source in self._sources: - yield source + yield from self._sources # track(): # diff --git a/src/buildstream/_pipeline.py b/src/buildstream/_pipeline.py index cb8afd04d..8a8a9e989 100644 --- a/src/buildstream/_pipeline.py +++ b/src/buildstream/_pipeline.py @@ -50,8 +50,7 @@ def dependencies(targets: List[Element], scope: int, *, recurse: bool = True) -> visited = (BitMap(), BitMap()) for target in targets: - for element in target._dependencies(scope, recurse=recurse, visited=visited): - yield element + yield from target._dependencies(scope, recurse=recurse, visited=visited) # get_selection() diff --git a/src/buildstream/_pluginfactory/__init__.py b/src/buildstream/_pluginfactory/__init__.py index bc3f798b8..3ed1cf1f3 100644 --- a/src/buildstream/_pluginfactory/__init__.py +++ b/src/buildstream/_pluginfactory/__init__.py @@ -42,6 +42,8 @@ def load_plugin_origin(project, origin_node): origin = PluginOriginPip() elif origin_type == PluginOriginType.JUNCTION: origin = PluginOriginJunction() + else: + assert False, "unreachable" origin.initialize(project, origin_node) diff --git a/src/buildstream/_pluginfactory/pluginoriginjunction.py b/src/buildstream/_pluginfactory/pluginoriginjunction.py index 1f719df01..4f5c10b20 100644 --- a/src/buildstream/_pluginfactory/pluginoriginjunction.py +++ b/src/buildstream/_pluginfactory/pluginoriginjunction.py @@ -41,6 +41,8 @@ class PluginOriginJunction(PluginOrigin): factory = project.source_factory elif plugin_type == PluginType.ELEMENT: factory = project.element_factory + else: + assert False, "unreachable" # Now ask for the paths from the subproject PluginFactory try: diff --git a/src/buildstream/_pluginfactory/pluginoriginpip.py b/src/buildstream/_pluginfactory/pluginoriginpip.py index 00ac58e96..a50534f60 100644 --- a/src/buildstream/_pluginfactory/pluginoriginpip.py +++ b/src/buildstream/_pluginfactory/pluginoriginpip.py @@ -41,6 +41,8 @@ class PluginOriginPip(PluginOrigin): entrypoint_group = "buildstream.plugins.sources" elif plugin_type == PluginType.ELEMENT: entrypoint_group = "buildstream.plugins.elements" + else: + assert False, "unreachable" # key by a tuple to avoid collision try: diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index c3d8aa9ff..5877619b1 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -1151,9 +1151,8 @@ class Project: elif filename == WORKSPACE_PROJECT_FILE: workspace_project_cache = self._context.get_workspace_project_cache() workspace_project = workspace_project_cache.get(found_directory) - if workspace_project: - project_directory = workspace_project.get_default_project_path() - workspace_element = workspace_project.get_default_element() + project_directory = workspace_project.get_default_project_path() + workspace_element = workspace_project.get_default_element() else: raise LoadError( "None of {names} found in '{path}' or any of its parent directories".format( diff --git a/src/buildstream/_scheduler/scheduler.py b/src/buildstream/_scheduler/scheduler.py index 7faa07a5e..5b475daea 100644 --- a/src/buildstream/_scheduler/scheduler.py +++ b/src/buildstream/_scheduler/scheduler.py @@ -142,7 +142,7 @@ class Scheduler: # background threads. # In Python 3.8+, `ThreadedChildWatcher` is the default watcher, and # not `SafeChildWatcher`. - asyncio.set_child_watcher(asyncio.SafeChildWatcher()) + asyncio.set_child_watcher(asyncio.SafeChildWatcher()) # pylint: disable=deprecated-class # Ensure that we have a fresh new event loop, in case we want # to run another test in this thread. diff --git a/src/buildstream/_testing/runcli.py b/src/buildstream/_testing/runcli.py index 87682f009..2443e9093 100644 --- a/src/buildstream/_testing/runcli.py +++ b/src/buildstream/_testing/runcli.py @@ -703,6 +703,7 @@ class TestArtifact: yield None else: if str(artifact.buildtree): + # pylint: disable-next=contextmanager-generator-missing-cleanup with self._extract_subdirectory(tmpdir, artifact.buildtree) as f: yield f else: diff --git a/src/buildstream/element.py b/src/buildstream/element.py index d8f5d6758..eacbcf0a6 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -1407,6 +1407,7 @@ class Element(Plugin): ) # bst shell and bst artifact checkout require a local sandbox. + # pylint: disable-next=contextmanager-generator-missing-cleanup with self.__sandbox(None, config=self.__sandbox_config, allow_remote=False) as sandbox: # Configure always comes first, and we need it. @@ -2847,6 +2848,7 @@ class Element(Plugin): os.makedirs(context.builddir, exist_ok=True) # Recursive contextmanager... + # pylint: disable-next=contextmanager-generator-missing-cleanup with utils._tempdir( prefix="{}-".format(self.normal_name), dir=context.builddir ) as rootdir, self.__sandbox( diff --git a/src/buildstream/source.py b/src/buildstream/source.py index 72440e578..1f8bb95ff 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -765,6 +765,9 @@ class Source(Plugin): elif alias_override is not None: override_subst = alias_override + else: + assert False, "unreachable" + # The default source mirror will give prefix URLs if isinstance(override_subst._mirror, str): return override_subst._mirror + url_body @@ -1060,7 +1063,7 @@ class Source(Plugin): def do_load_ref(node): try: - self.load_ref(ref_node) + self.load_ref(node) except ImplError as e: raise SourceError( "{}: Storing refs in project.refs is not supported by '{}' sources".format(self, self.get_kind()), diff --git a/src/buildstream/storage/_casbaseddirectory.py b/src/buildstream/storage/_casbaseddirectory.py index 545ee0832..9c2e83a7b 100644 --- a/src/buildstream/storage/_casbaseddirectory.py +++ b/src/buildstream/storage/_casbaseddirectory.py @@ -649,11 +649,12 @@ class CasBasedDirectory(Directory): if prop.name == "SubtreeReadOnly": self.__subtree_read_only = prop.value == "true" - for entry in pb2_directory.directories: - self.__index[entry.name] = _IndexEntry( - self.__cas_cache, entry.name, FileType.DIRECTORY, digest=entry.digest + for dentry in pb2_directory.directories: + self.__index[dentry.name] = _IndexEntry( + self.__cas_cache, dentry.name, FileType.DIRECTORY, digest=dentry.digest ) for entry in pb2_directory.files: + mtime: Optional[timestamp_pb2.Timestamp] if entry.node_properties.HasField("mtime"): mtime = entry.node_properties.mtime else: @@ -667,8 +668,10 @@ class CasBasedDirectory(Directory): is_executable=entry.is_executable, mtime=mtime, ) - for entry in pb2_directory.symlinks: - self.__index[entry.name] = _IndexEntry(self.__cas_cache, entry.name, FileType.SYMLINK, target=entry.target) + for lentry in pb2_directory.symlinks: + self.__index[lentry.name] = _IndexEntry( + self.__cas_cache, lentry.name, FileType.SYMLINK, target=lentry.target + ) def __add_directory(self, name: str) -> "CasBasedDirectory": assert name not in self.__index diff --git a/src/buildstream/utils.py b/src/buildstream/utils.py index a7720616c..320d5332f 100644 --- a/src/buildstream/utils.py +++ b/src/buildstream/utils.py @@ -921,7 +921,7 @@ def _force_rmtree(rootpath): os.remove(path) try: - shutil.rmtree(rootpath, onerror=fix_permissions) + shutil.rmtree(rootpath, onerror=fix_permissions) # pylint: disable=deprecated-argument except OSError as e: raise UtilError("Failed to remove cache directory '{}': {}".format(rootpath, e)) diff --git a/tests/frontend/artifact_list_contents.py b/tests/frontend/artifact_list_contents.py index 37cfe66a4..e4611313c 100644 --- a/tests/frontend/artifact_list_contents.py +++ b/tests/frontend/artifact_list_contents.py @@ -47,6 +47,8 @@ def test_artifact_list_exact_contents(cli, datafiles, target, with_project): elif target == "artifact-name": key = cli.get_element_key(project, "import-bin.bst") arg = "test/import-bin/" + key + else: + assert False, "unreachable" # Delete the project.conf if we're going to try this without a project if not with_project: @@ -89,6 +91,8 @@ def test_artifact_list_exact_contents_long(cli, datafiles, target): elif target == "artifact-name": key = cli.get_element_key(project, "import-bin.bst") arg = "test/import-bin/" + key + else: + assert False, "unreachable" # List the contents via the element name result = cli.run(project=project, args=["artifact", "list-contents", "--long", arg]) diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py index 7054abc22..38ad6c831 100644 --- a/tests/frontend/completions.py +++ b/tests/frontend/completions.py @@ -387,5 +387,7 @@ def test_argument_artifact(cli, datafiles): elif i == 2: expected1 = artifacts expected2 = list(reversed(artifacts)) + else: + assert False, "unreachable" assert words in (expected1, expected2) diff --git a/tests/remoteexecution/workspace.py b/tests/remoteexecution/workspace.py index 636b79afe..302fd1ded 100644 --- a/tests/remoteexecution/workspace.py +++ b/tests/remoteexecution/workspace.py @@ -301,6 +301,9 @@ def test_workspace_build(cli, tmpdir, datafiles, modification): fdata.write(re.sub(r"Hello", "Goodbye", line)) touched_time = int(os.stat(main_path).st_mtime) + else: + assert False, "unreachable" + # refresh input times ws_times = get_mtimes(workspace) diff --git a/tox.ini b/tox.ini index e5fd88b0b..25f273717 100644 --- a/tox.ini +++ b/tox.ini @@ -166,7 +166,7 @@ commands_pre = {envpython} setup.py build_ext --inplace commands = - pylint {posargs: src/buildstream tests doc/source/conf.py setup.py} + pylint {posargs: buildstream tests doc/source/conf.py setup.py} deps = -rrequirements/requirements.txt -rrequirements/dev-requirements.txt
