Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package python-fsspec for openSUSE:Factory
checked in at 2022-02-24 18:20:44
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-fsspec (Old)
and /work/SRC/openSUSE:Factory/.python-fsspec.new.1958 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-fsspec"
Thu Feb 24 18:20:44 2022 rev:18 rq:957036 version:2022.2.0
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-fsspec/python-fsspec.changes
2022-02-01 14:03:13.183964691 +0100
+++ /work/SRC/openSUSE:Factory/.python-fsspec.new.1958/python-fsspec.changes
2022-02-24 18:24:04.398648791 +0100
@@ -1,0 +2,11 @@
+Tue Feb 22 23:01:18 UTC 2022 - Matej Cepl <mc...@suse.com>
+
+- Update to 2022.02.0:
+ - reference FS performance
+ - directory/prefix FS
+ - FUSE
+ - iteration in threads
+ - OpenFiles slicing
+ - drop py36
+
+-------------------------------------------------------------------
Old:
----
fsspec-2022.01.0.tar.gz
New:
----
fsspec-2022.02.0.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-fsspec.spec ++++++
--- /var/tmp/diff_new_pack.axyqjF/_old 2022-02-24 18:24:04.886648664 +0100
+++ /var/tmp/diff_new_pack.axyqjF/_new 2022-02-24 18:24:04.890648662 +0100
@@ -26,9 +26,9 @@
%bcond_with test
%endif
%define skip_python2 1
-%define ghversion 2022.01.0
+%define ghversion 2022.02.0
Name: python-fsspec%{psuffix}
-Version: 2022.1.0
+Version: 2022.2.0
Release: 0
Summary: Filesystem specification package
License: BSD-3-Clause
@@ -69,6 +69,7 @@
BuildRequires: %{python_module panel}
BuildRequires: %{python_module paramiko}
BuildRequires: %{python_module pyftpdlib}
+BuildRequires: %{python_module pytest-mock}
BuildRequires: %{python_module pytest}
BuildRequires: %{python_module python-snappy}
BuildRequires: %{python_module requests}
++++++ fsspec-2022.01.0.tar.gz -> fsspec-2022.02.0.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2022.01.0/.github/workflows/main.yaml
new/filesystem_spec-2022.02.0/.github/workflows/main.yaml
--- old/filesystem_spec-2022.01.0/.github/workflows/main.yaml 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/.github/workflows/main.yaml 2022-02-22
18:44:54.000000000 +0100
@@ -13,7 +13,7 @@
strategy:
fail-fast: false
matrix:
- TOXENV: [py36, py37, py38, py39, s3fs, gcsfs]
+ TOXENV: [py37, py38, py39, s3fs, gcsfs]
env:
TOXENV: ${{ matrix.TOXENV }}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/.gitignore
new/filesystem_spec-2022.02.0/.gitignore
--- old/filesystem_spec-2022.01.0/.gitignore 2022-01-11 16:29:11.000000000
+0100
+++ new/filesystem_spec-2022.02.0/.gitignore 2022-02-22 18:44:54.000000000
+0100
@@ -117,4 +117,7 @@
# docker artifacts
.docker
+# vi*
+*.swp
+
build/
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/ci/environment-win.yml
new/filesystem_spec-2022.02.0/ci/environment-win.yml
--- old/filesystem_spec-2022.01.0/ci/environment-win.yml 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/ci/environment-win.yml 2022-02-22
18:44:54.000000000 +0100
@@ -12,8 +12,10 @@
- pyftpdlib
- cloudpickle
- pytest
+ - pytest-asyncio
- pytest-benchmark
- pytest-cov
+ - pytest-mock
- pytest-vcr
- python-libarchive-c
- numpy
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/docs/source/api.rst
new/filesystem_spec-2022.02.0/docs/source/api.rst
--- old/filesystem_spec-2022.01.0/docs/source/api.rst 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/docs/source/api.rst 2022-02-22
18:44:54.000000000 +0100
@@ -119,6 +119,7 @@
fsspec.implementations.libarchive.LibArchiveFileSystem
fsspec.implementations.dbfs.DatabricksFileSystem
fsspec.implementations.reference.ReferenceFileSystem
+ fsspec.implementations.dirfs.DirFileSystem
.. autoclass:: fsspec.implementations.ftp.FTPFileSystem
:members: __init__
@@ -183,6 +184,9 @@
.. autoclass:: fsspec.implementations.reference.ReferenceFileSystem
:members: __init__
+.. autoclass:: fsspec.implementations.dirfs.DirFileSystem
+ :members: __init__
+
Other Known Implementations
---------------------------
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/docs/source/changelog.rst
new/filesystem_spec-2022.02.0/docs/source/changelog.rst
--- old/filesystem_spec-2022.01.0/docs/source/changelog.rst 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/docs/source/changelog.rst 2022-02-22
18:44:54.000000000 +0100
@@ -1,6 +1,24 @@
Changelog
=========
+2022.02.0
+---------
+
+Enhancements
+
+- reference FS performance (#892, 900)
+- directory/prefix FS (#745)
+
+Fixes
+
+- FUSE (#905, 891)
+- iteration in threads (#893)
+- OpenFiles slicing (#887)
+
+Other
+
+- drop py36 (#889, 901)
+
2022.01.0
---------
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/docs/source/index.rst
new/filesystem_spec-2022.02.0/docs/source/index.rst
--- old/filesystem_spec-2022.01.0/docs/source/index.rst 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/docs/source/index.rst 2022-02-22
18:44:54.000000000 +0100
@@ -55,6 +55,7 @@
``fsspec`` filesystems are also supported by:
#. `pyarrow`_, the in-memory data layout engine
+#. `petl`_, a general purpose package for extracting, transforming and loading
tables of data.
... plus many more that we don't know about.
@@ -66,7 +67,7 @@
.. _DVC: https://dvc.org/
.. _kedro:
https://kedro.readthedocs.io/en/stable/01_introduction/01_introduction.html
.. _pyarrow: https://arrow.apache.org/docs/python/
-
+.. _petl:
https://petl.readthedocs.io/en/stable/io.html#petl.io.remotes.RemoteSource
Installation
------------
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/fsspec/_version.py
new/filesystem_spec-2022.02.0/fsspec/_version.py
--- old/filesystem_spec-2022.01.0/fsspec/_version.py 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/_version.py 2022-02-22
18:44:54.000000000 +0100
@@ -22,9 +22,9 @@
# setup.py/versioneer.py will grep for the variable names, so they must
# each be defined on a line of their own. _version.py will just call
# get_keywords().
- git_refnames = " (tag: 2022.01.0)"
- git_full = "ece2fe15b1cac41167b31b8421ac9936aef29010"
- git_date = "2022-01-11 10:29:11 -0500"
+ git_refnames = " (HEAD -> master, tag: 2022.02.0)"
+ git_full = "f9089f5ce97e1e52ab70ce1f372fc4c0feed5132"
+ git_date = "2022-02-22 12:44:54 -0500"
keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
return keywords
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/fsspec/asyn.py
new/filesystem_spec-2022.02.0/fsspec/asyn.py
--- old/filesystem_spec-2022.01.0/fsspec/asyn.py 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/asyn.py 2022-02-22
18:44:54.000000000 +0100
@@ -12,7 +12,7 @@
from .callbacks import _DEFAULT_CALLBACK
from .exceptions import FSTimeoutError
from .spec import AbstractFileSystem
-from .utils import PY36, is_exception, other_paths
+from .utils import is_exception, other_paths
private = re.compile("_[^_]")
@@ -29,12 +29,6 @@
event.set()
-if PY36:
- grl = asyncio.events._get_running_loop
-else:
- grl = asyncio.events.get_running_loop
-
-
def sync(loop, func, *args, timeout=None, **kwargs):
"""
Make loop run coroutine until it returns. Runs in other thread
@@ -45,7 +39,7 @@
if loop is None or loop.is_closed():
raise RuntimeError("Loop is not running")
try:
- loop0 = grl()
+ loop0 = asyncio.events.get_running_loop()
if loop0 is loop:
raise NotImplementedError("Calling sync() from within a running
loop")
except RuntimeError:
@@ -240,9 +234,7 @@
]
if callback is not _DEFAULT_CALLBACK:
[
- t.add_done_callback(
- lambda *_, **__: callback.call("relative_update", 1)
- )
+ t.add_done_callback(lambda *_, **__:
callback.relative_update(1))
for t in chunk
]
results.extend(
@@ -482,7 +474,7 @@
batch_size = batch_size or self.batch_size
coros = []
- callback.call("set_size", len(file_pairs))
+ callback.set_size(len(file_pairs))
for lfile, rfile in file_pairs:
callback.branch(lfile, rfile, kwargs)
coros.append(self._put_file(lfile, rfile, **kwargs))
@@ -521,7 +513,7 @@
batch_size = kwargs.pop("batch_size", self.batch_size)
coros = []
- callback.lazy_call("set_size", len, lpaths)
+ callback.set_size(len(lpaths))
for lpath, rpath in zip(lpaths, rpaths):
callback.branch(rpath, lpath, kwargs)
coros.append(self._get_file(rpath, lpath, **kwargs))
@@ -790,9 +782,6 @@
):
import traceback
- if PY36:
- raise NotImplementedError("Do not call this on Py 3.6")
-
tasks = [t for t in asyncio.tasks.all_tasks(loop[0]) if not t.done()]
if printout:
[task.print_stack() for task in tasks]
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/fsspec/core.py
new/filesystem_spec-2022.02.0/fsspec/core.py
--- old/filesystem_spec-2022.01.0/fsspec/core.py 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/core.py 2022-02-22
18:44:54.000000000 +0100
@@ -204,6 +204,12 @@
break
[s.__exit__(*args) for s in self]
+ def __getitem__(self, item):
+ out = super().__getitem__(item)
+ if isinstance(item, slice):
+ return OpenFiles(out, mode=self.mode, fs=self.fs)
+ return out
+
def __repr__(self):
return "<List of %s OpenFile instances>" % len(self)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/fsspec/fuse.py
new/filesystem_spec-2022.02.0/fsspec/fuse.py
--- old/filesystem_spec-2022.01.0/fsspec/fuse.py 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/fuse.py 2022-02-22
18:44:54.000000000 +0100
@@ -82,6 +82,7 @@
def write(self, path, data, offset, fh):
logger.debug("write %s", (path, offset))
f = self.cache[fh]
+ f.seek(offset)
f.write(data)
return len(data)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2022.01.0/fsspec/implementations/dirfs.py
new/filesystem_spec-2022.02.0/fsspec/implementations/dirfs.py
--- old/filesystem_spec-2022.01.0/fsspec/implementations/dirfs.py
1970-01-01 01:00:00.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/implementations/dirfs.py
2022-02-22 18:44:54.000000000 +0100
@@ -0,0 +1,325 @@
+from ..asyn import AsyncFileSystem
+
+
+class DirFileSystem(AsyncFileSystem):
+ def __init__(self, path, fs, *args, **storage_options):
+ """
+ Parameters
+ ----------
+ path: str
+ Path to the directory.
+ fs: AbstractFileSystem
+ An instantiated filesystem to wrap.
+ """
+ super().__init__(*args, **storage_options)
+
+ if self.asynchronous and not fs.async_impl:
+ raise ValueError("can't use asynchronous with non-async fs")
+
+ if fs.async_impl and self.asynchronous != fs.asynchronous:
+ raise ValueError("both dirfs and fs should be in the same
sync/async mode")
+
+ self.path = fs._strip_protocol(path)
+ self.fs = fs
+
+ def _join(self, path):
+ if isinstance(path, str):
+ if not self.path:
+ return path
+ if not path:
+ return self.path
+ return self.fs.sep.join((self.path, path))
+ return [self._join(_path) for _path in path]
+
+ def _relpath(self, path):
+ if isinstance(path, str):
+ if not self.path:
+ return path
+ if path == self.path:
+ return ""
+ prefix = self.path + self.fs.sep
+ assert path.startswith(prefix)
+ return path[len(prefix) :]
+ return [self._relpath(_path) for _path in path]
+
+ # Wrappers below
+
+ @property
+ def sep(self):
+ return self.fs.sep
+
+ async def set_session(self, *args, **kwargs):
+ return await self.fs.set_session(*args, **kwargs)
+
+ async def _rm_file(self, path, **kwargs):
+ return await self.fs._rm_file(self._join(path), **kwargs)
+
+ def rm_file(self, path, **kwargs):
+ return self.fs.rm_file(self._join(path), **kwargs)
+
+ async def _rm(self, path, *args, **kwargs):
+ return await self.fs._rm(self._join(path), *args, **kwargs)
+
+ def rm(self, path, *args, **kwargs):
+ return self.fs.rm(self._join(path), *args, **kwargs)
+
+ async def _cp_file(self, path1, path2, **kwargs):
+ return await self.fs._cp_file(self._join(path1), self._join(path2),
**kwargs)
+
+ def cp_file(self, path1, path2, **kwargs):
+ return self.fs.cp_file(self._join(path1), self._join(path2), **kwargs)
+
+ async def _copy(
+ self,
+ path1,
+ path2,
+ *args,
+ **kwargs,
+ ):
+ return await self.fs._copy(
+ self._join(path1),
+ self._join(path2),
+ *args,
+ **kwargs,
+ )
+
+ def copy(self, path1, path2, *args, **kwargs):
+ return self.fs.copy(
+ self._join(path1),
+ self._join(path2),
+ *args,
+ **kwargs,
+ )
+
+ async def _pipe(self, path, *args, **kwargs):
+ return await self.fs._pipe(self._join(path), *args, **kwargs)
+
+ def pipe(self, path, *args, **kwargs):
+ return self.fs.pipe(self._join(path), *args, **kwargs)
+
+ async def _cat_file(self, path, *args, **kwargs):
+ return await self.fs._cat_file(self._join(path), *args, **kwargs)
+
+ def cat_file(self, path, *args, **kwargs):
+ return self.fs.cat_file(self._join(path), *args, **kwargs)
+
+ async def _cat(self, path, *args, **kwargs):
+ ret = await self.fs._cat(
+ self._join(path),
+ *args,
+ **kwargs,
+ )
+
+ if isinstance(ret, dict):
+ return {self._relpath(key): value for key, value in ret.items()}
+
+ return ret
+
+ def cat(self, path, *args, **kwargs):
+ ret = self.fs.cat(
+ self._join(path),
+ *args,
+ **kwargs,
+ )
+
+ if isinstance(ret, dict):
+ return {self._relpath(key): value for key, value in ret.items()}
+
+ return ret
+
+ async def _put_file(self, lpath, rpath, **kwargs):
+ return await self.fs._put_file(lpath, self._join(rpath), **kwargs)
+
+ def put_file(self, lpath, rpath, **kwargs):
+ return self.fs.put_file(lpath, self._join(rpath), **kwargs)
+
+ async def _put(
+ self,
+ lpath,
+ rpath,
+ *args,
+ **kwargs,
+ ):
+ return await self.fs._put(
+ lpath,
+ self._join(rpath),
+ *args,
+ **kwargs,
+ )
+
+ def put(self, lpath, rpath, *args, **kwargs):
+ return self.fs.put(
+ lpath,
+ self._join(rpath),
+ *args,
+ **kwargs,
+ )
+
+ async def _get_file(self, rpath, lpath, **kwargs):
+ return await self.fs._get_file(self._join(rpath), lpath, **kwargs)
+
+ def get_file(self, rpath, lpath, **kwargs):
+ return self.fs.get_file(self._join(rpath), lpath, **kwargs)
+
+ async def _get(self, rpath, *args, **kwargs):
+ return await self.fs._get(self._join(rpath), *args, **kwargs)
+
+ def get(self, rpath, *args, **kwargs):
+ return self.fs.get(self._join(rpath), *args, **kwargs)
+
+ async def _isfile(self, path):
+ return await self.fs._isfile(self._join(path))
+
+ def isfile(self, path):
+ return self.fs.isfile(self._join(path))
+
+ async def _isdir(self, path):
+ return await self.fs._isdir(self._join(path))
+
+ def isdir(self, path):
+ return self.fs.isdir(self._join(path))
+
+ async def _size(self, path):
+ return await self.fs._size(self._join(path))
+
+ def size(self, path):
+ return self.fs.size(self._join(path))
+
+ async def _exists(self, path):
+ return await self.fs._exists(self._join(path))
+
+ def exists(self, path):
+ return self.fs.exists(self._join(path))
+
+ async def _info(self, path, **kwargs):
+ return await self.fs._info(self._join(path), **kwargs)
+
+ def info(self, path, **kwargs):
+ return self.fs.info(self._join(path), **kwargs)
+
+ async def _ls(self, path, detail=True, **kwargs):
+ ret = await self.fs._ls(self._join(path), detail=detail, **kwargs)
+ if detail:
+ for entry in ret:
+ entry["name"] = self._relpath(entry["name"])
+ return ret
+
+ return self._relpath(ret)
+
+ def ls(self, path, detail=True, **kwargs):
+ ret = self.fs.ls(self._join(path), detail=detail, **kwargs)
+ if detail:
+ for entry in ret:
+ entry["name"] = self._relpath(entry["name"])
+ return ret
+
+ return self._relpath(ret)
+
+ async def _walk(self, path, *args, **kwargs):
+ async for root, dirs, files in self.fs._walk(self._join(path), *args,
**kwargs):
+ yield self._relpath(root), dirs, files
+
+ def walk(self, path, *args, **kwargs):
+ for root, dirs, files in self.fs.walk(self._join(path), *args,
**kwargs):
+ yield self._relpath(root), dirs, files
+
+ async def _glob(self, path, **kwargs):
+ detail = kwargs.get("detail", False)
+ ret = await self.fs._glob(self._join(path), **kwargs)
+ if detail:
+ return {self._relpath(path): info for path, info in ret.items()}
+ return self._relpath(ret)
+
+ def glob(self, path, **kwargs):
+ detail = kwargs.get("detail", False)
+ ret = self.fs.glob(self._join(path), **kwargs)
+ if detail:
+ return {self._relpath(path): info for path, info in ret.items()}
+ return self._relpath(ret)
+
+ async def _du(self, path, *args, **kwargs):
+ total = kwargs.get("total", True)
+ ret = await self.fs._du(self._join(path), *args, **kwargs)
+ if total:
+ return ret
+
+ return {self._relpath(path): size for path, size in ret.items()}
+
+ def du(self, path, *args, **kwargs):
+ total = kwargs.get("total", True)
+ ret = self.fs.du(self._join(path), *args, **kwargs)
+ if total:
+ return ret
+
+ return {self._relpath(path): size for path, size in ret.items()}
+
+ async def _find(self, path, *args, **kwargs):
+ detail = kwargs.get("detail", False)
+ ret = await self.fs._find(self._join(path), *args, **kwargs)
+ if detail:
+ return {self._relpath(path): info for path, info in ret.items()}
+ return self._relpath(ret)
+
+ def find(self, path, *args, **kwargs):
+ detail = kwargs.get("detail", False)
+ ret = self.fs.find(self._join(path), *args, **kwargs)
+ if detail:
+ return {self._relpath(path): info for path, info in ret.items()}
+ return self._relpath(ret)
+
+ async def _expand_path(self, path, *args, **kwargs):
+ return self._relpath(
+ await self.fs._expand_path(self._join(path), *args, **kwargs)
+ )
+
+ def expand_path(self, path, *args, **kwargs):
+ return self._relpath(self.fs.expand_path(self._join(path), *args,
**kwargs))
+
+ async def _mkdir(self, path, *args, **kwargs):
+ return await self.fs._mkdir(self._join(path), *args, **kwargs)
+
+ def mkdir(self, path, *args, **kwargs):
+ return self.fs.mkdir(self._join(path), *args, **kwargs)
+
+ async def _makedirs(self, path, *args, **kwargs):
+ return await self.fs._makedirs(self._join(path), *args, **kwargs)
+
+ def makedirs(self, path, *args, **kwargs):
+ return self.fs.makedirs(self._join(path), *args, **kwargs)
+
+ def rmdir(self, path):
+ return self.fs.rmdir(self._join(path))
+
+ def mv_file(self, path1, path2, **kwargs):
+ return self.fs.mv_file(
+ self._join(path1),
+ self._join(path2),
+ **kwargs,
+ )
+
+ def touch(self, path, **kwargs):
+ return self.fs.touch(self._join(path), **kwargs)
+
+ def created(self, path):
+ return self.fs.created(self._join(path))
+
+ def modified(self, path):
+ return self.fs.modified(self._join(path))
+
+ def sign(self, path, *args, **kwargs):
+ return self.fs.sign(self._join(path), *args, **kwargs)
+
+ def __repr__(self):
+ return f"{self.__class__.__qualname__}(path='{self.path}',
fs={self.fs})"
+
+ def open(
+ self,
+ path,
+ *args,
+ **kwargs,
+ ):
+ return self.fs.open(
+ self._join(path),
+ *args,
+ **kwargs,
+ )
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2022.01.0/fsspec/implementations/github.py
new/filesystem_spec-2022.02.0/fsspec/implementations/github.py
--- old/filesystem_spec-2022.01.0/fsspec/implementations/github.py
2022-01-11 16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/implementations/github.py
2022-02-22 18:44:54.000000000 +0100
@@ -156,15 +156,17 @@
if r.status_code == 404:
raise FileNotFoundError(path)
r.raise_for_status()
+ types = {"blob": "file", "tree": "directory"}
out = [
{
"name": path + "/" + f["path"] if path else f["path"],
"mode": f["mode"],
- "type": {"blob": "file", "tree": "directory"}[f["type"]],
+ "type": types[f["type"]],
"size": f.get("size", 0),
"sha": f["sha"],
}
for f in r.json()["tree"]
+ if f["type"] in types
]
if sha in [self.root, None]:
self.dircache[path] = out
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2022.01.0/fsspec/implementations/memory.py
new/filesystem_spec-2022.02.0/fsspec/implementations/memory.py
--- old/filesystem_spec-2022.01.0/fsspec/implementations/memory.py
2022-01-11 16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/implementations/memory.py
2022-02-22 18:44:54.000000000 +0100
@@ -46,7 +46,7 @@
paths = set()
starter = path + "/"
out = []
- for p2 in self.store:
+ for p2 in tuple(self.store):
if p2.startswith(starter):
if "/" not in p2[len(starter) :]:
# exact child
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2022.01.0/fsspec/implementations/reference.py
new/filesystem_spec-2022.02.0/fsspec/implementations/reference.py
--- old/filesystem_spec-2022.01.0/fsspec/implementations/reference.py
2022-01-11 16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/implementations/reference.py
2022-02-22 18:44:54.000000000 +0100
@@ -221,6 +221,7 @@
part_or_url, start0, end0 = self._cat_common(path)
if isinstance(part_or_url, bytes):
return part_or_url[start:end]
+ # TODO: update start0, end0 if start/end given, instead of slicing
return self.fs.cat_file(part_or_url, start=start0, end=end0)[start:end]
def pipe_file(self, path, value, **_):
@@ -238,10 +239,10 @@
if self.isdir(rpath):
return os.makedirs(lpath, exist_ok=True)
data = self.cat_file(rpath, **kwargs)
- callback.lazy_call("set_size", len, data)
+ callback.set_size(len(data))
with open(lpath, "wb") as f:
f.write(data)
- callback.lazy_call("absolute_update", len, data)
+ callback.absolute_update(len(data))
def get(self, rpath, lpath, recursive=False, **kwargs):
if self.fs.async_impl:
@@ -279,7 +280,6 @@
if self.templates:
self.df["url"] = self.df["url"].map(_render_jinja)
- self._dircache_from_items()
def _process_references(self, references, template_overrides=None):
if isinstance(references, (str, bytes)):
@@ -293,7 +293,6 @@
raise ValueError(f"Unknown reference spec version: {vers}")
# TODO: we make dircache by iterating over all entries, but for Spec
>= 1,
# can replace with programmatic. Is it even needed for mapper
interface?
- self._dircache_from_items()
def _process_references0(self, references):
"""Make reference dict for Spec Version 0"""
@@ -320,7 +319,7 @@
if v.startswith("base64:"):
self.references[k] = base64.b64decode(v[7:])
self.references[k] = v
- else:
+ elif self.templates:
u = v[0]
if "{{" in u:
if self.simple_templates:
@@ -332,6 +331,8 @@
else:
u = _render_jinja(u)
self.references[k] = [u] if len(v) == 1 else [u, v[1], v[2]]
+ else:
+ self.references[k] = v
self.references.update(self._process_gen(references.get("gen", [])))
def _process_templates(self, tmp):
@@ -422,6 +423,8 @@
def ls(self, path, detail=True, **kwargs):
path = self._strip_protocol(path)
+ if not self.dircache:
+ self._dircache_from_items()
out = self._ls_from_cache(path)
if out is None:
raise FileNotFoundError
@@ -430,16 +433,18 @@
return [o["name"] for o in out]
def exists(self, path, **kwargs): # overwrite auto-sync version
- try:
- return self._ls_from_cache(path) is not None
- except FileNotFoundError:
- return False
+ return self.isdir(path) or self.isfile(path)
def isdir(self, path): # overwrite auto-sync version
- return self.exists(path) and self.info(path)["type"] == "directory"
+ if self.dircache:
+ return path in self.dircache
+ else:
+ # this may be faster than building dircache for single calls, but
+ # by looping will be slow for many calls; could cache it?
+ return any(_.startswith(f"{path}/") for _ in self.references)
def isfile(self, path): # overwrite auto-sync version
- return self.exists(path) and self.info(path)["type"] == "file"
+ return path in self.references
async def _ls(self, path, detail=True, **kwargs): # calls fast sync code
return self.ls(path, detail, **kwargs)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_dirfs.py
new/filesystem_spec-2022.02.0/fsspec/implementations/tests/test_dirfs.py
--- old/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_dirfs.py
1970-01-01 01:00:00.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/implementations/tests/test_dirfs.py
2022-02-22 18:44:54.000000000 +0100
@@ -0,0 +1,566 @@
+import sys
+
+import pytest
+
+from fsspec.asyn import AsyncFileSystem
+from fsspec.implementations.dirfs import DirFileSystem
+from fsspec.spec import AbstractFileSystem
+
+PATH = "path/to/dir"
+ARGS = ["foo", "bar"]
+KWARGS = {"baz": "baz", "qux": "qux"}
+
+
+@pytest.fixture
+def make_fs(mocker):
+ def _make_fs(async_impl=False, asynchronous=False):
+ attrs = {
+ "sep": "/",
+ "async_impl": async_impl,
+ "_strip_protocol": lambda path: path,
+ }
+
+ if async_impl:
+ if asynchronous and sys.version_info < (3, 8):
+ pytest.skip("no AsyncMock before Python 3.8")
+
+ attrs["asynchronous"] = asynchronous
+ cls = AsyncFileSystem
+ else:
+ cls = AbstractFileSystem
+
+ fs = mocker.MagicMock(spec=cls, **attrs)
+
+ return fs
+
+ return _make_fs
+
+
+@pytest.fixture(
+ params=[
+ pytest.param(False, id="sync"),
+ pytest.param(True, id="async"),
+ ]
+)
+def fs(make_fs, request):
+ return make_fs(async_impl=request.param)
+
+
+@pytest.fixture
+def asyncfs(make_fs):
+ return make_fs(async_impl=True, asynchronous=True)
+
+
+@pytest.fixture
+def make_dirfs():
+ def _make_dirfs(fs, asynchronous=False):
+ return DirFileSystem(PATH, fs, asynchronous=asynchronous)
+
+ return _make_dirfs
+
+
+@pytest.fixture
+def dirfs(make_dirfs, fs):
+ return make_dirfs(fs)
+
+
+@pytest.fixture
+def adirfs(make_dirfs, asyncfs):
+ return make_dirfs(asyncfs, asynchronous=True)
+
+
+def test_dirfs(fs, asyncfs):
+ DirFileSystem("path", fs)
+ DirFileSystem("path", asyncfs, asynchronous=True)
+
+ with pytest.raises(ValueError):
+ DirFileSystem("path", asyncfs)
+
+ with pytest.raises(ValueError):
+ DirFileSystem("path", fs, asynchronous=True)
+
+
+@pytest.mark.parametrize(
+ "root, rel, full",
+ [
+ ("", "", ""),
+ ("", "foo", "foo"),
+ ("root", "", "root"),
+ ("root", "foo", "root/foo"),
+ ],
+)
+def test_path(fs, root, rel, full):
+ dirfs = DirFileSystem(root, fs)
+ assert dirfs._join(rel) == full
+ assert dirfs._relpath(full) == rel
+
+
+def test_sep(mocker, dirfs):
+ sep = mocker.Mock()
+ dirfs.fs.sep = sep
+ assert dirfs.sep == sep
+
+
+@pytest.mark.asyncio
+async def test_set_session(mocker, adirfs):
+ adirfs.fs.set_session = mocker.AsyncMock()
+ assert (
+ await adirfs.set_session(*ARGS, **KWARGS) ==
adirfs.fs.set_session.return_value
+ )
+ adirfs.fs.set_session.assert_called_once_with(*ARGS, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_rm_file(adirfs):
+ await adirfs._rm_file("file", **KWARGS)
+ adirfs.fs._rm_file.assert_called_once_with(f"{PATH}/file", **KWARGS)
+
+
+def test_rm_file(dirfs):
+ dirfs.rm_file("file", **KWARGS)
+ dirfs.fs.rm_file.assert_called_once_with("path/to/dir/file", **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_rm(adirfs):
+ await adirfs._rm("file", *ARGS, **KWARGS)
+ adirfs.fs._rm.assert_called_once_with("path/to/dir/file", *ARGS, **KWARGS)
+
+
+def test_rm(dirfs):
+ dirfs.rm("file", *ARGS, **KWARGS)
+ dirfs.fs.rm.assert_called_once_with("path/to/dir/file", *ARGS, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_cp_file(adirfs):
+ await adirfs._cp_file("one", "two", **KWARGS)
+ adirfs.fs._cp_file.assert_called_once_with(f"{PATH}/one", f"{PATH}/two",
**KWARGS)
+
+
+def test_cp_file(dirfs):
+ dirfs.cp_file("one", "two", **KWARGS)
+ dirfs.fs.cp_file.assert_called_once_with(f"{PATH}/one", f"{PATH}/two",
**KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_copy(adirfs):
+ await adirfs._copy("one", "two", *ARGS, **KWARGS)
+ adirfs.fs._copy.assert_called_once_with(
+ f"{PATH}/one", f"{PATH}/two", *ARGS, **KWARGS
+ )
+
+
+def test_copy(dirfs):
+ dirfs.copy("one", "two", *ARGS, **KWARGS)
+ dirfs.fs.copy.assert_called_once_with(f"{PATH}/one", f"{PATH}/two", *ARGS,
**KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_pipe(adirfs):
+ await adirfs._pipe("file", *ARGS, **KWARGS)
+ adirfs.fs._pipe.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
+
+
+def test_pipe(dirfs):
+ dirfs.pipe("file", *ARGS, **KWARGS)
+ dirfs.fs.pipe.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_cat_file(adirfs):
+ assert (
+ await adirfs._cat_file("file", *ARGS, **KWARGS)
+ == adirfs.fs._cat_file.return_value
+ )
+ adirfs.fs._cat_file.assert_called_once_with(f"{PATH}/file", *ARGS,
**KWARGS)
+
+
+def test_cat_file(dirfs):
+ assert dirfs.cat_file("file", *ARGS, **KWARGS) ==
dirfs.fs.cat_file.return_value
+ dirfs.fs.cat_file.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_cat(adirfs):
+ assert await adirfs._cat("file", *ARGS, **KWARGS) ==
adirfs.fs._cat.return_value
+ adirfs.fs._cat.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
+
+
+def test_cat(dirfs):
+ assert dirfs.cat("file", *ARGS, **KWARGS) == dirfs.fs.cat.return_value
+ dirfs.fs.cat.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_cat_list(adirfs):
+ adirfs.fs._cat.return_value = {f"{PATH}/one": "foo", f"{PATH}/two": "bar"}
+ assert await adirfs._cat(["one", "two"], *ARGS, **KWARGS) == {
+ "one": "foo",
+ "two": "bar",
+ }
+ adirfs.fs._cat.assert_called_once_with(
+ [f"{PATH}/one", f"{PATH}/two"], *ARGS, **KWARGS
+ )
+
+
+def test_cat_list(dirfs):
+ dirfs.fs.cat.return_value = {f"{PATH}/one": "foo", f"{PATH}/two": "bar"}
+ assert dirfs.cat(["one", "two"], *ARGS, **KWARGS) == {"one": "foo", "two":
"bar"}
+ dirfs.fs.cat.assert_called_once_with(
+ [f"{PATH}/one", f"{PATH}/two"], *ARGS, **KWARGS
+ )
+
+
+@pytest.mark.asyncio
+async def test_async_put_file(adirfs):
+ await adirfs._put_file("local", "file", **KWARGS)
+ adirfs.fs._put_file.assert_called_once_with("local", f"{PATH}/file",
**KWARGS)
+
+
+def test_put_file(dirfs):
+ dirfs.put_file("local", "file", **KWARGS)
+ dirfs.fs.put_file.assert_called_once_with("local", f"{PATH}/file",
**KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_put(adirfs):
+ await adirfs._put("local", "file", **KWARGS)
+ adirfs.fs._put.assert_called_once_with("local", f"{PATH}/file", **KWARGS)
+
+
+def test_put(dirfs):
+ dirfs.put("local", "file", **KWARGS)
+ dirfs.fs.put.assert_called_once_with("local", f"{PATH}/file", **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_get_file(adirfs):
+ await adirfs._get_file("file", "local", **KWARGS)
+ adirfs.fs._get_file.assert_called_once_with(f"{PATH}/file", "local",
**KWARGS)
+
+
+def test_get_file(dirfs):
+ dirfs.get_file("file", "local", **KWARGS)
+ dirfs.fs.get_file.assert_called_once_with(f"{PATH}/file", "local",
**KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_get(adirfs):
+ await adirfs._get("file", "local", **KWARGS)
+ adirfs.fs._get.assert_called_once_with(f"{PATH}/file", "local", **KWARGS)
+
+
+def test_get(dirfs):
+ dirfs.get("file", "local", **KWARGS)
+ dirfs.fs.get.assert_called_once_with(f"{PATH}/file", "local", **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_isfile(adirfs):
+ assert await adirfs._isfile("file") == adirfs.fs._isfile.return_value
+ adirfs.fs._isfile.assert_called_once_with(f"{PATH}/file")
+
+
+def test_isfile(dirfs):
+ assert dirfs.isfile("file") == dirfs.fs.isfile.return_value
+ dirfs.fs.isfile.assert_called_once_with(f"{PATH}/file")
+
+
+@pytest.mark.asyncio
+async def test_async_isdir(adirfs):
+ assert await adirfs._isdir("file") == adirfs.fs._isdir.return_value
+ adirfs.fs._isdir.assert_called_once_with(f"{PATH}/file")
+
+
+def test_isdir(dirfs):
+ assert dirfs.isdir("file") == dirfs.fs.isdir.return_value
+ dirfs.fs.isdir.assert_called_once_with(f"{PATH}/file")
+
+
+@pytest.mark.asyncio
+async def test_async_size(adirfs):
+ assert await adirfs._size("file") == adirfs.fs._size.return_value
+ adirfs.fs._size.assert_called_once_with(f"{PATH}/file")
+
+
+def test_size(dirfs):
+ assert dirfs.size("file") == dirfs.fs.size.return_value
+ dirfs.fs.size.assert_called_once_with(f"{PATH}/file")
+
+
+@pytest.mark.asyncio
+async def test_async_exists(adirfs):
+ assert await adirfs._exists("file") == adirfs.fs._exists.return_value
+ adirfs.fs._exists.assert_called_once_with(f"{PATH}/file")
+
+
+def test_exists(dirfs):
+ assert dirfs.exists("file") == dirfs.fs.exists.return_value
+ dirfs.fs.exists.assert_called_once_with(f"{PATH}/file")
+
+
+@pytest.mark.asyncio
+async def test_async_info(adirfs):
+ assert await adirfs._info("file", **KWARGS) == adirfs.fs._info.return_value
+ adirfs.fs._info.assert_called_once_with(f"{PATH}/file", **KWARGS)
+
+
+def test_info(dirfs):
+ assert dirfs.info("file", **KWARGS) == dirfs.fs.info.return_value
+ dirfs.fs.info.assert_called_once_with(f"{PATH}/file", **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_ls(adirfs):
+ adirfs.fs._ls.return_value = [f"{PATH}/file"]
+ assert await adirfs._ls("file", detail=False, **KWARGS) == ["file"]
+ adirfs.fs._ls.assert_called_once_with(f"{PATH}/file", detail=False,
**KWARGS)
+
+
+def test_ls(dirfs):
+ dirfs.fs.ls.return_value = [f"{PATH}/file"]
+ assert dirfs.ls("file", detail=False, **KWARGS) == ["file"]
+ dirfs.fs.ls.assert_called_once_with(f"{PATH}/file", detail=False, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_ls_detail(adirfs):
+ adirfs.fs._ls.return_value = [{"name": f"{PATH}/file", "foo": "bar"}]
+ assert await adirfs._ls("file", detail=True, **KWARGS) == [
+ {"name": "file", "foo": "bar"}
+ ]
+ adirfs.fs._ls.assert_called_once_with(f"{PATH}/file", detail=True,
**KWARGS)
+
+
+def test_ls_detail(dirfs):
+ dirfs.fs.ls.return_value = [{"name": f"{PATH}/file", "foo": "bar"}]
+ assert dirfs.ls("file", detail=True, **KWARGS) == [{"name": "file", "foo":
"bar"}]
+ dirfs.fs.ls.assert_called_once_with(f"{PATH}/file", detail=True, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_walk(adirfs, mocker):
+ async def _walk(path, *args, **kwargs):
+ yield (f"{PATH}/root", ["foo", "bar"], ["baz", "qux"])
+
+ adirfs.fs._walk = mocker.MagicMock()
+ adirfs.fs._walk.side_effect = _walk
+
+ actual = []
+ async for entry in adirfs._walk("root", *ARGS, **KWARGS):
+ actual.append(entry)
+ assert actual == [("root", ["foo", "bar"], ["baz", "qux"])]
+ adirfs.fs._walk.assert_called_once_with(f"{PATH}/root", *ARGS, **KWARGS)
+
+
+def test_walk(dirfs):
+ dirfs.fs.walk.return_value = iter(
+ [(f"{PATH}/root", ["foo", "bar"], ["baz", "qux"])]
+ )
+ assert list(dirfs.walk("root", *ARGS, **KWARGS)) == [
+ ("root", ["foo", "bar"], ["baz", "qux"])
+ ]
+ dirfs.fs.walk.assert_called_once_with(f"{PATH}/root", *ARGS, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_glob(adirfs):
+ adirfs.fs._glob.return_value = [f"{PATH}/one", f"{PATH}/two"]
+ assert await adirfs._glob("*", **KWARGS) == ["one", "two"]
+ adirfs.fs._glob.assert_called_once_with(f"{PATH}/*", **KWARGS)
+
+
+def test_glob(dirfs):
+ dirfs.fs.glob.return_value = [f"{PATH}/one", f"{PATH}/two"]
+ assert dirfs.glob("*", **KWARGS) == ["one", "two"]
+ dirfs.fs.glob.assert_called_once_with(f"{PATH}/*", **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_glob_detail(adirfs):
+ adirfs.fs._glob.return_value = {
+ f"{PATH}/one": {"foo": "bar"},
+ f"{PATH}/two": {"baz": "qux"},
+ }
+ assert await adirfs._glob("*", detail=True, **KWARGS) == {
+ "one": {"foo": "bar"},
+ "two": {"baz": "qux"},
+ }
+ adirfs.fs._glob.assert_called_once_with(f"{PATH}/*", detail=True, **KWARGS)
+
+
+def test_glob_detail(dirfs):
+ dirfs.fs.glob.return_value = {
+ f"{PATH}/one": {"foo": "bar"},
+ f"{PATH}/two": {"baz": "qux"},
+ }
+ assert dirfs.glob("*", detail=True, **KWARGS) == {
+ "one": {"foo": "bar"},
+ "two": {"baz": "qux"},
+ }
+ dirfs.fs.glob.assert_called_once_with(f"{PATH}/*", detail=True, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_du(adirfs):
+ adirfs.fs._du.return_value = 1234
+ assert await adirfs._du("file", *ARGS, **KWARGS) == 1234
+ adirfs.fs._du.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
+
+
+def test_du(dirfs):
+ dirfs.fs.du.return_value = 1234
+ assert dirfs.du("file", *ARGS, **KWARGS) == 1234
+ dirfs.fs.du.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_du_granular(adirfs):
+ adirfs.fs._du.return_value = {f"{PATH}/dir/one": 1, f"{PATH}/dir/two": 2}
+ assert await adirfs._du("dir", *ARGS, total=False, **KWARGS) == {
+ "dir/one": 1,
+ "dir/two": 2,
+ }
+ adirfs.fs._du.assert_called_once_with(f"{PATH}/dir", *ARGS, total=False,
**KWARGS)
+
+
+def test_du_granular(dirfs):
+ dirfs.fs.du.return_value = {f"{PATH}/dir/one": 1, f"{PATH}/dir/two": 2}
+ assert dirfs.du("dir", *ARGS, total=False, **KWARGS) == {"dir/one": 1,
"dir/two": 2}
+ dirfs.fs.du.assert_called_once_with(f"{PATH}/dir", *ARGS, total=False,
**KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_find(adirfs):
+ adirfs.fs._find.return_value = [f"{PATH}/dir/one", f"{PATH}/dir/two"]
+ assert await adirfs._find("dir", *ARGS, **KWARGS) == ["dir/one", "dir/two"]
+ adirfs.fs._find.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)
+
+
+def test_find(dirfs):
+ dirfs.fs.find.return_value = [f"{PATH}/dir/one", f"{PATH}/dir/two"]
+ assert dirfs.find("dir", *ARGS, **KWARGS) == ["dir/one", "dir/two"]
+ dirfs.fs.find.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_find_detail(adirfs):
+ adirfs.fs._find.return_value = {
+ f"{PATH}/dir/one": {"foo": "bar"},
+ f"{PATH}/dir/two": {"baz": "qux"},
+ }
+ assert await adirfs._find("dir", *ARGS, detail=True, **KWARGS) == {
+ "dir/one": {"foo": "bar"},
+ "dir/two": {"baz": "qux"},
+ }
+ adirfs.fs._find.assert_called_once_with(f"{PATH}/dir", *ARGS, detail=True,
**KWARGS)
+
+
+def test_find_detail(dirfs):
+ dirfs.fs.find.return_value = {
+ f"{PATH}/dir/one": {"foo": "bar"},
+ f"{PATH}/dir/two": {"baz": "qux"},
+ }
+ assert dirfs.find("dir", *ARGS, detail=True, **KWARGS) == {
+ "dir/one": {"foo": "bar"},
+ "dir/two": {"baz": "qux"},
+ }
+ dirfs.fs.find.assert_called_once_with(f"{PATH}/dir", *ARGS, detail=True,
**KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_expand_path(adirfs):
+ adirfs.fs._expand_path.return_value = [f"{PATH}/file"]
+ assert await adirfs._expand_path("*", *ARGS, **KWARGS) == ["file"]
+ adirfs.fs._expand_path.assert_called_once_with(f"{PATH}/*", *ARGS,
**KWARGS)
+
+
+def test_expand_path(dirfs):
+ dirfs.fs.expand_path.return_value = [f"{PATH}/file"]
+ assert dirfs.expand_path("*", *ARGS, **KWARGS) == ["file"]
+ dirfs.fs.expand_path.assert_called_once_with(f"{PATH}/*", *ARGS, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_expand_path_list(adirfs):
+ adirfs.fs._expand_path.return_value = [f"{PATH}/1file", f"{PATH}/2file"]
+ assert await adirfs._expand_path(["1*", "2*"], *ARGS, **KWARGS) == [
+ "1file",
+ "2file",
+ ]
+ adirfs.fs._expand_path.assert_called_once_with(
+ [f"{PATH}/1*", f"{PATH}/2*"], *ARGS, **KWARGS
+ )
+
+
+def test_expand_path_list(dirfs):
+ dirfs.fs.expand_path.return_value = [f"{PATH}/1file", f"{PATH}/2file"]
+ assert dirfs.expand_path(["1*", "2*"], *ARGS, **KWARGS) == ["1file",
"2file"]
+ dirfs.fs.expand_path.assert_called_once_with(
+ [f"{PATH}/1*", f"{PATH}/2*"], *ARGS, **KWARGS
+ )
+
+
+@pytest.mark.asyncio
+async def test_async_mkdir(adirfs):
+ await adirfs._mkdir("dir", *ARGS, **KWARGS)
+ adirfs.fs._mkdir.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)
+
+
+def test_mkdir(dirfs):
+ dirfs.mkdir("dir", *ARGS, **KWARGS)
+ dirfs.fs.mkdir.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)
+
+
+@pytest.mark.asyncio
+async def test_async_makedirs(adirfs):
+ await adirfs._makedirs("dir", *ARGS, **KWARGS)
+ adirfs.fs._makedirs.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)
+
+
+def test_makedirs(dirfs):
+ dirfs.makedirs("dir", *ARGS, **KWARGS)
+ dirfs.fs.makedirs.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)
+
+
+def test_rmdir(mocker, dirfs):
+ dirfs.fs.rmdir = mocker.Mock()
+ dirfs.rmdir("dir")
+ dirfs.fs.rmdir.assert_called_once_with(f"{PATH}/dir")
+
+
+def test_mv_file(mocker, dirfs):
+ dirfs.fs.mv_file = mocker.Mock()
+ dirfs.mv_file("one", "two", **KWARGS)
+ dirfs.fs.mv_file.assert_called_once_with(f"{PATH}/one", f"{PATH}/two",
**KWARGS)
+
+
+def test_touch(mocker, dirfs):
+ dirfs.fs.touch = mocker.Mock()
+ dirfs.touch("file", **KWARGS)
+ dirfs.fs.touch.assert_called_once_with(f"{PATH}/file", **KWARGS)
+
+
+def test_created(mocker, dirfs):
+ dirfs.fs.created = mocker.Mock(return_value="date")
+ assert dirfs.created("file") == "date"
+ dirfs.fs.created.assert_called_once_with(f"{PATH}/file")
+
+
+def test_modified(mocker, dirfs):
+ dirfs.fs.modified = mocker.Mock(return_value="date")
+ assert dirfs.modified("file") == "date"
+ dirfs.fs.modified.assert_called_once_with(f"{PATH}/file")
+
+
+def test_sign(mocker, dirfs):
+ dirfs.fs.sign = mocker.Mock(return_value="url")
+ assert dirfs.sign("file", *ARGS, **KWARGS) == "url"
+ dirfs.fs.sign.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
+
+
+def test_open(mocker, dirfs):
+ dirfs.fs.open = mocker.Mock()
+ assert dirfs.open("file", *ARGS, **KWARGS) == dirfs.fs.open.return_value
+ dirfs.fs.open.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_http.py
new/filesystem_spec-2022.02.0/fsspec/implementations/tests/test_http.py
--- old/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_http.py
2022-01-11 16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/implementations/tests/test_http.py
2022-02-22 18:44:54.000000000 +0100
@@ -516,7 +516,6 @@
loop.call_soon_threadsafe(loop.stop)
-@pytest.mark.skipif(sys.version_info < (3, 7), reason="no asyncio.run in py36")
def test_async_this_thread(server):
async def _():
fs = fsspec.filesystem("http", asynchronous=True)
@@ -565,7 +564,6 @@
@pytest.mark.parametrize("get_client", [get_aiohttp, get_proxy])
-@pytest.mark.skipif(sys.version_info < (3, 7), reason="no asyncio.run in <3.7")
def test_close(get_client):
fs = fsspec.filesystem("http", skip_instance_cache=True)
fs.close_session(None, asyncio.run(get_client()))
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_local.py
new/filesystem_spec-2022.02.0/fsspec/implementations/tests/test_local.py
--- old/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_local.py
2022-01-11 16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/implementations/tests/test_local.py
2022-02-22 18:44:54.000000000 +0100
@@ -182,9 +182,6 @@
@pytest.mark.parametrize("mode", ["rt", "rb"])
@pytest.mark.parametrize("fmt", list(compression.compr))
def test_compressions(fmt, mode, tmpdir):
- if fmt == "zip" and sys.version_info < (3, 6):
- pytest.xfail("zip compression requires python3.6 or higher")
-
tmpdir = str(tmpdir)
fn = os.path.join(tmpdir, ".tmp.getsize")
fs = LocalFileSystem()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/fsspec/tests/test_api.py
new/filesystem_spec-2022.02.0/fsspec/tests/test_api.py
--- old/filesystem_spec-2022.01.0/fsspec/tests/test_api.py 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/tests/test_api.py 2022-02-22
18:44:54.000000000 +0100
@@ -3,7 +3,6 @@
import contextlib
import os
import pickle
-import sys
import tempfile
import pytest
@@ -213,7 +212,6 @@
assert f.read().decode("utf-8") == f.name
-@pytest.mark.skipif(sys.version_info < (3, 7), reason="no seek in old zipfile")
def test_multilevel_chained_fs_zip_zip_file():
"""This test reproduces fsspec/filesystem_spec#334"""
import zipfile
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/fsspec/tests/test_async.py
new/filesystem_spec-2022.02.0/fsspec/tests/test_async.py
--- old/filesystem_spec-2022.01.0/fsspec/tests/test_async.py 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/tests/test_async.py 2022-02-22
18:44:54.000000000 +0100
@@ -1,7 +1,6 @@
import asyncio
import inspect
import os
-import sys
import time
import pytest
@@ -18,7 +17,6 @@
assert not inspect.iscoroutinefunction(inst.info)
-@pytest.mark.skipif(fsspec.asyn.PY36, reason="missing asyncio features o py36")
def test_interrupt():
loop = fsspec.asyn.get_loop()
@@ -46,32 +44,27 @@
dummy_func = fsspec.asyn.sync_wrapper(_dummy_async_func)
-@pytest.mark.skipif(sys.version_info < (3, 7), reason="no asyncio.run in <3.7")
def
test_sync_wrapper_timeout_on_less_than_expected_wait_time_not_finish_function():
test_obj = _DummyAsyncKlass()
with pytest.raises(fsspec.FSTimeoutError):
test_obj.dummy_func(timeout=0.1)
-@pytest.mark.skipif(sys.version_info < (3, 7), reason="no asyncio.run in <3.7")
def
test_sync_wrapper_timeout_on_more_than_expected_wait_time_will_finish_function():
test_obj = _DummyAsyncKlass()
assert test_obj.dummy_func(timeout=5)
-@pytest.mark.skipif(sys.version_info < (3, 7), reason="no asyncio.run in <3.7")
def test_sync_wrapper_timeout_none_will_wait_func_finished():
test_obj = _DummyAsyncKlass()
assert test_obj.dummy_func(timeout=None)
-@pytest.mark.skipif(sys.version_info < (3, 7), reason="no asyncio.run in <3.7")
def test_sync_wrapper_treat_timeout_0_as_none():
test_obj = _DummyAsyncKlass()
assert test_obj.dummy_func(timeout=0)
-@pytest.mark.skipif(sys.version_info < (3, 7), reason="no asyncio.run in <3.7")
def test_run_coros_in_chunks(monkeypatch):
total_running = 0
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/fsspec/tests/test_core.py
new/filesystem_spec-2022.02.0/fsspec/tests/test_core.py
--- old/filesystem_spec-2022.01.0/fsspec/tests/test_core.py 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/tests/test_core.py 2022-02-22
18:44:54.000000000 +0100
@@ -184,6 +184,8 @@
assert isinstance(files, OpenFiles)
assert isinstance(files[0], OpenFile)
assert len(files) == 2
+ assert isinstance(files[:1], OpenFiles)
+ assert len(files[:1]) == 1
with files as of:
assert len(of) == 2
assert not of[0].closed
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/fsspec/tests/test_file.py
new/filesystem_spec-2022.02.0/fsspec/tests/test_file.py
--- old/filesystem_spec-2022.01.0/fsspec/tests/test_file.py 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/tests/test_file.py 2022-02-22
18:44:54.000000000 +0100
@@ -1,6 +1,5 @@
"""Tests abstract buffered file API, using FTP implementation"""
import pickle
-import sys
import pytest
@@ -9,10 +8,6 @@
data = b"hello" * 10000
-@pytest.mark.xfail(
- sys.version_info < (3, 6),
- reason="py35 error, see
https://github.com/fsspec/filesystem_spec/issues/147";,
-)
def test_pickle(ftp_writable):
host, port, user, pw = ftp_writable
ftp = FTPFileSystem(host=host, port=port, username=user, password=pw)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/fsspec/tests/test_fuse.py
new/filesystem_spec-2022.02.0/fsspec/tests/test_fuse.py
--- old/filesystem_spec-2022.01.0/fsspec/tests/test_fuse.py 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/tests/test_fuse.py 2022-02-22
18:44:54.000000000 +0100
@@ -1,5 +1,4 @@
import os
-import signal
import subprocess
import time
from multiprocessing import Process
@@ -69,8 +68,11 @@
os.rmdir(fn + "/inner")
os.rmdir(fn)
finally:
- os.kill(fuse_process.pid, signal.SIGTERM)
- fuse_process.join()
+ fuse_process.terminate()
+ fuse_process.join(timeout=10)
+ if fuse_process.is_alive():
+ fuse_process.kill()
+ fuse_process.join()
def host_mount_local(source_dir, mount_dir, debug_log):
@@ -94,8 +96,11 @@
try:
yield (source_dir, mount_dir)
finally:
- os.kill(fuse_process.pid, signal.SIGTERM)
- fuse_process.join()
+ fuse_process.terminate()
+ fuse_process.join(timeout=10)
+ if fuse_process.is_alive():
+ fuse_process.kill()
+ fuse_process.join()
def test_mount(mount_local):
@@ -124,3 +129,18 @@
assert cp.stdout == b""
assert set(os.listdir(source_dir)) == set(["text", "new"])
assert open(mount_dir / "new").read() == "test"
+
+
+def test_seek_rw(mount_local):
+ source_dir, mount_dir = mount_local
+ fh = open(mount_dir / "text", "w")
+ fh.write("teST")
+ fh.seek(2)
+ fh.write("st")
+ fh.close()
+
+ fh = open(mount_dir / "text", "r")
+ assert fh.read() == "test"
+ fh.seek(2)
+ assert fh.read() == "st"
+ fh.close()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/fsspec/utils.py
new/filesystem_spec-2022.02.0/fsspec/utils.py
--- old/filesystem_spec-2022.01.0/fsspec/utils.py 2022-01-11
16:29:11.000000000 +0100
+++ new/filesystem_spec-2022.02.0/fsspec/utils.py 2022-02-22
18:44:54.000000000 +0100
@@ -10,7 +10,6 @@
from urllib.parse import urlsplit
DEFAULT_BLOCK_SIZE = 5 * 2 ** 20
-PY36 = sys.version_info < (3, 7)
def infer_storage_options(urlpath, inherit_storage_options=None):
@@ -301,8 +300,8 @@
Notes
-----
- Objects supporting the fspath protocol (Python 3.6+) are coerced
- according to its __fspath__ method.
+ Objects supporting the fspath protocol are coerced according to its
+ __fspath__ method.
For backwards compatibility with older Python version, pathlib.Path
objects are specially coerced.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/setup.py
new/filesystem_spec-2022.02.0/setup.py
--- old/filesystem_spec-2022.01.0/setup.py 2022-01-11 16:29:11.000000000
+0100
+++ new/filesystem_spec-2022.02.0/setup.py 2022-02-22 18:44:54.000000000
+0100
@@ -18,7 +18,6 @@
"Intended Audience :: Developers",
"License :: OSI Approved :: BSD License",
"Operating System :: OS Independent",
- "Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
@@ -32,7 +31,7 @@
license="BSD",
keywords="file",
packages=["fsspec", "fsspec.implementations"],
- python_requires=">=3.6",
+ python_requires=">=3.7",
install_requires=open("requirements.txt").read().strip().split("\n"),
extras_require={
"entrypoints": ["importlib_metadata ; python_version < '3.8' "],
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2022.01.0/tox.ini
new/filesystem_spec-2022.02.0/tox.ini
--- old/filesystem_spec-2022.01.0/tox.ini 2022-01-11 16:29:11.000000000
+0100
+++ new/filesystem_spec-2022.02.0/tox.ini 2022-02-22 18:44:54.000000000
+0100
@@ -1,6 +1,6 @@
# content of: tox.ini , put in same dir as setup.py
[tox]
-envlist = {py36,py37,py38,py39}
+envlist = {py37,py38,py39}
[core]
conda_channels=
@@ -25,8 +25,10 @@
pyftpdlib
cloudpickle
pytest
+ pytest-asyncio
pytest-benchmark
pytest-cov
+ pytest-mock
pytest-vcr
fusepy
tomli < 2
@@ -38,7 +40,7 @@
deps=
hadoop-test-cluster==0.1.0
smbprotocol
- py36,py37: importlib_metadata
+ py37: importlib_metadata
[testenv]
description=Run test suite against target versions.
@@ -62,7 +64,7 @@
{[core]conda_deps}
httpretty
aiobotocore
- "moto<2.0"
+ "moto<3.0"
flask
changedir=.tox/s3fs/tmp
whitelist_externals=
