Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package python-fsspec for openSUSE:Factory
checked in at 2022-02-01 14:02:49
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-fsspec (Old)
and /work/SRC/openSUSE:Factory/.python-fsspec.new.1898 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-fsspec"
Tue Feb 1 14:02:49 2022 rev:17 rq:949718 version:2022.1.0
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-fsspec/python-fsspec.changes
2021-12-27 16:07:24.045700607 +0100
+++ /work/SRC/openSUSE:Factory/.python-fsspec.new.1898/python-fsspec.changes
2022-02-01 14:03:13.183964691 +0100
@@ -1,0 +2,16 @@
+Mon Jan 24 17:08:51 UTC 2022 - Ben Greiner <c...@bnavigator.de>
+
+- Update to 2022.1.0
+ * Fix blocks cache metadata (#746)
+ * Fix default SMB port (#853)
+ * Fix caching fixes (#856, 855)
+ * Fix explicit close for http files (#866)
+ * Fix put_file to continue when no bytes (#869, 870)
+ * temporary files location (#851, 871)
+ * async abstract methods (#858, 859, 860)
+ * md5 for FIPS (#872)
+ * remove deprecated pyarrow/distutils (#880, 881)
+- Update flavored requirements: python36 no longer exists,
+ python-distributed does not support python310 yet.
+
+-------------------------------------------------------------------
Old:
----
fsspec-2021.11.1.tar.gz
New:
----
fsspec-2022.01.0.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-fsspec.spec ++++++
--- /var/tmp/diff_new_pack.4vEhKh/_old 2022-02-01 14:03:13.739960880 +0100
+++ /var/tmp/diff_new_pack.4vEhKh/_new 2022-02-01 14:03:13.739960880 +0100
@@ -1,7 +1,7 @@
#
# spec file
#
-# Copyright (c) 2021 SUSE LLC
+# Copyright (c) 2022 SUSE LLC
#
# All modifications and additions to the file contributed by third parties
# remain the property of their copyright owners, unless otherwise agreed
@@ -26,9 +26,9 @@
%bcond_with test
%endif
%define skip_python2 1
-%define ghversion 2021.11.1
+%define ghversion 2022.01.0
Name: python-fsspec%{psuffix}
-Version: 2021.11.1
+Version: 2022.1.0
Release: 0
Summary: Filesystem specification package
License: BSD-3-Clause
@@ -61,8 +61,12 @@
%if %{with test}
BuildRequires: %{python_module aiohttp}
BuildRequires: %{python_module cloudpickle}
+BuildRequires: %{python_module distributed if %python-base < 3.10}
BuildRequires: %{python_module fusepy}
+BuildRequires: %{python_module gcsfs}
BuildRequires: %{python_module notebook}
+BuildRequires: %{python_module numpy}
+BuildRequires: %{python_module panel}
BuildRequires: %{python_module paramiko}
BuildRequires: %{python_module pyftpdlib}
BuildRequires: %{python_module pytest}
@@ -71,10 +75,6 @@
BuildRequires: %{python_module s3fs}
BuildRequires: %{python_module smbprotocol}
BuildRequires: %{python_module zstandard}
-BuildRequires: %{python_module distributed if (%python-base without
python36-base)}
-BuildRequires: %{python_module gcsfs if (%python-base without python36-base)}
-BuildRequires: %{python_module numpy if (%python-base without python36-base)}
-BuildRequires: %{python_module panel if (%python-base without python36-base)}
# cannot test git and http in the same installation (?)
# BuildRequires: %%{python_module pygit2}
# BuildRequires: git-core
@@ -86,8 +86,6 @@
%prep
%autosetup -p1 -n filesystem_spec-%{ghversion}
-# don't test nonexistent python36-numpy
-sed -i -e '/^import numpy as np/ d' -e '/^import pytest/ a np =
pytest.importorskip("numpy")' fsspec/tests/test_spec.py
%build
%python_build
++++++ fsspec-2021.11.1.tar.gz -> fsspec-2022.01.0.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2021.11.1/.gitignore
new/filesystem_spec-2022.01.0/.gitignore
--- old/filesystem_spec-2021.11.1/.gitignore 2021-11-26 22:20:48.000000000
+0100
+++ new/filesystem_spec-2022.01.0/.gitignore 2022-01-11 16:29:11.000000000
+0100
@@ -116,3 +116,5 @@
# docker artifacts
.docker
+
+build/
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2021.11.1/docs/source/changelog.rst
new/filesystem_spec-2022.01.0/docs/source/changelog.rst
--- old/filesystem_spec-2021.11.1/docs/source/changelog.rst 2021-11-26
22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/docs/source/changelog.rst 2022-01-11
16:29:11.000000000 +0100
@@ -1,6 +1,24 @@
Changelog
=========
+2022.01.0
+---------
+
+Fixes
+
+- blocks cache metadata (#746)
+- default SMB port (#853)
+- caching fixes (#856, 855)
+- explicit close for http files (#866)
+- put_file to continue when no bytes (#869, 870)
+
+Other
+
+- temporary files location (#851, 871)
+- async abstract methods (#858, 859, 860)
+- md5 for FIPS (#872)
+- remove deprecated pyarrow/distutils (#880, 881)
+
2021.11.1
---------
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2021.11.1/fsspec/_version.py
new/filesystem_spec-2022.01.0/fsspec/_version.py
--- old/filesystem_spec-2021.11.1/fsspec/_version.py 2021-11-26
22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/_version.py 2022-01-11
16:29:11.000000000 +0100
@@ -22,9 +22,9 @@
# setup.py/versioneer.py will grep for the variable names, so they must
# each be defined on a line of their own. _version.py will just call
# get_keywords().
- git_refnames = " (tag: 2021.11.1)"
- git_full = "1f3b6d81feb3927d368727012823292e1da7cd2d"
- git_date = "2021-11-26 16:20:48 -0500"
+ git_refnames = " (tag: 2022.01.0)"
+ git_full = "ece2fe15b1cac41167b31b8421ac9936aef29010"
+ git_date = "2022-01-11 10:29:11 -0500"
keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
return keywords
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2021.11.1/fsspec/asyn.py
new/filesystem_spec-2022.01.0/fsspec/asyn.py
--- old/filesystem_spec-2021.11.1/fsspec/asyn.py 2021-11-26
22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/asyn.py 2022-01-11
16:29:11.000000000 +0100
@@ -320,6 +320,9 @@
nofiles=True,
)
+ async def _cp_file(self, path1, path2, **kwargs):
+ raise NotImplementedError
+
async def _copy(
self,
path1,
@@ -435,6 +438,9 @@
batch_size = batch_size or self.batch_size
return await _run_coros_in_chunks(coros, batch_size=batch_size,
nofiles=True)
+ async def _put_file(self, lpath, rpath, **kwargs):
+ raise NotImplementedError
+
async def _put(
self,
lpath,
@@ -554,7 +560,7 @@
async def _info(self, path, **kwargs):
raise NotImplementedError
- async def _ls(self, path, **kwargs):
+ async def _ls(self, path, detail=True, **kwargs):
raise NotImplementedError
async def _walk(self, path, maxdepth=None, **kwargs):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2021.11.1/fsspec/exceptions.py
new/filesystem_spec-2022.01.0/fsspec/exceptions.py
--- old/filesystem_spec-2021.11.1/fsspec/exceptions.py 2021-11-26
22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/exceptions.py 2022-01-11
16:29:11.000000000 +0100
@@ -4,6 +4,15 @@
import asyncio
+class BlocksizeMismatchError(ValueError):
+ """
+ Raised when a cached file is opened with a different blocksize than it was
+ written with
+ """
+
+ ...
+
+
class FSTimeoutError(asyncio.TimeoutError):
"""
Raised when a fsspec function timed out occurs
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2021.11.1/fsspec/implementations/cached.py
new/filesystem_spec-2022.01.0/fsspec/implementations/cached.py
--- old/filesystem_spec-2021.11.1/fsspec/implementations/cached.py
2021-11-26 22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/implementations/cached.py
2022-01-11 16:29:11.000000000 +0100
@@ -11,6 +11,7 @@
from fsspec.callbacks import _DEFAULT_CALLBACK
from fsspec.compression import compr
from fsspec.core import BaseCache, MMapCache
+from fsspec.exceptions import BlocksizeMismatchError
from fsspec.spec import AbstractBufferedFile
from fsspec.utils import infer_compression
@@ -159,7 +160,13 @@
if c["blocks"] is True or cache[k]["blocks"] is True:
c["blocks"] = True
else:
- c["blocks"] =
set(c["blocks"]).union(cache[k]["blocks"])
+ # self.cached_files[*][*]["blocks"] must continue to
+ # point to the same set object so that updates
+ # performed by MMapCache are propagated back to
+ # self.cached_files.
+ blocks = cache[k]["blocks"]
+ blocks.update(c["blocks"])
+ c["blocks"] = blocks
c["time"] = max(c["time"], cache[k]["time"])
c["uid"] = cache[k]["uid"]
@@ -173,8 +180,8 @@
for c in cache.values():
if isinstance(c["blocks"], set):
c["blocks"] = list(c["blocks"])
- fn2 = tempfile.mktemp()
- with open(fn2, "wb") as f:
+ fd2, fn2 = tempfile.mkstemp()
+ with open(fd2, "wb") as f:
pickle.dump(cache, f)
self._mkcache()
move(fn2, fn)
@@ -229,9 +236,10 @@
raises PermissionError
"""
path = self._strip_protocol(path)
- _, fn = self._check_file(path)
- if fn is None:
+ details = self._check_file(path)
+ if not details:
return
+ _, fn = details
if fn.startswith(self.storage[-1]):
# is in in writable cache
os.remove(fn)
@@ -320,7 +328,7 @@
f = compr[comp](f, mode="rb")
if "blocksize" in detail:
if detail["blocksize"] != f.blocksize:
- raise ValueError(
+ raise BlocksizeMismatchError(
"Cached file must be reopened with same block"
"size as original (old: %i, new %i)"
"" % (detail["blocksize"], f.blocksize)
@@ -715,10 +723,13 @@
"""A temporary local file, which will be uploaded on commit"""
def __init__(self, fs, path, fn=None, mode="wb", autocommit=True, seek=0):
- fn = fn or tempfile.mktemp()
+ if fn:
+ self.fn = fn
+ self.fh = open(fn, mode)
+ else:
+ fd, self.fn = tempfile.mkstemp()
+ self.fh = open(fd, mode)
self.mode = mode
- self.fn = fn
- self.fh = open(fn, mode)
if seek:
self.fh.seek(seek)
self.path = path
@@ -752,6 +763,13 @@
def commit(self):
self.fs.put(self.fn, self.path)
+ @property
+ def name(self):
+ if isinstance(self.fh.name, str):
+ return self.fh.name # initialized by open()
+ else:
+ return self.fn # initialized by tempfile.mkstemp()
+
def __getattr__(self, item):
return getattr(self.fh, item)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2021.11.1/fsspec/implementations/ftp.py
new/filesystem_spec-2022.01.0/fsspec/implementations/ftp.py
--- old/filesystem_spec-2021.11.1/fsspec/implementations/ftp.py 2021-11-26
22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/implementations/ftp.py 2022-01-11
16:29:11.000000000 +0100
@@ -21,7 +21,7 @@
password=None,
acct=None,
block_size=None,
- tempdir="/tmp",
+ tempdir=None,
timeout=30,
**kwargs,
):
@@ -54,7 +54,7 @@
super(FTPFileSystem, self).__init__(**kwargs)
self.host = host
self.port = port
- self.tempdir = tempdir
+ self.tempdir = tempdir or "/tmp"
self.cred = username, password, acct
self.timeout = timeout
if block_size is not None:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2021.11.1/fsspec/implementations/http.py
new/filesystem_spec-2022.01.0/fsspec/implementations/http.py
--- old/filesystem_spec-2021.11.1/fsspec/implementations/http.py
2021-11-26 22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/implementations/http.py
2022-01-11 16:29:11.000000000 +0100
@@ -274,11 +274,11 @@
else:
callback.set_size(getattr(f, "size", None))
- chunk = f.read(64 * 1024)
+ chunk = f.read(chunk_size)
while chunk:
yield chunk
callback.relative_update(len(chunk))
- chunk = f.read(64 * 1024)
+ chunk = f.read(chunk_size)
kw = self.kwargs.copy()
kw.update(kwargs)
@@ -626,9 +626,6 @@
_fetch_range = sync_wrapper(async_fetch_range)
- def close(self):
- pass
-
def __reduce__(self):
return (
reopen,
@@ -690,6 +687,7 @@
def close(self):
asyncio.run_coroutine_threadsafe(self._close(), self.loop)
+ super().close()
def __reduce__(self):
return reopen, (self.fs, self.url, self.mode, self.blocksize,
self.cache.name)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2021.11.1/fsspec/implementations/reference.py
new/filesystem_spec-2022.01.0/fsspec/implementations/reference.py
--- old/filesystem_spec-2021.11.1/fsspec/implementations/reference.py
2021-11-26 22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/implementations/reference.py
2022-01-11 16:29:11.000000000 +0100
@@ -2,6 +2,7 @@
import io
import itertools
import logging
+import os
from functools import lru_cache
import fsspec.core
@@ -227,11 +228,15 @@
self.references[path] = value
async def _get_file(self, rpath, lpath, **kwargs):
+ if self.isdir(rpath):
+ return os.makedirs(lpath, exist_ok=True)
data = await self._cat_file(rpath)
with open(lpath, "wb") as f:
f.write(data)
def get_file(self, rpath, lpath, callback=_DEFAULT_CALLBACK, **kwargs):
+ if self.isdir(rpath):
+ return os.makedirs(lpath, exist_ok=True)
data = self.cat_file(rpath, **kwargs)
callback.lazy_call("set_size", len, data)
with open(lpath, "wb") as f:
@@ -241,7 +246,7 @@
def get(self, rpath, lpath, recursive=False, **kwargs):
if self.fs.async_impl:
return sync(self.loop, self._get, rpath, lpath, recursive,
**kwargs)
- return AbstractFileSystem.get(rpath, lpath, recursive=recursive,
**kwargs)
+ return AbstractFileSystem.get(self, rpath, lpath, recursive=recursive,
**kwargs)
def cat(self, path, recursive=False, **kwargs):
if self.fs.async_impl:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2021.11.1/fsspec/implementations/sftp.py
new/filesystem_spec-2022.01.0/fsspec/implementations/sftp.py
--- old/filesystem_spec-2021.11.1/fsspec/implementations/sftp.py
2021-11-26 22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/implementations/sftp.py
2022-01-11 16:29:11.000000000 +0100
@@ -41,7 +41,7 @@
if self._cached:
return
super(SFTPFileSystem, self).__init__(**ssh_kwargs)
- self.temppath = ssh_kwargs.pop("temppath", "/tmp")
+ self.temppath = ssh_kwargs.pop("temppath", "/tmp") # remote temp
directory
self.host = host
self.ssh_kwargs = ssh_kwargs
self._connect()
@@ -136,7 +136,7 @@
logger.debug("Opening file %s" % path)
if kwargs.get("autocommit", True) is False:
# writes to temporary file, move on commit
- path2 = "{}/{}".format(self.temppath, uuid.uuid4())
+ path2 = "/".join([self.temppath, str(uuid.uuid4())])
f = self.ftp.open(path2, mode, bufsize=block_size if block_size
else -1)
f.temppath = path2
f.targetpath = path
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2021.11.1/fsspec/implementations/smb.py
new/filesystem_spec-2022.01.0/fsspec/implementations/smb.py
--- old/filesystem_spec-2021.11.1/fsspec/implementations/smb.py 2021-11-26
22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/implementations/smb.py 2022-01-11
16:29:11.000000000 +0100
@@ -119,7 +119,7 @@
self.host,
username=self.username,
password=self.password,
- port=self.port,
+ port=445 if self.port is None else self.port,
encrypt=self.encrypt,
connection_timeout=self.timeout,
)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2021.11.1/fsspec/implementations/tests/test_cached.py
new/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_cached.py
--- old/filesystem_spec-2021.11.1/fsspec/implementations/tests/test_cached.py
2021-11-26 22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_cached.py
2022-01-11 16:29:11.000000000 +0100
@@ -7,6 +7,7 @@
import fsspec
from fsspec.compression import compr
+from fsspec.exceptions import BlocksizeMismatchError
from fsspec.implementations.cached import CachingFileSystem
from .test_ftp import FTPFileSystem
@@ -180,6 +181,7 @@
with pytest.raises(PermissionError):
fs.pop_from_cache(f1)
fs.pop_from_cache(f2)
+ fs.pop_from_cache(os.path.join(origin, "uncached-file"))
assert len(os.listdir(cache2)) == 1
assert not fs._check_file(f2)
assert fs._check_file(f1)
@@ -212,7 +214,7 @@
with fs.open("/out_block", block_size=20) as f:
assert f.read(1) == b"t"
- with pytest.raises(ValueError):
+ with pytest.raises(BlocksizeMismatchError):
fs.open("/out_block", block_size=30)
@@ -578,10 +580,12 @@
target_protocol="ftp",
target_options={"host": host, "port": port, "username": user,
"password": pw},
)
- with fs.open("/out_block") as f:
+ with fs.open("/out_block", block_size=1024) as f:
pass
- with fs.open("/out_block") as f:
+ with fs.open("/out_block", block_size=1024) as f:
assert f.read(1) == b"t"
+ # Regression test for
<https://github.com/fsspec/filesystem_spec/issues/845>
+ assert fs.cached_files[-1]["/out_block"]["blocks"] == {0}
@pytest.mark.parametrize("impl", ["filecache", "simplecache"])
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2021.11.1/fsspec/implementations/tests/test_ftp.py
new/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_ftp.py
--- old/filesystem_spec-2021.11.1/fsspec/implementations/tests/test_ftp.py
2021-11-26 22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_ftp.py
2022-01-11 16:29:11.000000000 +0100
@@ -119,7 +119,7 @@
assert not fs.exists(fn)
-def test_transaction_with_cache(ftp_writable):
+def test_transaction_with_cache(ftp_writable, tmpdir):
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host, port, user, pw)
fs.mkdir("/tmp")
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2021.11.1/fsspec/implementations/tests/test_http.py
new/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_http.py
--- old/filesystem_spec-2021.11.1/fsspec/implementations/tests/test_http.py
2021-11-26 22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_http.py
2022-01-11 16:29:11.000000000 +0100
@@ -358,6 +358,7 @@
else:
with pytest.raises(ValueError):
f.seek(5, 1)
+ assert f.closed
def test_mapper_url(server):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2021.11.1/fsspec/implementations/tests/test_local.py
new/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_local.py
--- old/filesystem_spec-2021.11.1/fsspec/implementations/tests/test_local.py
2021-11-26 22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_local.py
2022-01-11 16:29:11.000000000 +0100
@@ -9,7 +9,6 @@
import sys
import tempfile
from contextlib import contextmanager
-from distutils.version import LooseVersion
from pathlib import Path
from unittest.mock import patch
@@ -346,22 +345,6 @@
assert info2["mtime"] > info["mtime"]
-def test_get_pyarrow_filesystem():
- pa = pytest.importorskip("pyarrow")
-
- fs = LocalFileSystem()
- if LooseVersion(pa.__version__) < LooseVersion("2.0"):
- assert isinstance(fs, pa.filesystem.FileSystem)
- assert fs._get_pyarrow_filesystem() is fs
- else:
- assert not isinstance(fs, pa.filesystem.FileSystem)
-
- class UnknownFileSystem(object):
- pass
-
- assert not isinstance(UnknownFileSystem(), pa.filesystem.FileSystem)
-
-
def test_directories(tmpdir):
tmpdir = make_path_posix(str(tmpdir))
fs = LocalFileSystem()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2021.11.1/fsspec/implementations/tests/test_reference.py
new/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_reference.py
---
old/filesystem_spec-2021.11.1/fsspec/implementations/tests/test_reference.py
2021-11-26 22:20:48.000000000 +0100
+++
new/filesystem_spec-2022.01.0/fsspec/implementations/tests/test_reference.py
2022-01-11 16:29:11.000000000 +0100
@@ -3,6 +3,7 @@
import pytest
import fsspec
+from fsspec.implementations.local import LocalFileSystem
from fsspec.implementations.reference import _unmodel_hdf5
from .test_http import data, realfile, server # noqa: F401
@@ -210,3 +211,23 @@
pytest.importorskip("jinja2")
fs = fsspec.filesystem("reference", fo={"version": 1},
target_protocol="http")
assert fs.references == {}
+
+
+def test_get_sync(tmpdir):
+ localfs = LocalFileSystem()
+
+ real = tmpdir / "file"
+ real.write_binary(b"0123456789")
+
+ refs = {"a": b"data", "b": (str(real), 0, 5), "c/d": (str(real), 1, 6)}
+ fs = fsspec.filesystem("reference", fo=refs, fs=localfs)
+
+ fs.get("a", str(tmpdir / "a"))
+ assert (tmpdir / "a").read_binary() == b"data"
+ fs.get("b", str(tmpdir / "b"))
+ assert (tmpdir / "b").read_binary() == b"01234"
+ fs.get("c/d", str(tmpdir / "d"))
+ assert (tmpdir / "d").read_binary() == b"123456"
+ fs.get("c", str(tmpdir / "c"), recursive=True)
+ assert (tmpdir / "c").isdir()
+ assert (tmpdir / "c" / "d").read_binary() == b"123456"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/filesystem_spec-2021.11.1/fsspec/implementations/webhdfs.py
new/filesystem_spec-2022.01.0/fsspec/implementations/webhdfs.py
--- old/filesystem_spec-2021.11.1/fsspec/implementations/webhdfs.py
2021-11-26 22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/implementations/webhdfs.py
2022-01-11 16:29:11.000000000 +0100
@@ -1,8 +1,10 @@
# https://hadoop.apache.org/docs/r1.0.4/webhdfs.html
import logging
+import os
import secrets
import shutil
+import tempfile
import uuid
from contextlib import suppress
from urllib.parse import quote
@@ -27,7 +29,7 @@
spnego: when kerberos authentication is enabled, auth is negotiated by
requests_kerberos https://github.com/requests/requests-kerberos .
This establishes a session based on existing kinit login and/or
- specified principal/password; paraneters are passed with
``kerb_kwargs``
+ specified principal/password; parameters are passed with
``kerb_kwargs``
token: uses an existing Hadoop delegation token from another secured
service. Indeed, this client can also generate such tokens when
not insecure. Note that tokens expire, but can be renewed (by a
@@ -35,7 +37,7 @@
"""
- tempdir = "/tmp"
+ tempdir = str(tempfile.gettempdir())
protocol = "webhdfs", "webHDFS"
def __init__(
@@ -377,7 +379,7 @@
tempdir = kwargs.pop("tempdir")
if kwargs.pop("autocommit", False) is False:
self.target = self.path
- self.path = "/".join([tempdir, str(uuid.uuid4())])
+ self.path = os.path.join(tempdir, str(uuid.uuid4()))
def _upload_chunk(self, final=False):
"""Write one part of a multi-block file upload
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2021.11.1/fsspec/spec.py
new/filesystem_spec-2022.01.0/fsspec/spec.py
--- old/filesystem_spec-2021.11.1/fsspec/spec.py 2021-11-26
22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/spec.py 2022-01-11
16:29:11.000000000 +0100
@@ -4,7 +4,6 @@
import threading
import warnings
import weakref
-from distutils.version import LooseVersion
from errno import ESPIPE
from glob import has_magic
from hashlib import sha256
@@ -13,14 +12,7 @@
from .config import apply_config, conf
from .dircache import DirCache
from .transaction import Transaction
-from .utils import (
- _unstrip_protocol,
- get_package_version_without_import,
- other_paths,
- read_block,
- stringify_path,
- tokenize,
-)
+from .utils import _unstrip_protocol, other_paths, read_block, stringify_path,
tokenize
logger = logging.getLogger("fsspec")
@@ -88,20 +80,7 @@
return obj
-pa_version = get_package_version_without_import("pyarrow")
-if pa_version and LooseVersion(pa_version) < LooseVersion("2.0"):
- try:
- import pyarrow as pa
-
- up = pa.filesystem.DaskFileSystem
- except ImportError: # pragma: no cover
- # pyarrow exists but doesn't import for some reason
- up = object
-else: # pragma: no cover
- up = object
-
-
-class AbstractFileSystem(up, metaclass=_Cached):
+class AbstractFileSystem(metaclass=_Cached):
"""
An abstract super-class for pythonic file-systems
@@ -803,13 +782,13 @@
return None
with open(lpath, "rb") as f1:
- callback.set_size(f1.seek(0, 2))
+ size = f1.seek(0, 2)
+ callback.set_size(size)
f1.seek(0)
self.mkdirs(self._parent(os.fspath(rpath)), exist_ok=True)
with self.open(rpath, "wb", **kwargs) as f2:
- data = True
- while data:
+ while f1.tell() < size:
data = f1.read(self.blocksize)
segment_len = f2.write(data)
callback.relative_update(segment_len)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2021.11.1/fsspec/tests/test_spec.py
new/filesystem_spec-2022.01.0/fsspec/tests/test_spec.py
--- old/filesystem_spec-2021.11.1/fsspec/tests/test_spec.py 2021-11-26
22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/tests/test_spec.py 2022-01-11
16:29:11.000000000 +0100
@@ -495,7 +495,9 @@
source.write_text("x" * 100, "utf-8")
fs.put_file(source, file, callback=callback)
- assert callback.events == imitate_transfer(size, 10)
+
+ # -1 here since put_file no longer has final zero-size put
+ assert callback.events == imitate_transfer(size, 10)[:-1]
callback.events.clear()
fs.get_file(file, destination, callback=callback)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2021.11.1/fsspec/utils.py
new/filesystem_spec-2022.01.0/fsspec/utils.py
--- old/filesystem_spec-2021.11.1/fsspec/utils.py 2021-11-26
22:20:48.000000000 +0100
+++ new/filesystem_spec-2022.01.0/fsspec/utils.py 2022-01-11
16:29:11.000000000 +0100
@@ -281,7 +281,11 @@
"""
if kwargs:
args += (kwargs,)
- return md5(str(args).encode()).hexdigest()
+ try:
+ return md5(str(args).encode()).hexdigest()
+ except ValueError:
+ # FIPS systems: https://github.com/fsspec/filesystem_spec/issues/380
+ return md5(str(args).encode(), usedforsecurity=False).hexdigest()
def stringify_path(filepath):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/filesystem_spec-2021.11.1/tox.ini
new/filesystem_spec-2022.01.0/tox.ini
--- old/filesystem_spec-2021.11.1/tox.ini 2021-11-26 22:20:48.000000000
+0100
+++ new/filesystem_spec-2022.01.0/tox.ini 2022-01-11 16:29:11.000000000
+0100
@@ -29,6 +29,7 @@
pytest-cov
pytest-vcr
fusepy
+ tomli < 2
msgpack-python
python-libarchive-c
numpy
