Hello community,
here is the log from the commit of package python-s3fs for openSUSE:Factory
checked in at 2019-10-30 14:47:51
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-s3fs (Old)
and /work/SRC/openSUSE:Factory/.python-s3fs.new.2990 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-s3fs"
Wed Oct 30 14:47:51 2019 rev:4 rq:743964 version:0.3.5
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-s3fs/python-s3fs.changes 2019-08-19
23:02:08.311454091 +0200
+++ /work/SRC/openSUSE:Factory/.python-s3fs.new.2990/python-s3fs.changes
2019-10-30 14:47:58.202191945 +0100
@@ -1,0 +2,7 @@
+Tue Sep 24 11:00:09 UTC 2019 - Tomáš Chvátal <tchva...@suse.com>
+
+- Update to 0.3.5:
+ * Test expansion
+ * Minor bugfixes
+
+-------------------------------------------------------------------
Old:
----
s3fs-0.3.3.tar.gz
New:
----
s3fs-0.3.5.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-s3fs.spec ++++++
--- /var/tmp/diff_new_pack.JvEAYm/_old 2019-10-30 14:48:00.014193871 +0100
+++ /var/tmp/diff_new_pack.JvEAYm/_new 2019-10-30 14:48:00.022193880 +0100
@@ -19,17 +19,16 @@
%{?!python_module:%define python_module() python-%{**} python3-%{**}}
%define skip_python2 1
Name: python-s3fs
-Version: 0.3.3
+Version: 0.3.5
Release: 0
Summary: Python filesystem interface over S3
License: BSD-3-Clause
-Group: Development/Languages/Python
URL: https://github.com/dask/s3fs/
Source:
https://files.pythonhosted.org/packages/source/s/s3fs/s3fs-%{version}.tar.gz
BuildRequires: %{python_module boto3 >= 1.9.91}
BuildRequires: %{python_module botocore >= 1.12.91}
BuildRequires: %{python_module fsspec >= 0.2.2}
-BuildRequires: %{python_module moto >= 1.3.7}
+BuildRequires: %{python_module moto >= 1.3.12}
BuildRequires: %{python_module pytest >= 4.2.0}
BuildRequires: %{python_module setuptools}
BuildRequires: fdupes
++++++ s3fs-0.3.3.tar.gz -> s3fs-0.3.5.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/s3fs-0.3.3/PKG-INFO new/s3fs-0.3.5/PKG-INFO
--- old/s3fs-0.3.3/PKG-INFO 2019-08-08 15:06:16.000000000 +0200
+++ new/s3fs-0.3.5/PKG-INFO 2019-10-06 18:26:35.000000000 +0200
@@ -1,6 +1,6 @@
Metadata-Version: 1.2
Name: s3fs
-Version: 0.3.3
+Version: 0.3.5
Summary: Convenient Filesystem interface over S3
Home-page: http://github.com/dask/s3fs/
Maintainer: Martin Durant
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/s3fs-0.3.3/docs/source/index.rst
new/s3fs-0.3.5/docs/source/index.rst
--- old/s3fs-0.3.3/docs/source/index.rst 2019-08-04 22:54:14.000000000
+0200
+++ new/s3fs-0.3.5/docs/source/index.rst 2019-09-09 15:14:23.000000000
+0200
@@ -78,6 +78,13 @@
- no permissions/access-control (i.e., no chmod/chown methods)
+Logging
+-------
+
+The logger ``s3fs.core.logger`` provides information about the operations of
the
+file system. To see messages, set its level to DEBUG. You can also achieve
this via
+an environment variable ``S3FS_LOGGING_LEVEL=DEBUG``.
+
Credentials
-----------
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/s3fs-0.3.3/s3fs/_version.py
new/s3fs-0.3.5/s3fs/_version.py
--- old/s3fs-0.3.3/s3fs/_version.py 2019-08-08 15:06:16.000000000 +0200
+++ new/s3fs-0.3.5/s3fs/_version.py 2019-10-06 18:26:35.000000000 +0200
@@ -8,11 +8,11 @@
version_json = '''
{
- "date": "2019-08-08T09:02:10-0400",
+ "date": "2019-10-06T11:15:43-0400",
"dirty": false,
"error": null,
- "full-revisionid": "990ceebb5ba73030819ddd09d5696506f0f865d7",
- "version": "0.3.3"
+ "full-revisionid": "571a6463ac7aaaf1a6f80ee776e79e3b0d76a4f4",
+ "version": "0.3.5"
}
''' # END VERSION_JSON
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/s3fs-0.3.3/s3fs/core.py new/s3fs-0.3.5/s3fs/core.py
--- old/s3fs-0.3.3/s3fs/core.py 2019-08-08 14:57:46.000000000 +0200
+++ new/s3fs-0.3.5/s3fs/core.py 2019-10-06 17:15:47.000000000 +0200
@@ -1,7 +1,8 @@
# -*- coding: utf-8 -*-
-import errno
import logging
+import os
import socket
+import time
from hashlib import md5
from fsspec import AbstractFileSystem
@@ -13,7 +14,14 @@
from s3fs.errors import translate_boto_error
from s3fs.utils import ParamKwargsHelper
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('s3fs')
+handle = logging.StreamHandler()
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s '
+ '- %(message)s')
+handle.setFormatter(formatter)
+logger.addHandler(handle)
+if "S3FS_LOGGING_LEVEL" in os.environ:
+ logger.setLevel(os.environ["S3FS_LOGGING_LEVEL"])
logging.getLogger('boto3').setLevel(logging.WARNING)
logging.getLogger('botocore').setLevel(logging.WARNING)
@@ -100,12 +108,16 @@
client_kwargs : dict of parameters for the boto3 client
requester_pays : bool (False)
If RequesterPays buckets are supported.
- default_block_size: None, int
+ default_block_size: int (None)
If given, the default block size value used for ``open()``, if no
specific value is given at all time. The built-in default is 5MB.
default_fill_cache : Bool (True)
Whether to use cache filling with open by default. Refer to
``S3File.open``.
+ default_cache_type : string ('bytes')
+ If given, the default cache_type value used for ``open()``. Set to
"none"
+ if no caching is desired. See fsspec's documentation for other
available
+ cache_type values. Default cache_type is 'bytes'.
version_aware : bool (False)
Whether to support bucket versioning. If enable this will require the
user to have the necessary IAM permissions for dealing with versioned
@@ -135,7 +147,7 @@
def __init__(self, anon=False, key=None, secret=None, token=None,
use_ssl=True, client_kwargs=None, requester_pays=False,
default_block_size=None, default_fill_cache=True,
- version_aware=False, config_kwargs=None,
+ default_cache_type='bytes', version_aware=False,
config_kwargs=None,
s3_additional_kwargs=None, session=None, username=None,
password=None, **kwargs):
if key and username:
@@ -164,6 +176,7 @@
config_kwargs = {}
self.default_block_size = default_block_size or self.default_block_size
self.default_fill_cache = default_fill_cache
+ self.default_cache_type = default_cache_type
self.version_aware = version_aware
self.client_kwargs = client_kwargs
self.config_kwargs = config_kwargs
@@ -177,6 +190,9 @@
return self._kwargs_helper.filter_dict(s3_method.__name__, kwargs)
def _call_s3(self, method, *akwarglist, **kwargs):
+ kw2 = kwargs.copy()
+ kw2.pop('Body', None)
+ logger.debug("CALL: %s - %s - %s" % (method.__name__, akwarglist, kw2))
additional_kwargs = self._get_s3_method_kwargs(method, *akwarglist,
**kwargs)
return method(**additional_kwargs)
@@ -223,6 +239,7 @@
self.session = boto3.Session(self.key, self.secret, self.token,
**self.kwargs)
+ logger.debug("Setting up s3fs instance")
self.s3 = self.session.client('s3', config=conf, use_ssl=ssl,
**self.client_kwargs)
return self.s3
@@ -253,7 +270,7 @@
'token': cred['SessionToken'], 'anon': False}
def _open(self, path, mode='rb', block_size=None, acl='', version_id=None,
- fill_cache=None, cache_type='bytes', autocommit=True, **kwargs):
+ fill_cache=None, cache_type=None, autocommit=True, **kwargs):
""" Open a file for reading or writing
Parameters
@@ -280,7 +297,8 @@
The encoding to use if opening the file in text mode. The
platform's
default text encoding is used if not given.
cache_type : str
- "bytes", "mmap" or "none"
+ See fsspec's documentation for available cache_type values. Set to
"none"
+ if no caching is desired. If None, defaults to
``self.default_cache_type``.
kwargs: dict-like
Additional parameters used for s3 methods. Typically used for
ServerSideEncryption.
@@ -297,6 +315,9 @@
raise ValueError("version_id cannot be specified if the filesystem
"
"is not version aware")
+ if cache_type is None:
+ cache_type = self.default_cache_type
+
return S3File(self, path, mode, block_size=block_size, acl=acl,
version_id=version_id, fill_cache=fill_cache,
s3_additional_kwargs=kw, cache_type=cache_type,
@@ -310,6 +331,7 @@
prefix = prefix + '/' if prefix else ""
if path not in self.dircache or refresh:
try:
+ logger.debug("Get directory listing page for %s" % path)
pag = self.s3.get_paginator('list_objects_v2')
config = {}
if max_items is not None:
@@ -443,6 +465,17 @@
except FileNotFoundError:
return False
+ def touch(self, path, truncate=True, data=None, **kwargs):
+ """Create empty file or truncate"""
+ bucket, key = split_path(path)
+ if not truncate and self.exists(path):
+ raise ValueError("S3 does not support touching existent files")
+ try:
+ self._call_s3(self.s3.put_object, kwargs, Bucket=bucket, Key=key)
+ except ClientError as ex:
+ raise translate_boto_error(ex)
+ self.invalidate_cache(self._parent(path))
+
def info(self, path, version_id=None):
if path in ['/', '']:
return {'name': path, 'size': 0, 'type': 'directory'}
@@ -912,6 +945,8 @@
self.key = key
self.version_id = version_id
self.acl = acl
+ if self.acl and self.acl not in key_acls:
+ raise ValueError('ACL not in %s', key_acls)
self.mpu = None
self.parts = None
self.fill_cache = fill_cache
@@ -929,20 +964,20 @@
self.size = self.details['size']
elif self.fs.version_aware:
self.version_id = self.details.get('VersionId')
- # In this case we have not managed to get the VersionId out of
details and
- # we should invalidate the cache and perform a full
head_object since it
+ # In this case we have not managed to get the VersionId out of
details and
+ # we should invalidate the cache and perform a full
head_object since it
# has likely been partially populated by ls.
if self.version_id is None:
self.fs.invalidate_cache(self.path)
self.details = self.fs.info(self.path)
self.version_id = self.details.get('VersionId')
+ self.append_block = False
if 'a' in mode and s3.exists(path):
loc = s3.info(path)['size']
if loc < 5 * 2 ** 20:
# existing file too small for multi-upload: download
self.write(self.fs.cat(self.path))
- self.append_block = False
else:
self.append_block = True
self.loc = loc
@@ -952,12 +987,11 @@
**kwargs)
def _initiate_upload(self):
- if self.acl and self.acl not in key_acls:
- raise ValueError('ACL not in %s', key_acls)
+ if not self.append_block and self.tell() < self.blocksize:
+ # only happens when closing small file, use on-shot PUT
+ return
+ logger.debug("Initiate upload for %s" % self)
self.parts = []
- self.size = 0
- if self.blocksize < 5 * 2 ** 20:
- raise ValueError('Block size must be >=5MB')
try:
self.mpu = self._call_s3(
self.fs.s3.create_multipart_upload,
@@ -967,20 +1001,19 @@
except ParamValidationError as e:
raise ValueError('Initiating write to %r failed: %s' % (self.path,
e))
- if 'a' in self.mode and self.fs.exists(self.path):
- if self.append_block:
- # use existing data in key when appending,
- # and block is big enough
- out = self.fs._call_s3(
- self.fs.s3.upload_part_copy,
- self.s3_additional_kwargs,
- Bucket=self.bucket,
- Key=self.key,
- PartNumber=1,
- UploadId=self.mpu['UploadId'],
- CopySource=self.path)
- self.parts.append({'PartNumber': 1,
- 'ETag': out['CopyPartResult']['ETag']})
+ if self.append_block:
+ # use existing data in key when appending,
+ # and block is big enough
+ out = self.fs._call_s3(
+ self.fs.s3.upload_part_copy,
+ self.s3_additional_kwargs,
+ Bucket=self.bucket,
+ Key=self.key,
+ PartNumber=1,
+ UploadId=self.mpu['UploadId'],
+ CopySource=self.path)
+ self.parts.append({'PartNumber': 1,
+ 'ETag': out['CopyPartResult']['ETag']})
def metadata(self, refresh=False, **kwargs):
""" Return metadata of file.
@@ -1024,8 +1057,16 @@
def _upload_chunk(self, final=False):
bucket, key = split_path(self.path)
- self.buffer.seek(0)
- (data0, data1) = (None, self.buffer.read(self.blocksize))
+ logger.debug("Upload for %s, final=%s, loc=%s, buffer loc=%s" % (
+ self, final, self.loc, self.buffer.tell()
+ ))
+ if not self.append_block and final and self.tell() < self.blocksize:
+ # only happens when closing small file, use on-shot PUT
+ data1 = False
+ else:
+ self.buffer.seek(0)
+ (data0, data1) = (None, self.buffer.read(self.blocksize))
+
while data1:
(data0, data1) = (data1, self.buffer.read(self.blocksize))
data1_size = len(data1)
@@ -1041,6 +1082,7 @@
(data0, data1) = (remainder[:partition],
remainder[partition:])
part = len(self.parts) + 1
+ logger.debug("Upload chunk %s, %s" % (self, part))
for attempt in range(self.retries + 1):
try:
@@ -1054,6 +1096,7 @@
if attempt < self.retries:
logger.debug('Exception %r on S3 write, retrying', exc,
exc_info=True)
+ time.sleep(1.7**attempt * 0.1)
except Exception as exc:
raise IOError('Write failed: %r' % exc)
else:
@@ -1063,21 +1106,41 @@
if self.autocommit and final:
self.commit()
+ return not final
def commit(self):
- logger.debug("COMMIT")
- part_info = {'Parts': self.parts}
- write_result = self._call_s3(
- self.fs.s3.complete_multipart_upload,
- Bucket=self.bucket,
- Key=self.key,
- UploadId=self.mpu['UploadId'],
- MultipartUpload=part_info)
- if self.fs.version_aware:
- self.version_id = write_result.get('VersionId')
+ logger.debug("Commit %s" % self)
+ if self.tell() == 0:
+ if self.buffer is not None:
+ logger.debug("Empty file committed %s" % self)
+ self._abort_mpu()
+ self.fs.touch(self.path)
+ elif not self.parts:
+ if self.buffer is not None:
+ logger.debug("One-shot upload of %s" % self)
+ self.buffer.seek(0)
+ data = self.buffer.read()
+ self._call_s3(
+ self.fs.s3.put_object,
+ Key=self.key, Bucket=self.bucket, Body=data, **self.kwargs
+ )
+ else:
+ raise RuntimeError
+ else:
+ logger.debug("Complete multi-part upload for %s " % self)
+ part_info = {'Parts': self.parts}
+ write_result = self._call_s3(
+ self.fs.s3.complete_multipart_upload,
+ Bucket=self.bucket,
+ Key=self.key,
+ UploadId=self.mpu['UploadId'],
+ MultipartUpload=part_info)
+ if self.fs.version_aware:
+ self.version_id = write_result.get('VersionId')
# complex cache invalidation, since file's appearance can cause several
# directories
+ self.buffer = None
parts = self.path.split('/')
path = parts[0]
for p in parts[1:]:
@@ -1088,20 +1151,36 @@
path = path + '/' + p
def discard(self):
- if self.autocommit:
- raise ValueError("Cannot discard when autocommit is enabled")
- self._call_s3(
- self.fs.s3.abort_multipart_upload,
- Bucket=self.bucket,
- Key=self.key,
- UploadId=self.mpu['UploadId'],
- )
+ self._abort_mpu()
+ self.buffer = None # file becomes unusable
+
+ def _abort_mpu(self):
+ if self.mpu:
+ self._call_s3(
+ self.fs.s3.abort_multipart_upload,
+ Bucket=self.bucket,
+ Key=self.key,
+ UploadId=self.mpu['UploadId'],
+ )
+ self.mpu = None
def _fetch_range(client, bucket, key, version_id, start, end, max_attempts=10,
req_kw=None):
if req_kw is None:
req_kw = {}
+ if start == end:
+ # When these match, we would make a request with `range=start-end - 1`
+ # According to RFC2616, servers are supposed to ignore the Range
+ # field when it's invalid like this. S3 does ignore it, moto doesn't.
+ # To avoid differences in behavior under mocking, we just avoid
+ # making these requests. It's hoped that since we're being called
+ # from a caching object, this won't end up mattering.
+ logger.debug(
+ 'skip fetch for negative range - bucket=%s,key=%s,start=%d,end=%d',
+ bucket, key, start, end
+ )
+ return b''
logger.debug("Fetch: %s/%s, %s-%s", bucket, key, start, end)
for i in range(max_attempts):
try:
@@ -1116,10 +1195,12 @@
except S3_RETRYABLE_ERRORS as e:
logger.debug('Exception %r on S3 download, retrying', e,
exc_info=True)
+ time.sleep(1.7**i * 0.1)
continue
except ConnectionError as e:
logger.debug('ConnectionError %r on S3 download, retrying', e,
exc_info=True)
+ time.sleep(1.7**i * 0.1)
continue
except ClientError as e:
if e.response['Error'].get('Code', 'Unknown') in ['416',
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/s3fs-0.3.3/s3fs/tests/test_s3fs.py
new/s3fs-0.3.5/s3fs/tests/test_s3fs.py
--- old/s3fs-0.3.3/s3fs/tests/test_s3fs.py 2019-08-08 00:14:52.000000000
+0200
+++ new/s3fs-0.3.5/s3fs/tests/test_s3fs.py 2019-08-27 20:52:18.000000000
+0200
@@ -4,10 +4,10 @@
import json
from concurrent.futures import ProcessPoolExecutor
import io
-import re
import time
import pytest
from itertools import chain
+import fsspec.core
from s3fs.core import S3FileSystem
from s3fs.utils import seek_delimiter, ignoring, SSEParams
import moto
@@ -134,6 +134,21 @@
assert out == data
+@pytest.mark.parametrize('default_cache_type', ['none', 'bytes', 'mmap'])
+def test_default_cache_type(s3, default_cache_type):
+ data = b'a' * (10 * 2 ** 20)
+ s3 = S3FileSystem(anon=False, default_cache_type=default_cache_type)
+
+ with s3.open(a, 'wb') as f:
+ f.write(data)
+
+ with s3.open(a, 'rb') as f:
+ assert isinstance(f.cache, fsspec.core.caches[default_cache_type])
+ out = f.read(len(data))
+ assert len(data) == len(out)
+ assert out == data
+
+
def test_ssl_off():
s3 = S3FileSystem(use_ssl=False)
assert s3.s3.meta.endpoint_url.startswith('http://')
@@ -1211,6 +1226,7 @@
S3FileSystem.default_block_size = 5 * (1024 ** 2)
S3FileSystem.cachable = True
+
def test_passed_in_session_set_correctly(s3):
session = boto3.session.Session()
s3 = S3FileSystem(session=session)
@@ -1279,3 +1295,46 @@
# Cannot commit a file that was discarded
with pytest.raises(Exception):
fo.commit()
+
+
+def test_touch(s3):
+ # create
+ fn = test_bucket_name + "/touched"
+ assert not s3.exists(fn)
+ s3.touch(fn)
+ assert s3.exists(fn)
+ assert s3.size(fn) == 0
+
+ # truncates
+ with s3.open(fn, 'wb') as f:
+ f.write(b'data')
+ assert s3.size(fn) == 4
+ s3.touch(fn, truncate=True)
+ assert s3.size(fn) == 0
+
+ # exists error
+ with s3.open(fn, 'wb') as f:
+ f.write(b'data')
+ assert s3.size(fn) == 4
+ with pytest.raises(ValueError):
+ s3.touch(fn, truncate=False)
+ assert s3.size(fn) == 4
+
+
+def test_seek_reads(s3):
+ fn = test_bucket_name + "/myfile"
+ with s3.open(fn, 'wb') as f:
+ f.write(b'a' * 175_627_146)
+ with s3.open(fn, 'rb', blocksize=100) as f:
+ f.seek(175561610)
+ d1 = f.read(65536)
+
+ f.seek(4)
+ size = 17562198
+ d2 = f.read(size)
+ assert len(d2) == size
+
+ f.seek(17562288)
+ size = 17562187
+ d3 = f.read(size)
+ assert len(d3) == size
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/s3fs-0.3.3/s3fs.egg-info/PKG-INFO
new/s3fs-0.3.5/s3fs.egg-info/PKG-INFO
--- old/s3fs-0.3.3/s3fs.egg-info/PKG-INFO 2019-08-08 15:06:16.000000000
+0200
+++ new/s3fs-0.3.5/s3fs.egg-info/PKG-INFO 2019-10-06 18:26:35.000000000
+0200
@@ -1,6 +1,6 @@
Metadata-Version: 1.2
Name: s3fs
-Version: 0.3.3
+Version: 0.3.5
Summary: Convenient Filesystem interface over S3
Home-page: http://github.com/dask/s3fs/
Maintainer: Martin Durant
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/s3fs-0.3.3/setup.cfg new/s3fs-0.3.5/setup.cfg
--- old/s3fs-0.3.3/setup.cfg 2019-08-08 15:06:16.000000000 +0200
+++ new/s3fs-0.3.5/setup.cfg 2019-10-06 18:26:35.000000000 +0200
@@ -1,6 +1,3 @@
-[bdist_wheel]
-universal = 1
-
[metadata]
long_description = file: README.rst
