This is an automated email from the ASF dual-hosted git repository.
clewolff pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/libcloud.git
The following commit(s) were added to refs/heads/trunk by this push:
new f176f5b Add `prefix` to top-level storage API (#1397)
f176f5b is described below
commit f176f5b8b2b50162f68d4a1f1cb456b8d09e70a2
Author: Clemens Wolff <[email protected]>
AuthorDate: Thu Jan 9 11:09:03 2020 -0500
Add `prefix` to top-level storage API (#1397)
* Ensure ex_prefix is passed to all methods
* Implement ex_prefix for local storage
* Add changelog
* Add prefix to storage API and deprecated ex_pefix
* Remove redundant list_container_objects overrides
* Ensure all drivers implement prefix filtering
* Extract prefix argument normalization method
* Update changelog
* Update tests to use new prefix argument
* Make test work on Windows
* Document that filtering is performed client-side
* Fix lint
* Add upgrade note
Co-authored-by: Tomaz Muraus <[email protected]>
---
CHANGES.rst | 27 +++++++++-------
demos/example_aliyun_oss.py | 2 +-
docs/upgrade_notes.rst | 5 +++
libcloud/storage/base.py | 52 ++++++++++++++++++++++++++-----
libcloud/storage/drivers/atmos.py | 32 ++++++++++++++++---
libcloud/storage/drivers/azure_blobs.py | 25 ++++-----------
libcloud/storage/drivers/backblaze_b2.py | 24 ++++++++++++--
libcloud/storage/drivers/cloudfiles.py | 32 +++++++------------
libcloud/storage/drivers/dummy.py | 12 +++----
libcloud/storage/drivers/local.py | 13 ++++++--
libcloud/storage/drivers/oss.py | 32 +++++++------------
libcloud/storage/drivers/s3.py | 29 ++++++-----------
libcloud/test/storage/test_azure_blobs.py | 2 +-
libcloud/test/storage/test_cloudfiles.py | 4 +--
libcloud/test/storage/test_dummy.py | 4 +--
libcloud/test/storage/test_local.py | 5 +++
libcloud/test/storage/test_oss.py | 2 +-
libcloud/test/storage/test_s3.py | 2 +-
18 files changed, 185 insertions(+), 119 deletions(-)
diff --git a/CHANGES.rst b/CHANGES.rst
index 2e66951..aa58584 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -37,7 +37,6 @@ Compute
(GITHUB-1346)
[Tomaz Muraus]
-
- [AWS EC2] Add support for creating spot instances by utilizing new
``ex_spot``
and optionally also ``ex_spot_max_price`` keyword argument in the
``create_node`` method.
@@ -51,16 +50,6 @@ Container
(GITHUB-1395)
[Alexandros Giavaras - @pockerman]
-Compute
-~~~~~~~
-
-- [GCE] Fix ``ex_list_instancegroups`` method so it doesn't throw if ``zone``
- attribute is not present in the response.
-
- Reported by Kartik Subbarao (@kartiksubbarao)
- (GITHUB-1346)
- [Tomaz Muraus]
-
Storage
~~~~~~~
@@ -93,6 +82,22 @@ Storage
(GITHUB-1400)
[Clemens Wolff - @c-w]
+- [Common] Add ``prefix`` argument to ``iterate_container_objects`` and
+ ``list_container_objects`` to support object-list filtering in all
+ StorageDriver implementations.
+
+ A lot of the existing storage drivers already implemented the filtering
+ functionality via the ``ex_prefix`` extension argument so it was decided
+ to promote the argument to be part of the standard Libcloud storage API.
+ For any storage driver that doesn't natively implement filtering the results
+ list, a fall-back was implemented which filters the full object stream on
+ the client side.
+
+ For backward compatibility reasons, the ``ex_prefix`` argument will still
+ be respected until a next major release.
+ (GITHUB-1397)
+ [Clemens Wolff - @c-w]
+
Changes in Apache Libcloud v2.8.0
---------------------------------
diff --git a/demos/example_aliyun_oss.py b/demos/example_aliyun_oss.py
index 6330a28..9ef504e 100644
--- a/demos/example_aliyun_oss.py
+++ b/demos/example_aliyun_oss.py
@@ -36,7 +36,7 @@ objects = c1.list_objects()
count = len(objects)
print('Has %d objects' % count)
-objects = oss.list_container_objects(c1, ex_prefix='en')
+objects = oss.list_container_objects(c1, prefix='en')
print('Has %d objects with prefix "en"' % len(objects))
for each in objects:
print(each)
diff --git a/docs/upgrade_notes.rst b/docs/upgrade_notes.rst
index f7b9a65..b3eb56c 100644
--- a/docs/upgrade_notes.rst
+++ b/docs/upgrade_notes.rst
@@ -22,6 +22,11 @@ Libcloud 3.0.0
have been removed from the ``upload_object`` and ``upload_object_via_stream``
methods.
+* The ``ex_prefix`` keyword argument in the ``iterate_container_objects``
+ and ``list_container_objects`` methods in all storage drivers has been
+ renamed to ``prefix`` to indicate the promotion of the argument to the
+ standard storage driver API.
+
Libcloud 2.8.0
--------------
diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py
index 7d95976..183bb3b 100644
--- a/libcloud/storage/base.py
+++ b/libcloud/storage/base.py
@@ -24,6 +24,7 @@ from typing import Type
import os.path # pylint: disable-msg=W0404
import hashlib
+import warnings
from os.path import join as pjoin
from libcloud.utils.py3 import httplib
@@ -133,11 +134,15 @@ class Container(object):
self.extra = extra or {}
self.driver = driver
- def iterate_objects(self):
- return self.driver.iterate_container_objects(container=self)
+ def iterate_objects(self, prefix=None, ex_prefix=None):
+ return self.driver.iterate_container_objects(container=self,
+ prefix=prefix,
+ ex_prefix=ex_prefix)
- def list_objects(self):
- return self.driver.list_container_objects(container=self)
+ def list_objects(self, prefix=None, ex_prefix=None):
+ return self.driver.list_container_objects(container=self,
+ prefix=prefix,
+ ex_prefix=ex_prefix)
def get_cdn_url(self):
return self.driver.get_container_cdn_url(container=self)
@@ -211,33 +216,64 @@ class StorageDriver(BaseDriver):
"""
return list(self.iterate_containers())
- def iterate_container_objects(self, container):
+ def iterate_container_objects(self, container, prefix=None,
+ ex_prefix=None):
"""
Return a generator of objects for the given container.
:param container: Container instance
:type container: :class:`Container`
+ :param prefix: Filter objects starting with a prefix.
+ :type prefix: ``str``
+
+ :param ex_prefix: (Deprecated.) Filter objects starting with a prefix.
+ :type ex_prefix: ``str``
+
:return: A generator of Object instances.
:rtype: ``generator`` of :class:`Object`
"""
raise NotImplementedError(
'iterate_container_objects not implemented for this driver')
- def list_container_objects(self, container, ex_prefix=None):
+ def list_container_objects(self, container, prefix=None, ex_prefix=None):
"""
Return a list of objects for the given container.
:param container: Container instance.
:type container: :class:`Container`
- :param ex_prefix: Filter objects starting with a prefix.
+ :param prefix: Filter objects starting with a prefix.
+ :type prefix: ``str``
+
+ :param ex_prefix: (Deprecated.) Filter objects starting with a prefix.
:type ex_prefix: ``str``
:return: A list of Object instances.
:rtype: ``list`` of :class:`Object`
"""
- return list(self.iterate_container_objects(container))
+ return list(self.iterate_container_objects(container,
+ prefix=prefix,
+ ex_prefix=ex_prefix))
+
+ def _normalize_prefix_argument(self, prefix, ex_prefix):
+ if ex_prefix:
+ warnings.warn('The ``ex_prefix`` argument is deprecated - '
+ 'please update code to use ``prefix``',
+ DeprecationWarning)
+ return ex_prefix
+
+ return prefix
+
+ def _filter_listed_container_objects(self, objects, prefix):
+ if prefix is not None:
+ warnings.warn('Driver %s does not implement native object '
+ 'filtering; falling back to filtering the full '
+ 'object stream.' % self.__class__.__name__)
+
+ for obj in objects:
+ if prefix is None or obj.name.startswith(prefix):
+ yield obj
def get_container(self, container_name):
"""
diff --git a/libcloud/storage/drivers/atmos.py
b/libcloud/storage/drivers/atmos.py
index 8d2ab01..d232547 100644
--- a/libcloud/storage/drivers/atmos.py
+++ b/libcloud/storage/drivers/atmos.py
@@ -446,11 +446,35 @@ class AtmosDriver(StorageDriver):
meta = meta.split(', ')
return dict([x.split('=', 1) for x in meta])
- def iterate_container_objects(self, container):
+ def _entries_to_objects(self, container, entries):
+ for entry in entries:
+ metadata = {'object_id': entry['id']}
+ yield Object(entry['name'], 0, '', {}, metadata, container, self)
+
+ def iterate_container_objects(self, container, prefix=None,
+ ex_prefix=None):
+ """
+ Return a generator of objects for the given container.
+
+ :param container: Container instance
+ :type container: :class:`Container`
+
+ :param prefix: Filter objects starting with a prefix.
+ Filtering is performed client-side.
+ :type prefix: ``str``
+
+ :param ex_prefix: (Deprecated.) Filter objects starting with a prefix.
+ Filtering is performed client-side.
+ :type ex_prefix: ``str``
+
+ :return: A generator of Object instances.
+ :rtype: ``generator`` of :class:`Object`
+ """
+ prefix = self._normalize_prefix_argument(prefix, ex_prefix)
+
headers = {'x-emc-include-meta': '1'}
path = self._namespace_path(container.name) + '/'
result = self.connection.request(path, headers=headers)
entries = self._list_objects(result.object, object_type='regular')
- for entry in entries:
- metadata = {'object_id': entry['id']}
- yield Object(entry['name'], 0, '', {}, metadata, container, self)
+ objects = self._entries_to_objects(container, entries)
+ return self._filter_listed_container_objects(objects, prefix)
diff --git a/libcloud/storage/drivers/azure_blobs.py
b/libcloud/storage/drivers/azure_blobs.py
index ec9074d..3a2e2a3 100644
--- a/libcloud/storage/drivers/azure_blobs.py
+++ b/libcloud/storage/drivers/azure_blobs.py
@@ -410,17 +410,20 @@ class AzureBlobsStorageDriver(StorageDriver):
if not params['marker']:
break
- def iterate_container_objects(self, container, ex_prefix=None):
+ def iterate_container_objects(self, container, prefix=None,
+ ex_prefix=None):
"""
@inherits: :class:`StorageDriver.iterate_container_objects`
"""
+ prefix = self._normalize_prefix_argument(prefix, ex_prefix)
+
params = {'restype': 'container',
'comp': 'list',
'maxresults': RESPONSES_PER_REQUEST,
'include': 'metadata'}
- if ex_prefix:
- params['prefix'] = ex_prefix
+ if prefix:
+ params['prefix'] = prefix
container_path = self._get_container_path(container)
@@ -448,22 +451,6 @@ class AzureBlobsStorageDriver(StorageDriver):
if not params['marker']:
break
- def list_container_objects(self, container, ex_prefix=None):
- """
- Return a list of objects for the given container.
-
- :param container: Container instance.
- :type container: :class:`Container`
-
- :param ex_prefix: Only return objects starting with ex_prefix
- :type ex_prefix: ``str``
-
- :return: A list of Object instances.
- :rtype: ``list`` of :class:`Object`
- """
- return list(self.iterate_container_objects(container,
- ex_prefix=ex_prefix))
-
def get_container(self, container_name):
"""
@inherits: :class:`StorageDriver.get_container`
diff --git a/libcloud/storage/drivers/backblaze_b2.py
b/libcloud/storage/drivers/backblaze_b2.py
index 6a93bdf..ba04bf5 100644
--- a/libcloud/storage/drivers/backblaze_b2.py
+++ b/libcloud/storage/drivers/backblaze_b2.py
@@ -251,14 +251,34 @@ class BackblazeB2StorageDriver(StorageDriver):
containers = self._to_containers(data=resp.object)
return containers
- def iterate_container_objects(self, container):
+ def iterate_container_objects(self, container, prefix=None,
+ ex_prefix=None):
+ """
+ Return a generator of objects for the given container.
+
+ :param container: Container instance
+ :type container: :class:`Container`
+
+ :param prefix: Filter objects starting with a prefix.
+ Filtering is performed client-side.
+ :type prefix: ``str``
+
+ :param ex_prefix: (Deprecated.) Filter objects starting with a prefix.
+ Filtering is performed client-side.
+ :type ex_prefix: ``str``
+
+ :return: A generator of Object instances.
+ :rtype: ``generator`` of :class:`Object`
+ """
+ prefix = self._normalize_prefix_argument(prefix, ex_prefix)
+
# TODO: Support pagination
params = {'bucketId': container.extra['id']}
resp = self.connection.request(action='b2_list_file_names',
method='GET',
params=params)
objects = self._to_objects(data=resp.object, container=container)
- return objects
+ return self._filter_listed_container_objects(objects, prefix)
def get_container(self, container_name):
containers = self.iterate_containers()
diff --git a/libcloud/storage/drivers/cloudfiles.py
b/libcloud/storage/drivers/cloudfiles.py
index dc58853..bdc41e0 100644
--- a/libcloud/storage/drivers/cloudfiles.py
+++ b/libcloud/storage/drivers/cloudfiles.py
@@ -695,38 +695,30 @@ class CloudFilesStorageDriver(StorageDriver,
OpenStackDriverMixin):
return obj
- def list_container_objects(self, container, ex_prefix=None):
- """
- Return a list of objects for the given container.
-
- :param container: Container instance.
- :type container: :class:`Container`
-
- :param ex_prefix: Only get objects with names starting with ex_prefix
- :type ex_prefix: ``str``
-
- :return: A list of Object instances.
- :rtype: ``list`` of :class:`Object`
- """
- return list(self.iterate_container_objects(container,
- ex_prefix=ex_prefix))
-
- def iterate_container_objects(self, container, ex_prefix=None):
+ def iterate_container_objects(self, container, prefix=None,
+ ex_prefix=None):
"""
Return a generator of objects for the given container.
:param container: Container instance
:type container: :class:`Container`
- :param ex_prefix: Only get objects with names starting with ex_prefix
+ :param prefix: Only get objects with names starting with prefix
+ :type prefix: ``str``
+
+ :param ex_prefix: (Deprecated.) Only get objects with names starting
+ with ex_prefix
:type ex_prefix: ``str``
:return: A generator of Object instances.
:rtype: ``generator`` of :class:`Object`
"""
+ prefix = self._normalize_prefix_argument(prefix, ex_prefix)
+
params = {}
- if ex_prefix:
- params['prefix'] = ex_prefix
+
+ if prefix:
+ params['prefix'] = prefix
while True:
container_name_encoded = \
diff --git a/libcloud/storage/drivers/dummy.py
b/libcloud/storage/drivers/dummy.py
index cb3432f..647f3b8 100644
--- a/libcloud/storage/drivers/dummy.py
+++ b/libcloud/storage/drivers/dummy.py
@@ -178,13 +178,13 @@ class DummyStorageDriver(StorageDriver):
for container in list(self._containers.values()):
yield container['container']
- def list_container_objects(self, container, ex_prefix=None):
- container = self.get_container(container.name)
+ def iterate_container_objects(self, container, prefix=None,
+ ex_prefix=None):
+ prefix = self._normalize_prefix_argument(prefix, ex_prefix)
- objects = list(self._containers[container.name]['objects'].values())
- if ex_prefix is not None:
- objects = [o for o in objects if o.name.startswith(ex_prefix)]
- return objects
+ container = self.get_container(container.name)
+ objects = self._containers[container.name]['objects'].values()
+ return self._filter_listed_container_objects(objects, prefix)
def get_container(self, container_name):
"""
diff --git a/libcloud/storage/drivers/local.py
b/libcloud/storage/drivers/local.py
index f308e15..0c90408 100644
--- a/libcloud/storage/drivers/local.py
+++ b/libcloud/storage/drivers/local.py
@@ -219,18 +219,27 @@ class LocalStorageDriver(StorageDriver):
object_name = relpath(full_path, start=cpath)
yield self._make_object(container, object_name)
- def iterate_container_objects(self, container):
+ def iterate_container_objects(self, container, prefix=None,
+ ex_prefix=None):
"""
Returns a generator of objects for the given container.
:param container: Container instance
:type container: :class:`Container`
+ :param prefix: Filter objects starting with a prefix.
+ :type prefix: ``str``
+
+ :param ex_prefix: (Deprecated.) Filter objects starting with a prefix.
+ :type ex_prefix: ``str``
+
:return: A generator of Object instances.
:rtype: ``generator`` of :class:`Object`
"""
+ prefix = self._normalize_prefix_argument(prefix, ex_prefix)
- return self._get_objects(container)
+ objects = self._get_objects(container)
+ return self._filter_listed_container_objects(objects, prefix)
def get_container(self, container_name):
"""
diff --git a/libcloud/storage/drivers/oss.py b/libcloud/storage/drivers/oss.py
index c1d67fc..9e75e1b 100644
--- a/libcloud/storage/drivers/oss.py
+++ b/libcloud/storage/drivers/oss.py
@@ -279,38 +279,30 @@ class OSSStorageDriver(StorageDriver):
raise LibcloudError('Unexpected status code: %s' % (response.status),
driver=self)
- def list_container_objects(self, container, ex_prefix=None):
- """
- Return a list of objects for the given container.
-
- :param container: Container instance.
- :type container: :class:`Container`
-
- :keyword ex_prefix: Only return objects starting with ex_prefix
- :type ex_prefix: ``str``
-
- :return: A list of Object instances.
- :rtype: ``list`` of :class:`Object`
- """
- return list(self.iterate_container_objects(container,
- ex_prefix=ex_prefix))
-
- def iterate_container_objects(self, container, ex_prefix=None):
+ def iterate_container_objects(self, container, prefix=None,
+ ex_prefix=None):
"""
Return a generator of objects for the given container.
:param container: Container instance
:type container: :class:`Container`
- :keyword ex_prefix: Only return objects starting with ex_prefix
+ :keyword prefix: Only return objects starting with prefix
+ :type prefix: ``str``
+
+ :keyword ex_prefix: (Deprecated.) Only return objects starting with
+ ex_prefix
:type ex_prefix: ``str``
:return: A generator of Object instances.
:rtype: ``generator`` of :class:`Object`
"""
+ prefix = self._normalize_prefix_argument(prefix, ex_prefix)
+
params = {}
- if ex_prefix:
- params['prefix'] = ex_prefix
+
+ if prefix:
+ params['prefix'] = prefix
last_key = None
exhausted = False
diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py
index 33f930b..d70e9ed 100644
--- a/libcloud/storage/drivers/s3.py
+++ b/libcloud/storage/drivers/s3.py
@@ -286,38 +286,29 @@ class BaseS3StorageDriver(StorageDriver):
raise LibcloudError('Unexpected status code: %s' % (response.status),
driver=self)
- def list_container_objects(self, container, ex_prefix=None):
- """
- Return a list of objects for the given container.
-
- :param container: Container instance.
- :type container: :class:`Container`
-
- :param ex_prefix: Only return objects starting with ex_prefix
- :type ex_prefix: ``str``
-
- :return: A list of Object instances.
- :rtype: ``list`` of :class:`Object`
- """
- return list(self.iterate_container_objects(container,
- ex_prefix=ex_prefix))
-
- def iterate_container_objects(self, container, ex_prefix=None):
+ def iterate_container_objects(self, container, prefix=None,
+ ex_prefix=None):
"""
Return a generator of objects for the given container.
:param container: Container instance
:type container: :class:`Container`
+ :param prefix: Only return objects starting with prefix
+ :type prefix: ``str``
+
:param ex_prefix: Only return objects starting with ex_prefix
:type ex_prefix: ``str``
:return: A generator of Object instances.
:rtype: ``generator`` of :class:`Object`
"""
+ prefix = self._normalize_prefix_argument(prefix, ex_prefix)
+
params = {}
- if ex_prefix:
- params['prefix'] = ex_prefix
+
+ if prefix:
+ params['prefix'] = prefix
last_key = None
exhausted = False
diff --git a/libcloud/test/storage/test_azure_blobs.py
b/libcloud/test/storage/test_azure_blobs.py
index aca39a2..389d581 100644
--- a/libcloud/test/storage/test_azure_blobs.py
+++ b/libcloud/test/storage/test_azure_blobs.py
@@ -465,7 +465,7 @@ class AzureBlobsTests(unittest.TestCase):
container = Container(name='test_container', extra={},
driver=self.driver)
objects = self.driver.list_container_objects(container=container,
- ex_prefix='test_prefix')
+ prefix='test_prefix')
self.assertEqual(len(objects), 4)
obj = objects[1]
diff --git a/libcloud/test/storage/test_cloudfiles.py
b/libcloud/test/storage/test_cloudfiles.py
index 8e180fc..3fc83c0 100644
--- a/libcloud/test/storage/test_cloudfiles.py
+++ b/libcloud/test/storage/test_cloudfiles.py
@@ -181,12 +181,12 @@ class CloudFilesTests(unittest.TestCase):
container = Container(
name='test_container', extra={}, driver=self.driver)
objects = self.driver.list_container_objects(container=container,
- ex_prefix='test_prefix1')
+ prefix='test_prefix1')
self.assertEqual(len(objects), 0)
CloudFilesMockHttp.type = None
objects = self.driver.list_container_objects(container=container,
- ex_prefix='test_prefix2')
+ prefix='test_prefix2')
self.assertEqual(len(objects), 4)
obj = [o for o in objects if o.name == 'foo test 1'][0]
diff --git a/libcloud/test/storage/test_dummy.py
b/libcloud/test/storage/test_dummy.py
index c176c7e..9de1ba6 100644
--- a/libcloud/test/storage/test_dummy.py
+++ b/libcloud/test/storage/test_dummy.py
@@ -50,11 +50,11 @@ def test_list_container_objects_filter_by_prefix(
container = driver.get_container(container_name)
objects = driver.list_container_objects(
- container=container, ex_prefix=object_name[:3]
+ container=container, prefix=object_name[:3]
)
assert any(o for o in objects if o.name == object_name)
objects = driver.list_container_objects(
- container=container, ex_prefix='does-not-exist.dat'
+ container=container, prefix='does-not-exist.dat'
)
assert not objects
diff --git a/libcloud/test/storage/test_local.py
b/libcloud/test/storage/test_local.py
index 1349a33..72c623a 100644
--- a/libcloud/test/storage/test_local.py
+++ b/libcloud/test/storage/test_local.py
@@ -110,6 +110,11 @@ class LocalTests(unittest.TestCase):
objects = self.driver.list_container_objects(container=container)
self.assertEqual(len(objects), 5)
+ prefix = os.path.join('path', 'to')
+ objects = self.driver.list_container_objects(container=container,
+ prefix=prefix)
+ self.assertEqual(len(objects), 2)
+
for obj in objects:
self.assertNotEqual(obj.hash, None)
self.assertEqual(obj.size, 4096)
diff --git a/libcloud/test/storage/test_oss.py
b/libcloud/test/storage/test_oss.py
index b767b61..58cd987 100644
--- a/libcloud/test/storage/test_oss.py
+++ b/libcloud/test/storage/test_oss.py
@@ -367,7 +367,7 @@ class OSSStorageDriverTestCase(unittest.TestCase):
driver=self.driver)
self.prefix = 'test_prefix'
objects = self.driver.list_container_objects(container=container,
- ex_prefix=self.prefix)
+ prefix=self.prefix)
self.assertEqual(len(objects), 2)
def test_get_container_doesnt_exist(self):
diff --git a/libcloud/test/storage/test_s3.py b/libcloud/test/storage/test_s3.py
index df4b384..7ee46c6 100644
--- a/libcloud/test/storage/test_s3.py
+++ b/libcloud/test/storage/test_s3.py
@@ -497,7 +497,7 @@ class S3Tests(unittest.TestCase):
container = Container(name='test_container', extra={},
driver=self.driver)
objects = self.driver.list_container_objects(container=container,
- ex_prefix='test_prefix')
+ prefix='test_prefix')
self.assertEqual(len(objects), 1)
obj = [o for o in objects if o.name == '1.zip'][0]
