jroachgolf84 commented on code in PR #55070:
URL: https://github.com/apache/airflow/pull/55070#discussion_r2359622387
##########
providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py:
##########
@@ -931,7 +932,51 @@ def get_file_metadata(
max_items: int | None = None,
) -> list:
"""
- List metadata objects in a bucket under prefix.
+ [DEPRECATED] Retrieve metadata objects from a bucket under a prefix.
+
+ This method `get_file_metadata` is deprecated. Calling this method
will result in all matching keys
+ being loaded into a single list, and can often result in out-of-memory
exceptions. Instead, use
+ `iter_file_metadata`.
+ """ # noqa: D401
+ warnings.warn(
+ "This method `get_file_metadata` is deprecated. Calling this
method will result in all matching "
+ "keys being loaded into a single list, and can often result in
out-of-memory exceptions. "
+ "Instead, use `iter_file_metadata`.",
+ AirflowProviderDeprecationWarning,
+ stacklevel=2,
+ )
+
+ config = {
+ "PageSize": page_size,
+ "MaxItems": max_items,
+ }
+
+ paginator = self.get_conn().get_paginator("list_objects_v2")
+ params = {
+ "Bucket": bucket_name,
+ "Prefix": prefix,
+ "PaginationConfig": config,
+ }
+ if self._requester_pays:
+ params["RequestPayer"] = "requester"
+ response = paginator.paginate(**params)
+
+ files = []
+ for page in response:
+ if "Contents" in page:
+ files += page["Contents"]
+ return files
Review Comment:
Implemented the below:
```python
return list(self.iter_file_metadata(prefix, bucket_name, page_size,
max_items))
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
