jroachgolf84 commented on code in PR #55070:
URL: https://github.com/apache/airflow/pull/55070#discussion_r2324958378
##########
providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py:
##########
@@ -931,7 +932,51 @@ def get_file_metadata(
max_items: int | None = None,
) -> list:
"""
- List metadata objects in a bucket under prefix.
+ [DEPRECATED] Retrieve metadata objects from a bucket under a prefix.
+
+ This method `get_file_metadata` is deprecated. Calling this method
will result in all matching keys
+ being loaded into a single list, and can often result in out-of-memory
exceptions. Instead, use
+ `yield_file_metadata`.
+ """ # noqa: D401
+ warnings.warn(
+ "This method `get_file_metadata` is deprecated. Calling this
method will result in all matching "
+ "keys being loaded into a single list, and can often result in
out-of-memory exceptions. "
+ "Instead, use `yield_file_metadata`.",
+ AirflowProviderDeprecationWarning,
+ stacklevel=2,
+ )
+
+ config = {
+ "PageSize": page_size,
+ "MaxItems": max_items,
+ }
+
+ paginator = self.get_conn().get_paginator("list_objects_v2")
+ params = {
+ "Bucket": bucket_name,
+ "Prefix": prefix,
+ "PaginationConfig": config,
+ }
+ if self._requester_pays:
+ params["RequestPayer"] = "requester"
+ response = paginator.paginate(**params)
+
+ files = []
+ for page in response:
+ if "Contents" in page:
+ files += page["Contents"]
+ return files
+
+ @provide_bucket_name
+ def yield_file_metadata(
Review Comment:
@uranusjr - any thoughts here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
