This is an automated email from the ASF dual-hosted git repository.
vincbeck pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0b166f4ab19 Fix invalid RequestPayer usage in S3Hook.select_key()
method (#63148)
0b166f4ab19 is described below
commit 0b166f4ab192abc904ff29db4f4a461cec5a9586
Author: Justin Pakzad <[email protected]>
AuthorDate: Mon Mar 9 10:06:26 2026 -0400
Fix invalid RequestPayer usage in S3Hook.select_key() method (#63148)
---
.../amazon/src/airflow/providers/amazon/aws/hooks/s3.py | 7 ++++---
providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py | 12 ++++++++++--
2 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
b/providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
index d71a8383e8a..f9beb0d0400 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
@@ -1107,14 +1107,16 @@ class S3Hook(AwsBaseHook):
"""
expression = expression or "SELECT * FROM S3Object"
expression_type = expression_type or "SQL"
- extra_args = {}
if input_serialization is None:
input_serialization = {"CSV": {}}
if output_serialization is None:
output_serialization = {"CSV": {}}
if self._requester_pays:
- extra_args["RequestPayer"] = "requester"
+ raise ValueError(
+ "select_key cannot be used with requester_pays=True. "
+ "S3 Select does not support the RequestPayer parameter."
+ )
response = self.get_conn().select_object_content(
Bucket=bucket_name,
@@ -1123,7 +1125,6 @@ class S3Hook(AwsBaseHook):
ExpressionType=expression_type,
InputSerialization=input_serialization,
OutputSerialization=output_serialization,
- ExtraArgs=extra_args,
)
return b"".join(
diff --git a/providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py
b/providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py
index f84f7b8ba0a..14371da9cb4 100644
--- a/providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py
+++ b/providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py
@@ -474,18 +474,26 @@ class TestAwsS3Hook:
mock_get_client_type.return_value.select_object_content.return_value =
{
"Payload": [{"Records": {"Payload": b"Cont\xc3"}}, {"Records":
{"Payload": b"\xa9nt"}}]
}
- hook = S3Hook(requester_pays=True)
+ hook = S3Hook(requester_pays=False)
assert hook.select_key("my_key", s3_bucket) == "Contént"
mock_get_client_type.return_value.select_object_content.assert_called_with(
Bucket="airflow-test-s3-bucket",
Expression="SELECT * FROM S3Object",
ExpressionType="SQL",
- ExtraArgs={"RequestPayer": "requester"},
InputSerialization={"CSV": {}},
Key="my_key",
OutputSerialization={"CSV": {}},
)
+
@mock.patch("airflow.providers.amazon.aws.hooks.base_aws.AwsBaseHook.get_client_type")
+ def test_select_key_with_requester_pay(self, mock_get_client_type,
s3_bucket):
+ mock_get_client_type.return_value.select_object_content.return_value =
{
+ "Payload": [{"Records": {"Payload": b"Cont\xc3"}}, {"Records":
{"Payload": b"\xa9nt"}}]
+ }
+ hook = S3Hook(requester_pays=True)
+ with pytest.raises(ValueError, match="select_key cannot be used with
requester_pays"):
+ hook.select_key("my_key", s3_bucket)
+
def test_check_for_wildcard_key(self, s3_bucket):
hook = S3Hook()
bucket = hook.get_bucket(s3_bucket)
