This is an automated email from the ASF dual-hosted git repository.
bolke pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 55b015f995 Add support for anonymous access to s3 buckets for
objectstorage (#35273)
55b015f995 is described below
commit 55b015f995def3bc8a3a9eef6abd7bcad49888f7
Author: Bolke de Bruin <bo...@xs4all.nl>
AuthorDate: Tue Oct 31 13:27:11 2023 +0100
Add support for anonymous access to s3 buckets for objectstorage (#35273)
Open data buckets on S3 can require that authentication is turned off. If
no credentials
are obtained we now set the anon flag to make sure that credentials headers
(like aws_access_key)
are dropped and unsigned is set.
---
airflow/providers/amazon/aws/fs/s3.py | 8 +++++++-
tests/providers/amazon/aws/fs/test_fs.py | 16 ++++++++++++++++
2 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/airflow/providers/amazon/aws/fs/s3.py
b/airflow/providers/amazon/aws/fs/s3.py
index afe13be1b4..c2eefcc379 100644
--- a/airflow/providers/amazon/aws/fs/s3.py
+++ b/airflow/providers/amazon/aws/fs/s3.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import annotations
+import asyncio
import logging
from functools import partial
from typing import TYPE_CHECKING, Any, Callable, Dict
@@ -85,7 +86,12 @@ def get_fs(conn_id: str | None) -> AbstractFileSystem:
if proxy_uri := s3_service_config.get(S3_PROXY_URI, None):
config_kwargs["proxies"] = {"http": proxy_uri, "https": proxy_uri}
- fs = S3FileSystem(session=session, config_kwargs=config_kwargs,
endpoint_url=endpoint_url)
+ anon = False
+ if asyncio.run(session.get_credentials()) is None:
+ log.info("No credentials found, using anonymous access")
+ anon = True
+
+ fs = S3FileSystem(session=session, config_kwargs=config_kwargs,
endpoint_url=endpoint_url, anon=anon)
for event_name, event_function in register_events.items():
fs.s3.meta.events.register_last(event_name, event_function,
unique_id=1925)
diff --git a/tests/providers/amazon/aws/fs/test_fs.py
b/tests/providers/amazon/aws/fs/test_fs.py
index 7a392a2832..babddf078a 100644
--- a/tests/providers/amazon/aws/fs/test_fs.py
+++ b/tests/providers/amazon/aws/fs/test_fs.py
@@ -16,6 +16,9 @@
# under the License.
from __future__ import annotations
+import os
+from unittest.mock import patch
+
import pytest
import responses
from botocore.awsrequest import AWSRequest
@@ -39,6 +42,19 @@ class TestFilesystem:
assert "s3" in fs.protocol
+ @patch("s3fs.S3FileSystem", autospec=True)
+ def test_get_s3fs_anonymous(self, s3fs, monkeypatch):
+ from airflow.providers.amazon.aws.fs.s3 import get_fs
+
+ # remove all AWS_* env vars
+ for env_name in os.environ:
+ if env_name.startswith("AWS"):
+ monkeypatch.delenv(env_name, raising=False)
+
+ get_fs(conn_id=None)
+
+ assert s3fs.call_args.kwargs["anon"] is True
+
@responses.activate
def test_signer(self):
from airflow.providers.amazon.aws.fs.s3 import s3v4_rest_signer
