This is an automated email from the ASF dual-hosted git repository.
o-nikolas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6570b8282f2 Skip S3 keys that map to the local sync directory (#74322)
6570b8282f2 is described below
commit 6570b8282f2bc1b8b53a60ef7834ae0c80cb5c9c
Author: Jingi Kim <[email protected]>
AuthorDate: Sat Oct 24 00:36:05 2026 +0900
Skip S3 keys that map to the local sync directory (#74322)
Period-only object keys can interrupt Dag bundle refreshes because their
download destination is the bundle directory itself. Also Skip nested S3 keys
resolving to directories: parent-directory segments can resolve back to the
bundle root and interrupt refresh even after passing the containment check.
---
.../src/airflow/providers/amazon/aws/hooks/s3.py | 4 +-
.../tests/unit/amazon/aws/bundles/test_s3.py | 18 ++++++++
.../amazon/tests/unit/amazon/aws/hooks/test_s3.py | 51 ++++++++++++++++++++++
3 files changed, 72 insertions(+), 1 deletion(-)
diff --git a/providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
b/providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
index e9089f95bfc..53ef3cd3805 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
@@ -1877,11 +1877,13 @@ class S3Hook(AwsBaseHook):
obj_path = Path(obj.key)
local_target_path =
local_dir.joinpath(obj_path.relative_to(s3_prefix))
try:
- local_target_path.resolve().relative_to(local_dir_resolved)
+ resolved_relative_path =
local_target_path.resolve().relative_to(local_dir_resolved)
except ValueError:
raise S3HookPathTraversalError(
f"S3 object key {obj.key!r} resolves outside local
directory {local_dir}"
) from None
+ if resolved_relative_path == Path(".") or obj.key.endswith(("/.",
"/..")):
+ continue
if not local_target_path.parent.exists():
local_target_path.parent.mkdir(parents=True, exist_ok=True)
self.log.debug("Created local directory: %s",
local_target_path.parent)
diff --git a/providers/amazon/tests/unit/amazon/aws/bundles/test_s3.py
b/providers/amazon/tests/unit/amazon/aws/bundles/test_s3.py
index 9df5202fe71..e8899a2f201 100644
--- a/providers/amazon/tests/unit/amazon/aws/bundles/test_s3.py
+++ b/providers/amazon/tests/unit/amazon/aws/bundles/test_s3.py
@@ -222,6 +222,24 @@ class TestS3DagBundle:
assert bundle._log.debug.call_count == 2
assert bundle._log.debug.call_args_list == [download_log_call,
download_log_call]
+ def test_refresh_skips_key_mapping_to_bundle_root(self, s3_client):
+ s3_client.create_bucket(Bucket=S3_BUCKET_NAME)
+ s3_client.put_object(Bucket=S3_BUCKET_NAME, Key="dags/old.py",
Body=b"old content")
+ bundle = S3DagBundle(name="test", bucket_name=S3_BUCKET_NAME,
prefix="dags")
+ bundle.initialize()
+ assert (bundle.path / "old.py").read_bytes() == b"old content"
+
+ s3_client.delete_object(Bucket=S3_BUCKET_NAME, Key="dags/old.py")
+ s3_client.put_object(Bucket=S3_BUCKET_NAME, Key="dags/.",
Body=b"directory key")
+ s3_client.put_object(Bucket=S3_BUCKET_NAME, Key="dags/new.py",
Body=b"new content")
+
+ bundle.refresh()
+ bundle.refresh()
+
+ assert bundle.path.is_dir()
+ assert not (bundle.path / "old.py").exists()
+ assert (bundle.path / "new.py").read_bytes() == b"new content"
+
@pytest.mark.parametrize("prefix", ["dags", "dags/", "project/dags",
"project/dags/"])
@pytest.mark.parametrize("extra_suffix", ["_archive/other.py",
pytest.param("", id="key_equals_prefix")])
def test_refresh_uses_directory_prefix(self, s3_client, prefix,
extra_suffix):
diff --git a/providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py
b/providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py
index c5f373a4e28..5a4d4d9fe63 100644
--- a/providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py
+++ b/providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py
@@ -2148,6 +2148,57 @@ class TestAwsS3Hook:
assert "local file last modified" in logs_string
assert "Downloaded dag_04.py to" in logs_string
+ @pytest.mark.parametrize(
+ ("prefix", "key"),
+ [
+ pytest.param("dags/", "dags/.", id="prefix-dot"),
+ pytest.param("dags/", "dags/./.", id="repeated-dot"),
+ pytest.param("dags/", "dags/../s3_sync_dir",
id="parent-directory"),
+ pytest.param("", ".", id="bucket-root-dot"),
+ ],
+ )
+ def test_sync_to_local_dir_skips_keys_mapping_to_root(self, s3_bucket,
s3_client, tmp_path, prefix, key):
+ s3_client.put_object(Bucket=s3_bucket, Key=key, Body=b"directory key")
+ s3_client.put_object(Bucket=s3_bucket, Key=f"{prefix}nested/dag.py",
Body=b"dag content")
+ s3_client.put_object(Bucket=s3_bucket, Key=f"{prefix}.airflowignore",
Body=b"ignore content")
+ sync_local_dir = tmp_path / "s3_sync_dir"
+ sync_local_dir.mkdir()
+ stale_file = sync_local_dir / "stale.py"
+ stale_file.write_bytes(b"stale")
+
+ S3Hook().sync_to_local_dir(bucket_name=s3_bucket,
local_dir=sync_local_dir, s3_prefix=prefix)
+
+ assert sync_local_dir.is_dir()
+ assert (sync_local_dir / "nested/dag.py").read_bytes() == b"dag
content"
+ assert (sync_local_dir / ".airflowignore").read_bytes() == b"ignore
content"
+ assert not stale_file.exists()
+ assert s3_client.get_object(Bucket=s3_bucket, Key=key)["Body"].read()
== b"directory key"
+
+ @pytest.mark.parametrize(
+ ("key", "when_added"),
+ [
+ pytest.param("dags/nested/.", "before", id="fresh_sync_dot"),
+ pytest.param("dags/nested/deep/..", "after", id="refresh_parent"),
+ ],
+ )
+ def test_sync_to_local_dir_skips_keys_mapping_to_nested_directory(
+ self, s3_bucket, s3_client, tmp_path, key, when_added
+ ):
+ if when_added == "before":
+ s3_client.put_object(Bucket=s3_bucket, Key=key, Body=b"directory
key")
+ s3_client.put_object(Bucket=s3_bucket, Key="dags/nested/dag.py",
Body=b"dag content")
+ sync_local_dir = tmp_path / "s3_sync_dir"
+ sync_local_dir.mkdir()
+
+ hook = S3Hook()
+ hook.sync_to_local_dir(bucket_name=s3_bucket,
local_dir=sync_local_dir, s3_prefix="dags/")
+ assert (sync_local_dir / "nested/dag.py").read_bytes() == b"dag
content"
+
+ if when_added == "after":
+ s3_client.put_object(Bucket=s3_bucket, Key=key, Body=b"directory
key")
+ hook.sync_to_local_dir(bucket_name=s3_bucket,
local_dir=sync_local_dir, s3_prefix="dags/")
+ assert (sync_local_dir / "nested/dag.py").read_bytes() == b"dag
content"
+
def test_sync_to_local_dir_rejects_key_path_traversal(self, s3_bucket,
s3_client, tmp_path):
s3_client.put_object(Bucket=s3_bucket, Key="dags/../../outside.py",
Body=b"test data")
