This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f5a2a44b6a4 fix flaky test (#60398)
f5a2a44b6a4 is described below
commit f5a2a44b6a40547706bee1fec30109428f6d810c
Author: Henry Chen <[email protected]>
AuthorDate: Thu Jan 15 04:12:42 2026 +0800
fix flaky test (#60398)
* fix flaky test
* Add a new assertion
* Use list to log reason_msg
---
.../src/airflow/providers/amazon/aws/hooks/s3.py | 22 ++++++++++---------
.../amazon/tests/unit/amazon/aws/hooks/test_s3.py | 25 ++++++++++++----------
2 files changed, 26 insertions(+), 21 deletions(-)
diff --git a/providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
b/providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
index f24cebc8be6..909423c06d3 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
@@ -1743,29 +1743,31 @@ class S3Hook(AwsBaseHook):
def _sync_to_local_dir_if_changed(self, s3_bucket, s3_object,
local_target_path: Path):
should_download = False
- download_msg = ""
+ download_logs: list[str] = []
+ download_log_params: list[Any] = []
+
if not local_target_path.exists():
should_download = True
- download_msg = f"Local file {local_target_path} does not exist."
+ download_logs.append("Local file %s does not exist.")
+ download_log_params.append(local_target_path)
else:
local_stats = local_target_path.stat()
-
if s3_object.size != local_stats.st_size:
should_download = True
- download_msg = (
- f"S3 object size ({s3_object.size}) and local file size
({local_stats.st_size}) differ."
- )
+ download_logs.append("S3 object size (%s) and local file size
(%s) differ.")
+ download_log_params.extend([s3_object.size,
local_stats.st_size])
s3_last_modified = s3_object.last_modified
if local_stats.st_mtime < s3_last_modified.timestamp():
should_download = True
- download_msg = f"S3 object last modified
({s3_last_modified.microsecond}) and local file last modified
({local_stats.st_mtime}) differ."
+ download_logs.append("S3 object last modified (%s) and local
file last modified (%s) differ.")
+ download_log_params.extend([s3_last_modified.timestamp(),
local_stats.st_mtime])
if should_download:
s3_bucket.download_file(s3_object.key, local_target_path)
- self.log.debug(
- "%s Downloaded %s to %s", download_msg, s3_object.key,
local_target_path.as_posix()
- )
+ download_logs.append("Downloaded %s to %s")
+ download_log_params.extend([s3_object.key,
local_target_path.as_posix()])
+ self.log.debug(" ".join(download_logs), *download_log_params)
else:
self.log.debug(
"Local file %s is up-to-date with S3 object %s. Skipping
download.",
diff --git a/providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py
b/providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py
index 4b6dae9c206..ef3c9fddb8a 100644
--- a/providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py
+++ b/providers/amazon/tests/unit/amazon/aws/hooks/test_s3.py
@@ -1799,9 +1799,9 @@ class TestAwsS3Hook:
)
logs_string = get_logs_string(hook.log.debug.call_args_list)
assert f"Downloading data from s3://{s3_bucket}" in logs_string
- assert f"does not exist. Downloaded dag_01.py to
{sync_local_dir}/dag_01.py" in logs_string
- assert "does not exist. Downloaded dag_01.py to" in logs_string
- assert f"does not exist. Downloaded subproject1/dag_a.py to
{sync_local_dir}" in logs_string
+ assert "does not exist" in logs_string
+ assert f"Downloaded dag_01.py to {sync_local_dir}/dag_01.py" in
logs_string
+ assert f"Downloaded subproject1/dag_a.py to {sync_local_dir}" in
logs_string
# add new file to bucket and sync
hook.log.debug = MagicMock()
@@ -1810,11 +1810,9 @@ class TestAwsS3Hook:
bucket_name=s3_bucket, local_dir=sync_local_dir, s3_prefix="",
delete_stale=True
)
logs_string = get_logs_string(hook.log.debug.call_args_list)
- assert (
- "subproject1/dag_b.py is up-to-date with S3 object
subproject1/dag_b.py. Skipping download"
- in logs_string
- )
- assert f"does not exist. Downloaded dag_03.py to
{sync_local_dir}/dag_03.py" in logs_string
+ assert "is up-to-date with S3 object subproject1/dag_b.py" in
logs_string
+ assert "does not exist" in logs_string
+ assert f"Downloaded dag_03.py to {sync_local_dir}/dag_03.py" in
logs_string
# read that file is donloaded and has same content
assert Path(sync_local_dir).joinpath("dag_03.py").read_text() == "test
data"
@@ -1837,8 +1835,12 @@ class TestAwsS3Hook:
bucket_name=s3_bucket, local_dir=sync_local_dir, s3_prefix="",
delete_stale=True
)
logs_string = get_logs_string(hook.log.debug.call_args_list)
- assert "S3 object size" in logs_string
- assert "differ. Downloaded dag_03.py to" in logs_string
+ # When file size differs, we should see size difference message
+ assert ("S3 object size" in logs_string and "differ" in logs_string)
or (
+ "S3 object last modified" in logs_string
+ )
+ assert "Downloaded dag_03.py to" in logs_string
+ assert Path(sync_local_dir).joinpath("dag_03.py").read_text() == "test
data-changed"
local_file_same_size = Path(sync_local_dir).joinpath("dag_04.py")
local_file_same_size.write_text("same size")
@@ -1853,9 +1855,10 @@ class TestAwsS3Hook:
bucket_name=s3_bucket, local_dir=sync_local_dir, s3_prefix="",
delete_stale=True
)
logs_string = get_logs_string(hook.log.debug.call_args_list)
+ # When file size is same but timestamp differs, we should see the
timestamp difference message
assert "S3 object last modified" in logs_string
assert "local file last modified" in logs_string
- assert "differ. Downloaded dag_04.py to" in logs_string
+ assert "Downloaded dag_04.py to" in logs_string
@pytest.mark.parametrize(
