This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 04e68bf6247 Keep .airflowignore rules when the Dags folder symlink is
switched (#74853)
04e68bf6247 is described below
commit 04e68bf6247b0accc404d745e4cd092bf87d0042
Author: Haram Lee <[email protected]>
AuthorDate: Mon Oct 19 05:36:22 2026 +0900
Keep .airflowignore rules when the Dags folder symlink is switched (#74853)
git-sync publishes each revision by repointing the Dags folder symlink
to a new checkout. When that happens during file discovery, child
directories resolve into the new checkout and no longer find the ignore
rules inherited from their parent, so files under ignored directories
are picked up for that scan.
---
.../airflow_shared/module_loading/file_discovery.py | 11 +++++++----
.../tests/module_loading/test_file_discovery.py | 18 ++++++++++++++++++
2 files changed, 25 insertions(+), 4 deletions(-)
diff --git
a/shared/module_loading/src/airflow_shared/module_loading/file_discovery.py
b/shared/module_loading/src/airflow_shared/module_loading/file_discovery.py
index 523dc8d35f1..00660816b25 100644
--- a/shared/module_loading/src/airflow_shared/module_loading/file_discovery.py
+++ b/shared/module_loading/src/airflow_shared/module_loading/file_discovery.py
@@ -147,11 +147,13 @@ def _find_path_from_directory(
:return: a generator of file paths which should not be ignored.
"""
- # A Dict of patterns, keyed using resolved, absolute paths
+ # A Dict of patterns, keyed using the walked (not resolved) paths, so that
a symlink switched
+ # during the walk (e.g. by git-sync) does not drop the patterns inherited
by subdirectories
patterns_by_dir: dict[Path, list[_IgnoreRule]] = {}
+ visited_dirs: set[Path] = set()
for root, dirs, files in os.walk(base_dir_path, followlinks=True):
- patterns: list[_IgnoreRule] =
patterns_by_dir.get(Path(root).resolve(), [])
+ patterns: list[_IgnoreRule] = patterns_by_dir.get(Path(root), [])
ignore_file_path = Path(root) / ignore_file_name
if ignore_file_path.is_file():
@@ -176,12 +178,13 @@ def _find_path_from_directory(
# explicit loop for infinite recursion detection since we are
following symlinks in this walk
for sd in dirs:
dirpath = (Path(root) / sd).resolve()
- if dirpath in patterns_by_dir:
+ if dirpath in visited_dirs:
raise RuntimeError(
"Detected recursive loop when walking DAG directory "
f"{base_dir_path}: {dirpath} has appeared more than once."
)
- patterns_by_dir.update({dirpath: patterns.copy()})
+ visited_dirs.add(dirpath)
+ patterns_by_dir[Path(root) / sd] = patterns.copy()
for file in files:
if file != ignore_file_name:
diff --git a/shared/module_loading/tests/module_loading/test_file_discovery.py
b/shared/module_loading/tests/module_loading/test_file_discovery.py
index a745f47fce6..388ad79c239 100644
--- a/shared/module_loading/tests/module_loading/test_file_discovery.py
+++ b/shared/module_loading/tests/module_loading/test_file_discovery.py
@@ -73,6 +73,24 @@ class TestFindPathFromDirectory:
with pytest.raises(RuntimeError, match=error_message):
list(find_path_from_directory(test_dir, ignore_list_file,
ignore_file_syntax="glob"))
+ def
test_find_path_from_directory_keeps_inherited_rules_on_symlink_switch(self,
tmp_path):
+ """Ignore rules inherited by a subdirectory still apply if the Dag
folder symlink is switched mid-walk."""
+ for revision in ("rev1", "rev2"):
+ (tmp_path / revision / "tests").mkdir(parents=True)
+ (tmp_path / revision / ".airflowignore").write_text("tests/")
+ (tmp_path / revision / "dag.py").touch()
+ (tmp_path / revision / "tests" / "test_dag.py").touch()
+ dags_folder = tmp_path / "dags"
+ dags_folder.symlink_to(tmp_path / "rev1")
+
+ paths = find_path_from_directory(dags_folder, ".airflowignore",
"regexp")
+ found = [next(paths)]
+ (tmp_path / "replacement").symlink_to(tmp_path / "rev2")
+ os.replace(tmp_path / "replacement", dags_folder)
+ found.extend(paths)
+
+ assert found == [os.path.join(dags_folder, "dag.py")]
+
def test_airflowignore_negation_unignore_subfolder_file_glob(self,
tmp_path):
"""Ensure negation rules can unignore a subfolder and a file inside it
when using glob syntax.
