This is an automated email from the ASF dual-hosted git repository.
kaxil pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 12127b07978 Fix check-conn-id-templated crashing on a vanishing
__pycache__ (#74016)
12127b07978 is described below
commit 12127b079788b5f294ababc508ece35e037dd6da
Author: Kaxil Naik <[email protected]>
AuthorDate: Thu Oct 1 10:53:37 2026 +0100
Fix check-conn-id-templated crashing on a vanishing __pycache__ (#74016)
The hook walked the providers tree with Path.rglob, which on Python 3.10
raises FileNotFoundError when a directory is removed after rglob lists it
but
before it reads it. Other processes create and remove __pycache__
directories
under providers/ while static checks run, so the hook failed intermittently.
Walk with os.walk, which skips a directory that has gone, and prune
__pycache__ and tests directories instead of filtering paths afterwards.
---
scripts/ci/prek/check_conn_id_templated.py | 18 ++++++++++-----
.../tests/ci/prek/test_check_conn_id_templated.py | 26 +++++++++++++++++++++-
2 files changed, 38 insertions(+), 6 deletions(-)
diff --git a/scripts/ci/prek/check_conn_id_templated.py
b/scripts/ci/prek/check_conn_id_templated.py
index 9e242dc3954..b011f0c9c26 100755
--- a/scripts/ci/prek/check_conn_id_templated.py
+++ b/scripts/ci/prek/check_conn_id_templated.py
@@ -39,6 +39,7 @@ or it is never stored under ``self.<argument>`` (there is
nothing for the render
from __future__ import annotations
import ast
+import os
import sys
from dataclasses import dataclass, field
from pathlib import Path
@@ -89,12 +90,19 @@ class ClassInfo:
def iter_source_files(roots: list[Path]):
- """Yield source (non-test) Python files under the roots."""
+ """
+ Yield source (non-test) Python files under the roots.
+
+ ``os.walk`` rather than ``Path.rglob``: other processes create and remove
``__pycache__``
+ directories under the providers tree while this walks it, and ``rglob``
raises
+ ``FileNotFoundError`` on a directory that disappears mid-walk, where
``os.walk`` skips it.
+ """
for root in roots:
- for path in root.rglob("*.py"):
- parts = path.relative_to(root).parts
- if "tests" not in parts:
- yield path
+ for dirpath, dirnames, filenames in os.walk(root):
+ dirnames[:] = [name for name in dirnames if name not in
("__pycache__", "tests")]
+ for filename in filenames:
+ if filename.endswith(".py"):
+ yield Path(dirpath, filename)
def parse_classes(path: Path) -> list[ClassInfo]:
diff --git a/scripts/tests/ci/prek/test_check_conn_id_templated.py
b/scripts/tests/ci/prek/test_check_conn_id_templated.py
index 3272b5d5e08..1a0bf767271 100644
--- a/scripts/tests/ci/prek/test_check_conn_id_templated.py
+++ b/scripts/tests/ci/prek/test_check_conn_id_templated.py
@@ -16,10 +16,11 @@
# under the License.
from __future__ import annotations
+import shutil
import textwrap
import pytest
-from check_conn_id_templated import build_index, check_file, main
+from check_conn_id_templated import build_index, check_file,
iter_source_files, main
BASE = """
class BaseOperator:
@@ -223,6 +224,29 @@ def test_missing_conn_ids_are_reported(check, code,
expected):
assert errors[0].endswith(f"{expected} missing from template_fields")
+def test_iter_source_files_skips_tests_and_bytecode_directories(tmp_path):
+ for relative in ("pkg/source.py", "pkg/tests/test_source.py",
"pkg/__pycache__/stray.py"):
+ (tmp_path / relative).parent.mkdir(parents=True, exist_ok=True)
+ (tmp_path / relative).write_text("")
+
+ assert list(iter_source_files([tmp_path])) == [tmp_path / "pkg" /
"source.py"]
+
+
+def test_iter_source_files_survives_a_directory_removed_mid_walk(tmp_path):
+ """Another process can remove a __pycache__ directory after the walk has
listed it but before it reads it."""
+ for name in ("a", "b"):
+ (tmp_path / name).mkdir()
+ (tmp_path / name / f"{name}.py").write_text("")
+
+ found = []
+ for path in iter_source_files([tmp_path]):
+ found.append(path)
+ # Both siblings have been listed by now; remove the one not read yet.
+ shutil.rmtree(tmp_path / ("b" if path.parent.name == "a" else "a"),
ignore_errors=True)
+
+ assert len(found) == 1
+
+
def test_main_skips_files_without_conn_id(tmp_path):
path = tmp_path / "code.py"
path.write_text("class MyOperator:\n pass\n")
