This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new fd6b7dba6f6 Resolve provider namespaces from the real directory tree
in prek hooks (#70468)
fd6b7dba6f6 is described below
commit fd6b7dba6f63fe4a702767b3967828c89fb76f28
Author: PoAn Yang <[email protected]>
AuthorDate: Thu Aug 13 22:31:09 2026 +0800
Resolve provider namespaces from the real directory tree in prek hooks
(#70468)
Signed-off-by: PoAn Yang <[email protected]>
---
.../check_providers_subpackages_all_have_init.py | 36 ++++++---
scripts/ci/prek/common_prek_utils.py | 27 ++++++-
scripts/ci/prek/mypy_folder.py | 21 ++---
...st_check_providers_subpackages_all_have_init.py | 59 ++++++++++++++
scripts/tests/ci/prek/test_common_prek_utils.py | 91 ++++++++++++++++++++++
5 files changed, 204 insertions(+), 30 deletions(-)
diff --git a/scripts/ci/prek/check_providers_subpackages_all_have_init.py
b/scripts/ci/prek/check_providers_subpackages_all_have_init.py
index a67f914e982..7a0eb4c15cb 100755
--- a/scripts/ci/prek/check_providers_subpackages_all_have_init.py
+++ b/scripts/ci/prek/check_providers_subpackages_all_have_init.py
@@ -30,8 +30,8 @@ from pathlib import Path
from common_prek_utils import (
AIRFLOW_PROVIDERS_ROOT_PATH,
AIRFLOW_ROOT_PATH,
- KNOWN_SECOND_LEVEL_PATHS,
console,
+ get_provider_namespace_from_path,
)
ACCEPTED_NON_INIT_DIRS = [
@@ -66,7 +66,9 @@ missing_init_dirs: list[Path] = []
missing_path_extension_dirs: list[Path] = []
-def _what_kind_of_test_init_py_needed(base_path: Path, folder: Path) ->
tuple[bool, bool]:
+def _what_kind_of_test_init_py_needed(
+ base_path: Path, folder: Path, namespace: str | None
+) -> tuple[bool, bool]:
"""Returns a tuple of two booleans indicating need and type of __init__.py
file.
The first boolean is True if __init__.py is needed, False otherwise.
@@ -86,12 +88,24 @@ def _what_kind_of_test_init_py_needed(base_path: Path,
folder: Path) -> tuple[bo
_ErrorSignals.fatal_error = True
return True, True
if depth == 2:
- # For known sub-packages that can occur in several packages we need to
add __path__ extension
- return True, folder.name in KNOWN_SECOND_LEVEL_PATHS
+ # For sub-packages that can occur in several packages we need to add
__path__ extension
+ return True, folder.name == namespace
# all other sub-packages should have plain __init__.py
return True, False
+def _needs_path_extension_in_src(relative_root_path: Path, namespace: str |
None) -> bool:
+ """Whether a folder under ``src/airflow`` is also shipped by other
distributions.
+
+ ``airflow`` and ``airflow/providers`` are shared by every distribution,
and the namespace
+ package below them by every distribution of that namespace.
+ """
+ parts = relative_root_path.parts
+ if len(parts) < 2:
+ return True
+ return len(parts) == 2 and parts[0] == "providers" and parts[1] ==
namespace
+
+
def _determine_init_py_action(need_path_extension: bool, root_path: Path):
init_py_file = root_path.joinpath("__init__.py")
if not init_py_file.exists():
@@ -112,12 +126,15 @@ def check_dir_init_test_folders(folders: list[Path]) ->
None:
for root_distribution_path in folders:
# We need init folders for all folders and for the common ones we need
path extension
tests_folder = root_distribution_path / "tests"
+ namespace = get_provider_namespace_from_path(root_distribution_path /
"provider.yaml")
print("Checking for __init__.py files in distribution for tests: ",
tests_folder)
for root, dirs, _ in os.walk(tests_folder):
# Edit it in place, so we don't recurse to folders we don't care
about
dirs[:] = [d for d in dirs if d not in ACCEPTED_NON_INIT_DIRS]
root_path = Path(root)
- need_init_py, need_path_extension =
_what_kind_of_test_init_py_needed(tests_folder, root_path)
+ need_init_py, need_path_extension =
_what_kind_of_test_init_py_needed(
+ tests_folder, root_path, namespace
+ )
if need_init_py:
_determine_init_py_action(need_path_extension, root_path)
@@ -127,6 +144,7 @@ def check_dir_init_src_folders(folders: list[Path]) -> None:
for root_distribution_path in folders:
# We need init folders for all folders and for the common ones we need
path extension
providers_base_folder = root_distribution_path / "src" / "airflow"
+ namespace = get_provider_namespace_from_path(root_distribution_path /
"provider.yaml")
print("Checking for __init__.py files in distribution for src: ",
providers_base_folder)
for root, dirs, _ in os.walk(providers_base_folder):
print("Checking: ", root)
@@ -139,13 +157,7 @@ def check_dir_init_src_folders(folders: list[Path]) ->
None:
and not any(pattern in root for pattern in IGNORE_DIR_PATTERNS)
]
relative_root_path = root_path.relative_to(providers_base_folder)
- need_path_extension = (
- root_path == providers_base_folder
- or len(relative_root_path.parts) == 1
- or len(relative_root_path.parts) == 2
- and relative_root_path.parts[1] in KNOWN_SECOND_LEVEL_PATHS
- and relative_root_path.parts[0] == "providers"
- )
+ need_path_extension =
_needs_path_extension_in_src(relative_root_path, namespace)
print("Needs path extension: ", need_path_extension)
_determine_init_py_action(need_path_extension, root_path)
diff --git a/scripts/ci/prek/common_prek_utils.py
b/scripts/ci/prek/common_prek_utils.py
index 559c2f0a047..62154dedad0 100644
--- a/scripts/ci/prek/common_prek_utils.py
+++ b/scripts/ci/prek/common_prek_utils.py
@@ -41,9 +41,6 @@ AIRFLOW_PROVIDERS_ROOT_PATH = AIRFLOW_ROOT_PATH / "providers"
AIRFLOW_TASK_SDK_ROOT_PATH = AIRFLOW_ROOT_PATH / "task-sdk"
AIRFLOW_TASK_SDK_SOURCES_PATH = AIRFLOW_TASK_SDK_ROOT_PATH / "src"
-# Here we should add the second level paths that we want to have sub-packages
in
-KNOWN_SECOND_LEVEL_PATHS = ["apache", "atlassian", "common", "cncf", "dbt",
"ibm", "microsoft"]
-
DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.10"
# Maps a Docker build platform string (as declared in ``provider.yaml`` under
@@ -591,6 +588,30 @@ def get_provider_base_dir_from_path(file_path: Path) ->
Path | None:
return None
+def get_provider_namespace_from_path(file_path: Path) -> str | None:
+ """Get the namespace of the nested provider the file belongs to, None if
it is not nested."""
+ provider_id = get_provider_id_from_path(file_path)
+ if not provider_id or "." not in provider_id:
+ return None
+ return provider_id.split(".")[0]
+
+
+def is_duplicated_namespace_init(file_path: Path) -> bool:
+ """Check whether the file is a namespace ``__init__.py`` repeated across
the namespace."""
+ if file_path.name != "__init__.py":
+ return False
+ namespace = get_provider_namespace_from_path(file_path)
+ base_dir = get_provider_base_dir_from_path(file_path)
+ if namespace is None or base_dir is None:
+ return False
+ return file_path.relative_to(base_dir).parts in {
+ ("src", "airflow", "providers", namespace, "__init__.py"),
+ ("tests", "unit", namespace, "__init__.py"),
+ ("tests", "integration", namespace, "__init__.py"),
+ ("tests", "system", namespace, "__init__.py"),
+ }
+
+
def get_all_provider_ids(
exclude_suspended_providers: bool = False, exclude_not_ready_providers:
bool = False
) -> list[str]:
diff --git a/scripts/ci/prek/mypy_folder.py b/scripts/ci/prek/mypy_folder.py
index 43106cf6201..420017c1616 100755
--- a/scripts/ci/prek/mypy_folder.py
+++ b/scripts/ci/prek/mypy_folder.py
@@ -30,10 +30,10 @@ import sys
from common_prek_utils import (
AIRFLOW_ROOT_PATH,
- KNOWN_SECOND_LEVEL_PATHS,
console,
get_all_provider_ids,
initialize_breeze_prek,
+ is_duplicated_namespace_init,
is_hidden_within_root,
run_command_via_breeze_run,
)
@@ -76,18 +76,6 @@ MYPY_FILE_LIST.parent.mkdir(parents=True, exist_ok=True)
FILE_ARGUMENT = "@/files/mypy_files.txt"
-all_provider_duplicated_path_to_ignore = []
-
-for second_level_path in KNOWN_SECOND_LEVEL_PATHS:
- all_provider_duplicated_path_to_ignore.extend(
- [
-
rf"^.*/providers/{second_level_path}/.*/src/airflow/providers/{second_level_path}/__init__.py$",
-
rf"^.*/providers/{second_level_path}/.*/tests/unit/{second_level_path}/__init__.py$",
-
rf"^.*/providers/{second_level_path}/.*/tests/integration/{second_level_path}/__init__.py$",
-
rf"^.*/providers/{second_level_path}/.*/tests/system/{second_level_path}/__init__.py$",
- ]
- )
-
exclude_regexps = [
re.compile(x)
for x in [
@@ -100,7 +88,6 @@ exclude_regexps = [
r"^.*/providers/.*/tests/unit/__init__.py$",
r"^.*/providers/.*/tests/integration/__init__.py$",
r"^.*/providers/.*/tests/system/__init__.py$",
- *all_provider_duplicated_path_to_ignore,
]
]
@@ -110,7 +97,11 @@ def get_all_files(folder: str) -> list[str]:
python_file_paths = (AIRFLOW_ROOT_PATH / folder).resolve().rglob("*.py")
for file in python_file_paths:
if (
- (file.name not in ("conftest.py",) and not
any(x.match(file.as_posix()) for x in exclude_regexps))
+ (
+ file.name not in ("conftest.py",)
+ and not any(x.match(file.as_posix()) for x in exclude_regexps)
+ and not is_duplicated_namespace_init(file)
+ )
and not is_hidden_within_root(file, AIRFLOW_ROOT_PATH)
) and not
file.as_posix().endswith("src/airflow/providers/__init__.py"):
files_to_check.append(file.relative_to(AIRFLOW_ROOT_PATH).as_posix())
diff --git
a/scripts/tests/ci/prek/test_check_providers_subpackages_all_have_init.py
b/scripts/tests/ci/prek/test_check_providers_subpackages_all_have_init.py
new file mode 100644
index 00000000000..96a59b8fb81
--- /dev/null
+++ b/scripts/tests/ci/prek/test_check_providers_subpackages_all_have_init.py
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+from ci.prek.check_providers_subpackages_all_have_init import (
+ _needs_path_extension_in_src,
+ _what_kind_of_test_init_py_needed,
+)
+
+
+class TestPathExtensionDecisions:
+ @pytest.mark.parametrize(
+ ("folder", "namespace", "expected"),
+ [
+ pytest.param("unit/apache", "apache", True,
id="namespace-folder-is-shared"),
+ pytest.param("unit/acme", "acme", True,
id="unknown-namespace-folder-is-shared"),
+ pytest.param("unit/hive", "apache", False,
id="provider-own-folder-is-not-shared"),
+ pytest.param("unit/amazon", None, False,
id="top-level-provider-has-no-shared-folder"),
+ ],
+ )
+ def test_only_the_namespace_folder_needs_path_extension(self, tmp_path,
folder, namespace, expected):
+ need_init_py, need_path_extension = _what_kind_of_test_init_py_needed(
+ tmp_path, tmp_path / folder, namespace
+ )
+
+ assert need_init_py is True
+ assert need_path_extension is expected
+
+ @pytest.mark.parametrize(
+ ("relative_path", "namespace", "expected"),
+ [
+ pytest.param(".", None, True, id="airflow-itself"),
+ pytest.param("providers", None, True, id="providers-package"),
+ pytest.param("providers/apache", "apache", True,
id="namespace-package-is-shared"),
+ pytest.param("providers/acme", "acme", True,
id="unknown-namespace-package-is-shared"),
+ pytest.param("providers/hive", "apache", False,
id="provider-package-is-not-shared"),
+ pytest.param("providers/amazon", None, False,
id="top-level-provider-package-is-not-shared"),
+ pytest.param("providers/apache/hive", "apache", False,
id="deeper-package-is-not-shared"),
+ ],
+ )
+ def test_only_shared_folders_need_path_extension(self, relative_path,
namespace, expected):
+ assert _needs_path_extension_in_src(Path(relative_path), namespace) is
expected
diff --git a/scripts/tests/ci/prek/test_common_prek_utils.py
b/scripts/tests/ci/prek/test_common_prek_utils.py
index e23baabc542..b169b9b90dc 100644
--- a/scripts/tests/ci/prek/test_common_prek_utils.py
+++ b/scripts/tests/ci/prek/test_common_prek_utils.py
@@ -27,8 +27,10 @@ from ci.prek.common_prek_utils import (
get_imports_from_file,
get_provider_base_dir_from_path,
get_provider_id_from_path,
+ get_provider_namespace_from_path,
initialize_breeze_prek,
insert_documentation,
+ is_duplicated_namespace_init,
is_hidden_within_root,
pre_process_mypy_files,
read_airflow_version,
@@ -541,6 +543,95 @@ class TestGetProviderBaseDirFromPath:
assert result == outer
+class TestGetProviderNamespaceFromPath:
+ @pytest.mark.parametrize(
+ ("provider_path", "expected"),
+ [
+ pytest.param("providers/apache/hive", "apache",
id="nested-provider"),
+ pytest.param("providers/acme/widget", "acme",
id="unknown-nested-provider"),
+ pytest.param("providers/amazon", None, id="top-level-provider"),
+ ],
+ )
+ def test_namespace_comes_from_the_directory_tree(self,
create_provider_tree, provider_path, expected):
+ assert
get_provider_namespace_from_path(create_provider_tree(provider_path)) ==
expected
+
+ def test_returns_none_outside_any_provider(self, tmp_path):
+ unrelated = tmp_path / "file.py"
+ unrelated.touch()
+ assert get_provider_namespace_from_path(unrelated) is None
+
+
+class TestIsDuplicatedNamespaceInit:
+ @staticmethod
+ def _make_provider(tmp_path, provider_path: str, init_relative_path: str):
+ provider_dir = tmp_path / provider_path
+ provider_dir.mkdir(parents=True)
+ (provider_dir / "provider.yaml").touch()
+ init_file = provider_dir / init_relative_path
+ init_file.parent.mkdir(parents=True, exist_ok=True)
+ init_file.touch()
+ return init_file
+
+ @pytest.mark.parametrize(
+ ("provider_path", "init_relative_path", "expected"),
+ [
+ pytest.param(
+ "providers/apache/hive",
+ "src/airflow/providers/apache/__init__.py",
+ True,
+ id="src-namespace-init",
+ ),
+ pytest.param(
+ "providers/apache/hive", "tests/unit/apache/__init__.py",
True, id="unit-namespace-init"
+ ),
+ pytest.param(
+ "providers/apache/hive",
+ "tests/integration/apache/__init__.py",
+ True,
+ id="integration-namespace-init",
+ ),
+ pytest.param(
+ "providers/apache/hive", "tests/system/apache/__init__.py",
True, id="system-namespace-init"
+ ),
+ pytest.param(
+ "providers/acme/widget",
+ "src/airflow/providers/acme/__init__.py",
+ True,
+ id="unknown-namespace-init",
+ ),
+ pytest.param(
+ "providers/apache/hive",
+ "src/airflow/providers/apache/hive/__init__.py",
+ False,
+ id="provider-own-init-is-unique",
+ ),
+ pytest.param(
+ "providers/amazon",
+ "src/airflow/providers/amazon/__init__.py",
+ False,
+ id="top-level-provider-init-is-unique",
+ ),
+ pytest.param(
+ "providers/apache/hive",
+ "src/airflow/providers/apache/hive/hooks/hive.py",
+ False,
+ id="not-an-init-file",
+ ),
+ ],
+ )
+ def test_only_repeated_namespace_inits_are_reported(
+ self, tmp_path, provider_path, init_relative_path, expected
+ ):
+ init_file = self._make_provider(tmp_path, provider_path,
init_relative_path)
+ assert is_duplicated_namespace_init(init_file) is expected
+
+ def test_returns_false_outside_any_provider(self, tmp_path):
+ unrelated = tmp_path / "airflow" / "providers" / "apache" /
"__init__.py"
+ unrelated.parent.mkdir(parents=True)
+ unrelated.touch()
+ assert is_duplicated_namespace_init(unrelated) is False
+
+
class TestInitializeBreezePrek:
def test_raises_when_not_main(self):
with pytest.raises(SystemExit, match="intended to be executed"):
