This is an automated email from the ASF dual-hosted git repository.
Lee-W pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 030efa6f74a Resolve nested provider paths for conn-fields check from
the real directory tree (#70261)
030efa6f74a is described below
commit 030efa6f74a50c54e4d4e28df6d8d98eb227c7c4
Author: Wei Lee <[email protected]>
AuthorDate: Fri Jul 24 15:48:54 2026 +0800
Resolve nested provider paths for conn-fields check from the real directory
tree (#70261)
---
scripts/ci/prek/check_provider_yaml_files.py | 48 ++++++-------
.../ci/prek/test_check_provider_yaml_files.py | 84 ++++++++++++++++++++++
2 files changed, 106 insertions(+), 26 deletions(-)
diff --git a/scripts/ci/prek/check_provider_yaml_files.py
b/scripts/ci/prek/check_provider_yaml_files.py
index 3f3ec5183a5..5e8d8da7a98 100755
--- a/scripts/ci/prek/check_provider_yaml_files.py
+++ b/scripts/ci/prek/check_provider_yaml_files.py
@@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.
# /// script
-# requires-python = ">=3.10,<3.11"
+# requires-python = ">=3.10"
# dependencies = [
# "rich>=13.6.0",
# ]
@@ -27,14 +27,12 @@ import pathlib
import sys
from common_prek_utils import (
- KNOWN_SECOND_LEVEL_PATHS,
+ get_provider_base_dir_from_path,
initialize_breeze_prek,
run_command_via_breeze_run,
validate_cmd_result,
)
-initialize_breeze_prek(__name__, __file__)
-
def _resolve_provider_yaml_files(raw_files: list[str]) -> list[str]:
"""
@@ -46,11 +44,12 @@ def _resolve_provider_yaml_files(raw_files: list[str]) ->
list[str]:
conn-fields check runs even when only the hook changes.
All paths are relative to the ``providers/`` directory, as supplied by
- prek. The first path segment is the provider package name
- (e.g. ``samba/src/airflow/...`` → ``samba/provider.yaml``), except for
- namespace packages in ``KNOWN_SECOND_LEVEL_PATHS`` (e.g. ``apache``,
- ``common``), which nest an extra level (e.g.
- ``apache/beam/src/airflow/...`` → ``apache/beam/provider.yaml``).
+ prek. Rather than guessing how many path segments make up the provider
+ package name, this walks up the real directory tree (via
+ ``get_provider_base_dir_from_path``) until it finds the actual
+ ``provider.yaml`` file, so nested/namespace provider packages
+ (e.g. ``apache/beam``, ``ibm/mq``) resolve correctly without maintaining
+ a list of known namespace prefixes.
"""
result: set[str] = set()
for f in raw_files:
@@ -58,23 +57,20 @@ def _resolve_provider_yaml_files(raw_files: list[str]) ->
list[str]:
if p.name == "provider.yaml":
result.add(f)
else:
- # Map any Python file to the provider.yaml of its package root.
- # Path structure: <provider-pkg>/<rest...>
- # e.g. samba/src/airflow/providers/samba/hooks/samba.py
- parts = p.parts
- if parts:
- if parts[0] in KNOWN_SECOND_LEVEL_PATHS and len(parts) > 1:
- result.add(f"{parts[0]}/{parts[1]}/provider.yaml")
- else:
- result.add(f"{parts[0]}/provider.yaml")
+ provider_dir = get_provider_base_dir_from_path(pathlib.Path(f))
+ if provider_dir is not None:
+ result.add((provider_dir / "provider.yaml").as_posix())
return sorted(result)
-files_to_test = _resolve_provider_yaml_files(sys.argv[1:])
-cmd_result = run_command_via_breeze_run(
- ["python3",
"/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py",
*files_to_test],
- backend="sqlite",
- warn_image_upgrade_needed=True,
- extra_env={"PYTHONWARNINGS": "default"},
-)
-validate_cmd_result(cmd_result, include_ci_env_check=True)
+if __name__ == "__main__":
+ initialize_breeze_prek(__name__, __file__)
+
+ files_to_test = _resolve_provider_yaml_files(sys.argv[1:])
+ cmd_result = run_command_via_breeze_run(
+ ["python3",
"/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py",
*files_to_test],
+ backend="sqlite",
+ warn_image_upgrade_needed=True,
+ extra_env={"PYTHONWARNINGS": "default"},
+ )
+ validate_cmd_result(cmd_result, include_ci_env_check=True)
diff --git a/scripts/tests/ci/prek/test_check_provider_yaml_files.py
b/scripts/tests/ci/prek/test_check_provider_yaml_files.py
new file mode 100644
index 00000000000..590ad73efb2
--- /dev/null
+++ b/scripts/tests/ci/prek/test_check_provider_yaml_files.py
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pytest
+from ci.prek.check_provider_yaml_files import _resolve_provider_yaml_files
+
+
+def _touch(path):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.touch()
+
+
+class TestResolveProviderYamlFiles:
+ @pytest.mark.parametrize(
+ "file_names, raw_files, expected",
+ (
+ (
+ ["samba/provider.yaml"],
+ ["samba/provider.yaml"],
+ ["samba/provider.yaml"],
+ ),
+ (
+ [
+ "samba/provider.yaml",
+ "samba/src/airflow/providers/samba/hooks/samba.py",
+ ],
+ ["samba/src/airflow/providers/samba/hooks/samba.py"],
+ ["samba/provider.yaml"],
+ ),
+ (
+ [
+ "ibm/mq/provider.yaml",
+ "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py",
+ ],
+ ["ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py"],
+ ["ibm/mq/provider.yaml"],
+ ),
+ (
+ [
+ "samba/provider.yaml",
+ "samba/src/airflow/providers/samba/hooks/samba.py",
+ "ibm/mq/provider.yaml",
+ "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py",
+ ],
+ [
+ "samba/provider.yaml",
+ "samba/src/airflow/providers/samba/hooks/samba.py",
+ "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py",
+ ],
+ ["ibm/mq/provider.yaml", "samba/provider.yaml"],
+ ),
+ (["unrelated/hooks/hook.py"], ["unrelated/hooks/hook.py"], []),
+ ),
+ ids=[
+ "provider_yaml_path_preserved",
+ "top_level_provider_hook_file_resolves",
+ "nested_namespace_provider_hook_file_resolves_without_known_list",
+ "mixed_input_dedups_and_sorts",
+ "no_provider_found",
+ ],
+ )
+ def test__resolve_provider_yaml_files(self, tmp_path, monkeypatch,
file_names, raw_files, expected):
+ monkeypatch.chdir(tmp_path)
+ for file_name in file_names:
+ _touch(tmp_path / file_name)
+
+ result = _resolve_provider_yaml_files(raw_files)
+
+ assert result == expected
