This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 23c6e77067f Fix provider.yaml doc URL check with several suspended
providers (#73782)
23c6e77067f is described below
commit 23c6e77067fb3c04d30b330988d5ebc7bdd8b351
Author: rjgoyln <[email protected]>
AuthorDate: Mon Sep 28 04:06:29 2026 +0800
Fix provider.yaml doc URL check with several suspended providers (#73782)
Both suspended-provider filters in the check cross-multiply their inputs, so
an entry survives whenever it fails to match any single suspended provider.
The check has only ever run with zero or one suspended provider, where that
reading is accidentally correct, so suspending a second one would fail the
check on the suspended providers' own documentation and report unrelated
errors once per suspended provider.
---
scripts/in_container/provider_suspension.py | 38 +++++++++++++
.../in_container/run_provider_yaml_files_check.py | 18 +-----
.../tests/in_container/test_provider_suspension.py | 64 ++++++++++++++++++++++
3 files changed, 105 insertions(+), 15 deletions(-)
diff --git a/scripts/in_container/provider_suspension.py
b/scripts/in_container/provider_suspension.py
new file mode 100644
index 00000000000..947d1d2f916
--- /dev/null
+++ b/scripts/in_container/provider_suspension.py
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Filters that drop what a suspended provider leaves behind in the
provider.yaml checks."""
+
+from __future__ import annotations
+
+from collections.abc import Collection
+
+
+def remove_suspended_doc_urls(doc_urls: Collection[str], suspended_packages:
Collection[str]) -> set[str]:
+ """Drop documentation urls that belong to one of the suspended
providers."""
+ return {url for url in doc_urls if not any(package in url for package in
suspended_packages)}
+
+
+def remove_suspended_import_errors(errors: Collection[str],
suspended_packages: Collection[str]) -> list[str]:
+ """Drop import errors raised for the modules of one of the suspended
providers."""
+ suspended_modules = {
+ package.replace("apache-", "", 1).replace("-", ".") for package in
suspended_packages
+ }
+ return [
+ error
+ for error in errors
+ if not any(f"No module named '{module}'" in error for module in
suspended_modules)
+ ]
diff --git a/scripts/in_container/run_provider_yaml_files_check.py
b/scripts/in_container/run_provider_yaml_files_check.py
index 1b95b1aa888..4ad360c9939 100755
--- a/scripts/in_container/run_provider_yaml_files_check.py
+++ b/scripts/in_container/run_provider_yaml_files_check.py
@@ -43,6 +43,7 @@ from in_container_utils import (
AIRFLOW_ROOT_PATH,
)
from jsonpath_ng.ext import parse
+from provider_suspension import remove_suspended_doc_urls,
remove_suspended_import_errors
from rich.console import Console
from tabulate import tabulate
@@ -1048,13 +1049,7 @@ def check_doc_files(yaml_files: dict[str, dict]) ->
tuple[int, int]:
if f.name != "index.rst" and "_partials" not in f.parts and f.parts[2]
== "docs"
}
- if suspended_providers:
- expected_doc_urls = {
- doc_url
- for doc_url in expected_doc_urls
- for suspend_provider in suspended_providers
- if suspend_provider not in doc_url
- }
+ expected_doc_urls = remove_suspended_doc_urls(expected_doc_urls,
suspended_providers)
if suspended_logos:
console.print("[yellow]Suspended logos:[/]")
@@ -1218,15 +1213,8 @@ if __name__ == "__main__":
check_invalid_integration(all_parsed_yaml_files)
check_providers_are_mentioned_in_issue_template(all_parsed_yaml_files)
- # remove errors related to suspended module imports.
print("suspended_providers ", suspended_providers)
- if suspended_providers and errors:
- errors = [
- error
- for error in errors
- for module in suspended_providers
- if f"No module named '{module.replace('apache-', '',
1).replace('-', '.')}'" not in error
- ]
+ errors = remove_suspended_import_errors(errors, suspended_providers)
if errors:
error_num = len(errors)
diff --git a/scripts/tests/in_container/test_provider_suspension.py
b/scripts/tests/in_container/test_provider_suspension.py
new file mode 100644
index 00000000000..52bb2b7674f
--- /dev/null
+++ b/scripts/tests/in_container/test_provider_suspension.py
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pytest
+from provider_suspension import remove_suspended_doc_urls,
remove_suspended_import_errors
+
+BEAM_URL = "/docs/apache-airflow-providers-apache-beam/operators.rst"
+YANDEX_URL = "/docs/apache-airflow-providers-yandex/operators/dataproc.rst"
+GOOGLE_URL = "/docs/apache-airflow-providers-google/operators/bigquery.rst"
+ALL_URLS = [BEAM_URL, YANDEX_URL, GOOGLE_URL]
+
+BEAM_ERROR = "No module named 'airflow.providers.apache.beam'"
+YANDEX_ERROR = "No module named 'airflow.providers.yandex'"
+OTHER_ERROR = "Provider google has an invalid integration"
+ALL_ERRORS = [BEAM_ERROR, YANDEX_ERROR, OTHER_ERROR]
+
+
[email protected](
+ "suspended_packages, expected",
+ [
+ pytest.param(set(), ALL_URLS, id="none-suspended"),
+ pytest.param({"apache-airflow-providers-apache-beam"}, [YANDEX_URL,
GOOGLE_URL], id="one-suspended"),
+ pytest.param(
+ {"apache-airflow-providers-apache-beam",
"apache-airflow-providers-yandex"},
+ [GOOGLE_URL],
+ id="several-suspended",
+ ),
+ ],
+)
+def test_remove_suspended_doc_urls(suspended_packages, expected):
+ assert remove_suspended_doc_urls(ALL_URLS, suspended_packages) ==
set(expected)
+
+
[email protected](
+ "suspended_packages, expected",
+ [
+ pytest.param(set(), ALL_ERRORS, id="none-suspended"),
+ pytest.param(
+ {"apache-airflow-providers-apache-beam"}, [YANDEX_ERROR,
OTHER_ERROR], id="one-suspended"
+ ),
+ pytest.param(
+ {"apache-airflow-providers-apache-beam",
"apache-airflow-providers-yandex"},
+ [OTHER_ERROR],
+ id="several-suspended",
+ ),
+ ],
+)
+def test_remove_suspended_import_errors(suspended_packages, expected):
+ assert remove_suspended_import_errors(ALL_ERRORS, suspended_packages) ==
expected
