This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8d311c7d874 Better handling missing airflow package doc inventories
(#54464)
8d311c7d874 is described below
commit 8d311c7d874eedd8c4d82ee5865be4058a3b3cc0
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Aug 13 16:54:12 2025 +0200
Better handling missing airflow package doc inventories (#54464)
When airflow package inventories cannot be downloaded (for example
because they were deleted) we can always rebuild them locally from
sources. This might happen for example when our "main" developer
documentation published in `apache-airflow-docs` is accidentally
or purposefully deleted (for example to refresh it from scratch).
This can also happen when we have a new provider and it has never
been published, or when we add new distribution that has not yet
had a documentation built (such as task-sdk, or airflowctl in
the future).
This PR updates the existing mechanism to:
* not only handle missing "apache-airflow-" packages but also the
other airflow packages that do not follow this naming convention
* adds better diagnostics so that if we miss to add a distribution
in the future we know what to do
---
.../sphinx_exts/docs_build/fetch_inventories.py | 41 ++++++++++++++++++----
1 file changed, 34 insertions(+), 7 deletions(-)
diff --git a/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
b/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
index 7975b81f6a1..ca2560662e6 100644
--- a/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
+++ b/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
@@ -99,6 +99,14 @@ def should_be_refreshed(pkg_name: str,
refresh_airflow_inventories: bool) -> boo
return False
+def is_airflow_package(pkg_name: str) -> bool:
+ """
+ Check if the package name is an Airflow package.
+ This includes the main Airflow package and any provider packages.
+ """
+ return pkg_name.startswith("apache-airflow") or pkg_name in ["helm-chart",
"docker-stack", "task-sdk"]
+
+
def fetch_inventories(clean_build: bool, refresh_airflow_inventories: bool =
False) -> list[str]:
"""Fetch all inventories for Airflow documentation packages and store in
cache."""
if clean_build:
@@ -166,20 +174,39 @@ def fetch_inventories(clean_build: bool,
refresh_airflow_inventories: bool = Fal
print(f"Result: {len(success)} success, {len(failed)} failed")
if failed:
terminate = False
+ missing_airflow_packages = False
print("Failed packages:")
for pkg_no, (pkg_name, _) in enumerate(failed, start=1):
print(f"{pkg_no}. {pkg_name}")
- if not terminate and not pkg_name.startswith("apache-airflow"):
- # For solve situation that newly created Community Provider
doesn't upload inventory yet.
- # And we terminate execution only if any error happen during
fetching
- # third party intersphinx inventories.
+ if not is_airflow_package(pkg_name):
+ print(
+ f"[yellow]Missing {pkg_name} inventory is not for an
Airflow package: "
+ f"it will terminate the execution."
+ )
terminate = True
+ else:
+ print(
+ f"[yellow]Missing {pkg_name} inventory is for an Airflow
package, "
+ f"it will be built from sources."
+ )
+ missing_airflow_packages = True
if terminate:
- print("Terminate execution.")
+ print(
+ "[red]Terminate execution[/]\n\n"
+ "[yellow]Some non-airflow inventories are missing. If missing
inventory is really "
+ "an Airflow package, please add it to the `is_airflow_package`
in `fetch_inventories.py`."
+ )
raise SystemExit(1)
-
+ if missing_airflow_packages:
+ print(f"[yellow]Failed to fetch those airflow inventories -
building them first: {failed}.")
return [pkg_name for pkg_name, status in failed]
if __name__ == "__main__":
- fetch_inventories(clean_build=len(sys.argv) > 1 and sys.argv[1] == "clean")
+ if len(sys.argv) > 1 and sys.argv[1] == "clean":
+ print("[bright_blue]Cleaning inventory cache before fetching
inventories")
+ clean_build = True
+ else:
+ print("[bright_blue]Just fetching inventories without cleaning cache")
+ clean_build = False
+ fetch_inventories(clean_build=clean_build)
