This is an automated email from the ASF dual-hosted git repository.
kaxil pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a31a6a340e3 Render docs mermaid diagrams at build time so they show on
airflow.apache.org (#74757)
a31a6a340e3 is described below
commit a31a6a340e37778a815907ac7e0daa3b292a2311
Author: Kaxil Naik <[email protected]>
AuthorDate: Wed Oct 14 22:41:47 2026 +0100
Render docs mermaid diagrams at build time so they show on
airflow.apache.org (#74757)
airflow.apache.org is served with the ASF Content-Security-Policy, whose
script-src does not allow cdn.jsdelivr.net. sphinxcontrib-mermaid's default
output imports mermaid from there, so every mermaid diagram in the docs has
shown as raw source text on the published site since the first one was
added.
Diagrams are now drawn into SVG files by mermaid-cli during the docs build.
mermaid-cli and Chromium are installed into the breeze container only when a
package being built contains a mermaid diagram, so the CI image is
unchanged.
---
devel-common/pyproject.toml | 2 +-
devel-common/src/docs/README.rst | 19 ++++
devel-common/src/docs/build_docs.py | 37 ++++++++
devel-common/src/docs/utils/conf_constants.py | 2 +-
devel-common/src/sphinx_exts/mermaid_config.json | 3 +
.../src/sphinx_exts/mermaid_puppeteer_config.json | 3 +
devel-common/src/sphinx_exts/mermaid_svg.py | 66 +++++++++++++
.../docs_build/test_build_docs_mermaid.py | 104 +++++++++++++++++++++
.../tests/unit/sphinx_exts/test_mermaid_svg.py | 83 ++++++++++++++++
scripts/in_container/install_mermaid_cli.sh | 40 ++++++++
task-sdk/docs/conf.py | 2 +-
uv.lock | 2 +-
12 files changed, 359 insertions(+), 4 deletions(-)
diff --git a/devel-common/pyproject.toml b/devel-common/pyproject.toml
index f40755d7ddc..c8873085d2d 100644
--- a/devel-common/pyproject.toml
+++ b/devel-common/pyproject.toml
@@ -96,7 +96,7 @@ dependencies = [
"sphinxcontrib-httpdomain>=1.8.1",
"sphinxcontrib-jquery>=4.1",
"sphinxcontrib-jsmath>=1.0.1",
- "sphinxcontrib-mermaid>=1.0.0",
+ "sphinxcontrib-mermaid>=2.1.0",
"sphinxcontrib-qthelp>=1.0.3",
"sphinxcontrib-serializinghtml>=1.1.5",
"sphinxcontrib-spelling>=8.0.0",
diff --git a/devel-common/src/docs/README.rst b/devel-common/src/docs/README.rst
index 2d021da3af2..af5c68973af 100644
--- a/devel-common/src/docs/README.rst
+++ b/devel-common/src/docs/README.rst
@@ -255,6 +255,25 @@ In this example error, You should change the line with
error so that the whole p
Incorrect Spelling: 'BaseAsyncSessionFactory'
Line with Error: '
airflow.providers.amazon.aws.hook.base_aws.BaseAsyncSessionFactory has been
deprecated'
+mermaid diagram fails to render
+*******************************
+
+The ``.. mermaid::`` diagrams are drawn into SVG files during the docs build
by mermaid-cli (``mmdc``).
+They are not drawn in the reader's browser: airflow.apache.org is served with
the
+`ASF Content-Security-Policy <https://infra.apache.org/csp.html>`__, which
blocks the mermaid script the
+browser would otherwise load from a CDN.
+
+When a package you build contains a diagram, ``breeze build-docs`` first
installs mermaid-cli and the
+Chromium it drives into its container. Packages without diagrams skip this
step.
+
+A diagram with a syntax error fails the build with ``Mermaid exited with
error`` followed by mermaid's
+own message, which names the line it could not parse. If you build the docs
outside breeze without
+``mmdc`` on your ``PATH``, every diagram fails the build with:
+
+.. code-block:: text
+
+ WARNING: command 'mmdc' cannot be run (needed for mermaid output), check
the mermaid_cmd setting
+
Support
========
diff --git a/devel-common/src/docs/build_docs.py
b/devel-common/src/docs/build_docs.py
index 0683e166b11..66bc8311ce2 100755
--- a/devel-common/src/docs/build_docs.py
+++ b/devel-common/src/docs/build_docs.py
@@ -22,7 +22,9 @@ from __future__ import annotations
import itertools
import multiprocessing
import os
+import re
import shutil
+import subprocess
import sys
from collections import defaultdict
from collections.abc import Callable, Iterable, Sequence
@@ -74,6 +76,10 @@ ERRORS_ELIGIBLE_TO_REBUILD = [
ON_GITHUB_ACTIONS = os.environ.get("GITHUB_ACTIONS", "false") == "true"
+MERMAID_DIRECTIVE = re.compile(r"^\s*\.\. mermaid::", re.MULTILINE)
+INSTALL_MERMAID_CLI_SCRIPT = AIRFLOW_CONTENT_ROOT_PATH / "scripts" /
"in_container" / "install_mermaid_cli.sh"
+CHROMIUM_PATH = Path("/usr/bin/chromium")
+
console = Console(force_terminal=True, color_system="standard",
width=CONSOLE_WIDTH)
T = TypeVar("T")
@@ -324,6 +330,36 @@ def is_command_available(command):
return shutil.which(command) is not None
+def _uses_mermaid(package_name: str) -> bool:
+ src_dir = AirflowDocsBuilder(package_name=package_name)._src_dir
+ return any(MERMAID_DIRECTIVE.search(rst.read_text()) for rst in
src_dir.rglob("*.rst"))
+
+
+def ensure_mermaid_cli(packages_to_build: Iterable[str]) -> None:
+ """
+ Make mermaid-cli available when a package to build contains a ``..
mermaid::`` diagram.
+
+ The diagrams are drawn at build time (see ``sphinx_exts/mermaid_svg.py``).
mermaid-cli and
+ Chromium are installed into the breeze container on demand rather than
baked into the CI image,
+ which every CI job loads, because only a handful of packages have diagrams.
+ """
+ if not any(_uses_mermaid(package_name) for package_name in
packages_to_build):
+ return
+ if not is_command_available("mmdc"):
+ if os.environ.get("BREEZE", "").lower() != "true":
+ console.print(
+ "[yellow]A package to build contains mermaid diagrams, and
mermaid-cli (mmdc) is not "
+ "installed. Run `breeze build-docs`, or install it with "
+ "`npm install --global @mermaid-js/mermaid-cli`.[/]"
+ )
+ return
+ with with_group("Installing mermaid-cli to draw mermaid diagrams"):
+ subprocess.run([INSTALL_MERMAID_CLI_SCRIPT.as_posix()], check=True)
+ if CHROMIUM_PATH.exists():
+ # Point puppeteer at Debian's chromium; mermaid-cli was installed
without puppeteer's own.
+ os.environ.setdefault("PUPPETEER_EXECUTABLE_PATH",
CHROMIUM_PATH.as_posix())
+
+
click.rich_click.OPTION_GROUPS = {
"build-docs": [
{
@@ -480,6 +516,7 @@ def build_docs(
shutil.rmtree(api_dir)
except StopIteration:
pass
+ ensure_mermaid_cli(packages_to_build)
with with_group("Fetching inventories"):
# Inventories that could not be retrieved should be built first. This
may mean this is a
# new package.
diff --git a/devel-common/src/docs/utils/conf_constants.py
b/devel-common/src/docs/utils/conf_constants.py
index 98918055653..fffdadea96a 100644
--- a/devel-common/src/docs/utils/conf_constants.py
+++ b/devel-common/src/docs/utils/conf_constants.py
@@ -88,7 +88,7 @@ BASIC_SPHINX_EXTENSIONS = [
"airflow_intersphinx",
"python_builtin_xrefs",
"common_compat_alias",
- "sphinxcontrib.mermaid",
+ "mermaid_svg",
"airflow_spelling",
"sphinx_airflow_theme",
"redirects",
diff --git a/devel-common/src/sphinx_exts/mermaid_config.json
b/devel-common/src/sphinx_exts/mermaid_config.json
new file mode 100644
index 00000000000..168e10a9940
--- /dev/null
+++ b/devel-common/src/sphinx_exts/mermaid_config.json
@@ -0,0 +1,3 @@
+{
+ "fontFamily": "arial, \"Liberation Sans\", helvetica, sans-serif"
+}
diff --git a/devel-common/src/sphinx_exts/mermaid_puppeteer_config.json
b/devel-common/src/sphinx_exts/mermaid_puppeteer_config.json
new file mode 100644
index 00000000000..3201af7b736
--- /dev/null
+++ b/devel-common/src/sphinx_exts/mermaid_puppeteer_config.json
@@ -0,0 +1,3 @@
+{
+ "args": ["--no-sandbox"]
+}
diff --git a/devel-common/src/sphinx_exts/mermaid_svg.py
b/devel-common/src/sphinx_exts/mermaid_svg.py
new file mode 100644
index 00000000000..eb29593bf97
--- /dev/null
+++ b/devel-common/src/sphinx_exts/mermaid_svg.py
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Draw ``.. mermaid::`` diagrams into SVG files at build time instead of in the
reader's browser.
+
+sphinxcontrib-mermaid's default ``raw`` output puts the diagram source on the
page together with a
+script that imports mermaid from cdn.jsdelivr.net. airflow.apache.org is
served with the ASF
+Content-Security-Policy (https://infra.apache.org/csp.html), whose
``script-src`` does not allow
+that host, so the import is blocked and readers see the diagram source as
text. Rendered at build
+time by mermaid-cli (``mmdc``), each diagram is an SVG file served from the
site itself, with no
+script involved.
+
+``breeze build-docs`` installs ``mmdc`` and the Chromium it drives into its
container when a package
+it builds has a diagram (``scripts/in_container/install_mermaid_cli.sh``).
When ``mmdc`` is missing or
+fails, sphinxcontrib-mermaid logs a warning naming the diagram, and the
warning fails the docs build.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from sphinx.application import Sphinx
+ from sphinx.config import Config
+
+# mermaid sizes each label's box from the font it measures in the build
container, and the reader's
+# browser draws the text in whatever font it has. With mermaid's default
"trebuchet ms" first in the
+# list, the two differ and labels are clipped. Arial is on macOS and Windows,
and the
+# fonts-liberation package installed with mermaid-cli gives the container
Liberation Sans, which has
+# the same metrics as Arial, so the boxes fit the text the reader sees.
+MERMAID_CONFIG_PATH = Path(__file__).parent / "mermaid_config.json"
+
+# The docs build runs as root inside breeze, and Chromium refuses to start as
root with its sandbox
+# on. The container is the isolation boundary, and the diagrams come from this
repository.
+PUPPETEER_CONFIG_PATH = Path(__file__).parent / "mermaid_puppeteer_config.json"
+
+
+def _render_at_build_time(app: Sphinx, config: Config) -> None:
+ config.mermaid_output_format = "svg"
+ config.mermaid_params = [
+ "--configFile",
+ MERMAID_CONFIG_PATH.as_posix(),
+ "--puppeteerConfigFile",
+ PUPPETEER_CONFIG_PATH.as_posix(),
+ ]
+
+
+def setup(app: Sphinx) -> dict[str, bool]:
+ app.setup_extension("sphinxcontrib.mermaid")
+ app.connect("config-inited", _render_at_build_time)
+ return {"parallel_read_safe": True, "parallel_write_safe": True}
diff --git
a/devel-common/tests/unit/sphinx_exts/docs_build/test_build_docs_mermaid.py
b/devel-common/tests/unit/sphinx_exts/docs_build/test_build_docs_mermaid.py
new file mode 100644
index 00000000000..c6c8e24557b
--- /dev/null
+++ b/devel-common/tests/unit/sphinx_exts/docs_build/test_build_docs_mermaid.py
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import os
+import subprocess
+from unittest import mock
+
+import pytest
+from rich.console import Console
+
+from docs import build_docs
+
+
[email protected](autouse=True)
+def isolated_environ():
+ with mock.patch.dict(os.environ):
+ os.environ.pop("PUPPETEER_EXECUTABLE_PATH", None)
+ yield
+
+
[email protected]
+def chromium_path(tmp_path, monkeypatch):
+ path = tmp_path / "chromium"
+ path.touch()
+ monkeypatch.setattr(build_docs, "CHROMIUM_PATH", path)
+ return path
+
+
+def test_uses_mermaid_finds_diagrams_only_in_packages_that_have_them():
+ assert build_docs._uses_mermaid("apache-airflow-providers-common-ai")
+ assert not build_docs._uses_mermaid("apache-airflow-providers-sqlite")
+
+
[email protected](subprocess, "run", autospec=True)
[email protected](build_docs, "is_command_available", autospec=True,
return_value=False)
[email protected](build_docs, "_uses_mermaid", autospec=True,
return_value=False)
+def test_ensure_mermaid_cli_skips_install_without_diagrams(
+ mock_uses_mermaid, mock_is_command_available, mock_run, monkeypatch
+):
+ monkeypatch.setenv("BREEZE", "true")
+
+ build_docs.ensure_mermaid_cli(["apache-airflow-providers-sqlite"])
+
+ mock_run.assert_not_called()
+
+
[email protected](subprocess, "run", autospec=True)
[email protected](build_docs, "is_command_available", autospec=True,
return_value=False)
[email protected](build_docs, "_uses_mermaid", autospec=True,
return_value=True)
+def test_ensure_mermaid_cli_installs_in_breeze_when_mmdc_missing(
+ mock_uses_mermaid, mock_is_command_available, mock_run, monkeypatch,
chromium_path
+):
+ monkeypatch.setenv("BREEZE", "true")
+
+ build_docs.ensure_mermaid_cli(["apache-airflow-providers-common-ai"])
+
+
mock_run.assert_called_once_with([build_docs.INSTALL_MERMAID_CLI_SCRIPT.as_posix()],
check=True)
+ assert os.environ["PUPPETEER_EXECUTABLE_PATH"] == chromium_path.as_posix()
+
+
[email protected](subprocess, "run", autospec=True)
[email protected](build_docs, "is_command_available", autospec=True,
return_value=True)
[email protected](build_docs, "_uses_mermaid", autospec=True,
return_value=True)
+def test_ensure_mermaid_cli_does_not_reinstall_when_mmdc_present(
+ mock_uses_mermaid, mock_is_command_available, mock_run, monkeypatch,
chromium_path
+):
+ monkeypatch.setenv("BREEZE", "true")
+
+ build_docs.ensure_mermaid_cli(["apache-airflow-providers-common-ai"])
+
+ mock_run.assert_not_called()
+ assert os.environ["PUPPETEER_EXECUTABLE_PATH"] == chromium_path.as_posix()
+
+
[email protected](subprocess, "run", autospec=True)
[email protected](build_docs, "is_command_available", autospec=True,
return_value=False)
[email protected](build_docs, "_uses_mermaid", autospec=True,
return_value=True)
+def test_ensure_mermaid_cli_does_not_install_outside_breeze(
+ mock_uses_mermaid, mock_is_command_available, mock_run, monkeypatch
+):
+ monkeypatch.delenv("BREEZE", raising=False)
+ mock_console = mock.create_autospec(Console, instance=True)
+ monkeypatch.setattr(build_docs, "console", mock_console)
+
+ build_docs.ensure_mermaid_cli(["apache-airflow-providers-common-ai"])
+
+ mock_run.assert_not_called()
+ (message,), _ = mock_console.print.call_args
+ assert "npm install --global @mermaid-js/mermaid-cli" in message
diff --git a/devel-common/tests/unit/sphinx_exts/test_mermaid_svg.py
b/devel-common/tests/unit/sphinx_exts/test_mermaid_svg.py
new file mode 100644
index 00000000000..6bb78eb5de8
--- /dev/null
+++ b/devel-common/tests/unit/sphinx_exts/test_mermaid_svg.py
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import io
+import json
+import shlex
+import sys
+from pathlib import Path
+
+from sphinx.application import Sphinx
+
+SPHINX_EXTS_PATH = Path(__file__).parents[3] / "src" / "sphinx_exts"
+if SPHINX_EXTS_PATH.as_posix() not in sys.path:
+ # The extensions are loaded by Sphinx from this directory and import each
other by bare name.
+ sys.path.append(SPHINX_EXTS_PATH.as_posix())
+
+INDEX_RST = """\
+Index
+=====
+
+.. mermaid::
+
+ flowchart LR
+ a --> b
+"""
+
+# Stands in for mermaid-cli: records the arguments it was given and writes an
SVG to the -o path.
+FAKE_MMDC = """\
+import json
+import sys
+from pathlib import Path
+
+args = sys.argv[1:]
+Path(sys.argv[0]).with_name("mmdc-args.json").write_text(json.dumps(args))
+Path(args[args.index("-o") + 1]).write_text('<svg
xmlns="http://www.w3.org/2000/svg";></svg>')
+"""
+
+
+def test_mermaid_diagram_is_drawn_at_build_time_without_cdn_script(tmp_path):
+ src = tmp_path / "src"
+ src.mkdir()
+ (src / "index.rst").write_text(INDEX_RST)
+ (src / "conf.py").write_text("extensions = ['mermaid_svg']\n")
+ fake_mmdc = tmp_path / "fake_mmdc.py"
+ fake_mmdc.write_text(FAKE_MMDC)
+ out = tmp_path / "out"
+ warnings = io.StringIO()
+
+ app = Sphinx(
+ srcdir=src,
+ confdir=src,
+ outdir=out,
+ doctreedir=tmp_path / "doctrees",
+ buildername="html",
+ confoverrides={"mermaid_cmd": shlex.join([sys.executable,
fake_mmdc.as_posix()])},
+ status=None,
+ warning=warnings,
+ )
+ app.build()
+
+ html = (out / "index.html").read_text()
+ mmdc_args = json.loads((tmp_path / "mmdc-args.json").read_text())
+ assert warnings.getvalue() == ""
+ assert '<object data="_images/mermaid-' in html
+ assert "cdn.jsdelivr.net" not in html
+ assert 'class="mermaid"' not in html
+ assert mmdc_args[mmdc_args.index("--configFile") +
1].endswith("mermaid_config.json")
+ assert mmdc_args[mmdc_args.index("--puppeteerConfigFile") +
1].endswith("mermaid_puppeteer_config.json")
diff --git a/scripts/in_container/install_mermaid_cli.sh
b/scripts/in_container/install_mermaid_cli.sh
new file mode 100755
index 00000000000..c4b9518ee9f
--- /dev/null
+++ b/scripts/in_container/install_mermaid_cli.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Installs mermaid-cli (mmdc) and the Chromium it drives into the running CI
image, so the docs
+# build can draw ``.. mermaid::`` diagrams into SVG files (see
devel-common/src/sphinx_exts/mermaid_svg.py).
+#
+# The docs build calls this only when a package it builds contains a mermaid
diagram. Together the
+# two take about 1 GB on disk, which would otherwise be in the CI image that
every CI job loads.
+set -euo pipefail
+
+MERMAID_CLI_VERSION="11.17.0"
+
+# Debian's chromium is used instead of the browser puppeteer would download.
fonts-liberation gives
+# mermaid Liberation Sans, which has Arial's metrics, to measure its labels
with.
+apt-get update -qq
+apt-get install -y -qq --no-install-recommends chromium fonts-liberation >
/dev/null
+rm -rf /var/lib/apt/lists/*
+
+# The same supply-chain rules ts-sdk applies through pnpm: no dependency
lifecycle scripts, and no
+# transitive dependency published less than 14 days ago.
+npm install --global --ignore-scripts --no-fund --no-audit --loglevel=error \
+ --before "$(date -u -d '14 days ago' +%Y-%m-%dT%H:%M:%SZ)" \
+ "@mermaid-js/mermaid-cli@${MERMAID_CLI_VERSION}"
+
+echo "Installed $(mmdc --version | head -1) and $(chromium --version)"
diff --git a/task-sdk/docs/conf.py b/task-sdk/docs/conf.py
index 3b9c6f17eee..25e16dbb7d2 100644
--- a/task-sdk/docs/conf.py
+++ b/task-sdk/docs/conf.py
@@ -57,7 +57,7 @@ extensions = [
"sphinx.ext.intersphinx",
"airflow_intersphinx",
"exampleinclude",
- "sphinxcontrib.mermaid",
+ "mermaid_svg",
"airflow_spelling",
]
diff --git a/uv.lock b/uv.lock
index 79c86d58b35..30394db8f29 100644
--- a/uv.lock
+++ b/uv.lock
@@ -2707,7 +2707,7 @@ requires-dist = [
{ name = "sphinxcontrib-httpdomain", marker = "extra == 'docs'", specifier
= ">=1.8.1" },
{ name = "sphinxcontrib-jquery", marker = "extra == 'docs'", specifier =
">=4.1" },
{ name = "sphinxcontrib-jsmath", marker = "extra == 'docs'", specifier =
">=1.0.1" },
- { name = "sphinxcontrib-mermaid", marker = "extra == 'docs'", specifier =
">=1.0.0" },
+ { name = "sphinxcontrib-mermaid", marker = "extra == 'docs'", specifier =
">=2.1.0" },
{ name = "sphinxcontrib-qthelp", marker = "extra == 'docs'", specifier =
">=1.0.3" },
{ name = "sphinxcontrib-serializinghtml", marker = "extra == 'docs'",
specifier = ">=1.1.5" },
{ name = "sphinxcontrib-spelling", marker = "extra == 'docs'", specifier =
">=8.0.0" },
