This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new afda438816b Warn instead of failing on missing 3rd-party doc
inventories (#63630)
afda438816b is described below
commit afda438816b8e8cd43ef105630c8f33da8ec98b5
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Mar 15 14:32:45 2026 +0100
Warn instead of failing on missing 3rd-party doc inventories (#63630)
* Warn instead of failing on missing 3rd-party doc inventories
Third-party Sphinx intersphinx inventories (e.g., Pandas) are sometimes
temporarily unavailable. Previously, any download failure terminated the
entire doc build. Now missing 3rd-party inventories produce warnings and
fall back to cached versions when available. A marker file is written for
CI to detect missing inventories and send Slack notifications on canary
builds. Publishing workflows fail by default but can opt out.
- Add --fail-on-missing-third-party-inventories flag (default: off)
- Add --clean-inventory-cache flag (--clean-build no longer deletes cache)
- Cache inventories via stash action in CI and publish workflows
- Send Slack warning on canary builds when inventories are missing
* Add documentation for inventory cache handling options
Document the new --clean-inventory-cache,
--fail-on-missing-third-party-inventories,
and --ignore-missing-inventories flags in the contributing docs, Breeze
developer
tasks, and release management docs.
* Skip missing third-party inventories in intersphinx mapping
When a third-party inventory file doesn't exist in the cache,
skip it from the Sphinx intersphinx_mapping instead of referencing
a non-existent file. This prevents Sphinx build errors when
third-party inventory downloads fail.
---
.github/workflows/ci-amd-arm.yml | 1 +
.github/workflows/ci-image-checks.yml | 40 ++++++++++++++++--
.github/workflows/publish-docs-to-s3.yml | 22 +++++++++-
contributing-docs/11_documentation_building.rst | 25 +++++++++++-
dev/breeze/doc/03_developer_tasks.rst | 13 ++++++
dev/breeze/doc/09_release_management_tasks.rst | 4 ++
dev/breeze/doc/images/output_build-docs.svg | 36 ++++++++++++-----
dev/breeze/doc/images/output_build-docs.txt | 2 +-
.../images/output_workflow-run_publish-docs.svg | 22 +++++++---
.../images/output_workflow-run_publish-docs.txt | 2 +-
.../airflow_breeze/commands/developer_commands.py | 26 ++++++++++--
.../commands/developer_commands_config.py | 7 +++-
.../airflow_breeze/commands/workflow_commands.py | 7 ++++
.../commands/workflow_commands_config.py | 6 +++
.../src/airflow_breeze/params/doc_build_params.py | 6 +++
devel-common/src/docs/build_docs.py | 32 ++++++++++++---
devel-common/src/docs/utils/conf_constants.py | 31 +++++++++-----
.../sphinx_exts/docs_build/fetch_inventories.py | 47 ++++++++++++++++------
18 files changed, 274 insertions(+), 55 deletions(-)
diff --git a/.github/workflows/ci-amd-arm.yml b/.github/workflows/ci-amd-arm.yml
index 9e509925317..ed82047dd0c 100644
--- a/.github/workflows/ci-amd-arm.yml
+++ b/.github/workflows/ci-amd-arm.yml
@@ -328,6 +328,7 @@ jobs:
secrets:
DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
+ SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
providers:
name: "provider distributions tests"
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index de3d48ca581..a3eb7caf84f 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -117,6 +117,8 @@ on: # yamllint disable-line rule:truthy
required: true
DOCS_AWS_SECRET_ACCESS_KEY:
required: true
+ SLACK_BOT_TOKEN:
+ required: false
permissions:
@@ -252,23 +254,55 @@ jobs:
uses:
apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
path: ./generated/_inventory_cache/
- key: cache-docs-inventory-v1-${{ hashFiles('**/pyproject.toml') }}
+ key: cache-docs-inventory-v1
id: restore-docs-inventory-cache
- name: "Building docs with ${{ matrix.flag }} flag"
env:
DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }}
run: >
breeze build-docs ${DOCS_LIST_AS_STRING} ${{ matrix.flag }}
--refresh-airflow-inventories
+ - name: "Check for missing third-party inventories"
+ id: check-missing-inventories
+ if: always()
+ shell: bash
+ run: |
+
MARKER_FILE="./generated/_inventory_cache/.missing_third_party_inventories"
+ if [[ -f "${MARKER_FILE}" ]]; then
+ echo "missing=true" >> "${GITHUB_OUTPUT}"
+ echo "::warning::Missing third-party inventories:"
+ cat "${MARKER_FILE}"
+ echo "packages<<EOF" >> "${GITHUB_OUTPUT}"
+ cat "${MARKER_FILE}" >> "${GITHUB_OUTPUT}"
+ echo "EOF" >> "${GITHUB_OUTPUT}"
+ else
+ echo "missing=false" >> "${GITHUB_OUTPUT}"
+ fi
+ - name: "Notify Slack about missing inventories (canary only)"
+ if: >-
+ inputs.canary-run == 'true' &&
+ steps.check-missing-inventories.outputs.missing == 'true' &&
+ matrix.flag == '--docs-only'
+ uses:
slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+ with:
+ method: chat.postMessage
+ token: ${{ env.SLACK_BOT_TOKEN }}
+ # yamllint disable rule:line-length
+ payload: |
+ channel: "internal-airflow-ci-cd"
+ text: "⚠️ Missing 3rd-party doc inventories in canary build on
*${{ github.ref_name }}*\n\nPackages:\n${{
steps.check-missing-inventories.outputs.packages }}\n\n<https://github.com/${{
github.repository }}/actions/runs/${{ github.run_id }}|View build log>"
+ # yamllint enable rule:line-length
+ env:
+ SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
- name: "Save docs inventory cache"
uses:
apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
path: ./generated/_inventory_cache/
- key: cache-docs-inventory-v1-${{ hashFiles('**/pyproject.toml') }}
+ key: cache-docs-inventory-v1
if-no-files-found: 'error'
retention-days: '2'
# If we upload from multiple matrix jobs we could end up with a race
condition. so just pick one job
# to be responsible for updating it.
https://github.com/actions/upload-artifact/issues/506
- if: steps.restore-docs-inventory-cache != 'true' && matrix.flag ==
'--docs-only'
+ if: steps.restore-docs-inventory-cache.outputs.stash-hit != 'true' &&
matrix.flag == '--docs-only'
- name: "Upload build docs"
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
# v7.0.0
with:
diff --git a/.github/workflows/publish-docs-to-s3.yml
b/.github/workflows/publish-docs-to-s3.yml
index f70995ef5d3..0fec8dc9410 100644
--- a/.github/workflows/publish-docs-to-s3.yml
+++ b/.github/workflows/publish-docs-to-s3.yml
@@ -64,6 +64,11 @@ on: # yamllint disable-line rule:truthy
required: false
description: "Optionally apply commit hashes before building - to
patch the docs (coma separated)"
type: string
+ ignore-missing-inventories:
+ required: false
+ description: "Do not fail the build on missing third-party inventories"
+ default: false
+ type: boolean
permissions:
contents: read
jobs:
@@ -253,12 +258,27 @@ jobs:
--build-arg VERSION_SUFFIX_FOR_PYPI=dev0
-t ghcr.io/apache/airflow/main/ci/python3.9:latest --target main .
-f Dockerfile.ci --platform linux/amd64
+ - name: "Restore docs inventory cache"
+ uses:
apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
+ with:
+ path: ./generated/_inventory_cache/
+ key: cache-docs-inventory-v1
+ id: restore-docs-inventory-cache
- name: "Building docs with --docs-only flag using ${{ inputs.ref }}
reference breeze"
env:
INCLUDE_DOCS: ${{ needs.build-info.outputs.include-docs }}
INCLUDE_COMMITS: ${{ startsWith(inputs.ref, 'providers') && 'true'
|| 'false' }}
+ FAIL_ON_INVENTORIES: ${{ inputs.ignore-missing-inventories != true
&& '--fail-on-missing-third-party-inventories' || '' }}
run: >
- breeze build-docs ${INCLUDE_DOCS} --docs-only
+ breeze build-docs ${INCLUDE_DOCS} --docs-only ${FAIL_ON_INVENTORIES}
+ - name: "Save docs inventory cache"
+ uses:
apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
+ if: steps.restore-docs-inventory-cache.outputs.stash-hit != 'true'
+ with:
+ path: ./generated/_inventory_cache/
+ key: cache-docs-inventory-v1
+ if-no-files-found: 'error'
+ retention-days: '2'
- name: "Store stable versions"
run: uv run /tmp/store_stable_versions.py
- name: "Saving build docs folder"
diff --git a/contributing-docs/11_documentation_building.rst
b/contributing-docs/11_documentation_building.rst
index b7dfe3abf52..7ae771fd212 100644
--- a/contributing-docs/11_documentation_building.rst
+++ b/contributing-docs/11_documentation_building.rst
@@ -239,7 +239,30 @@ For example:
breeze build-docs --doc-only --clean fab
-Will build ``fab`` provider documentation and clean inventories and other
build artifacts before.
+Will build ``fab`` provider documentation and clean build artifacts before.
+
+Inventory cache handling
+........................
+
+When building documentation, Sphinx downloads intersphinx inventories from
external sources (both Airflow
+packages hosted on S3 and third-party packages like Pandas, SQLAlchemy, etc.).
These inventories enable
+cross-references between documentation sets.
+
+By default, missing third-party inventories produce warnings but do **not**
fail the build. This is
+because third-party inventory servers can be temporarily unavailable and
should not block documentation
+builds. If a cached version of the inventory exists, it will be used instead.
+
+The following flags control inventory behavior:
+
+- ``--clean-inventory-cache`` — deletes the inventory cache before fetching.
Use this when you want
+ to force a completely fresh download of all inventories.
+- ``--clean-build`` — cleans build artifacts (``_build``, ``_doctrees``,
``apis``) but does **not**
+ delete the inventory cache. This allows rebuilding docs from scratch while
preserving cached
+ inventories.
+- ``--refresh-airflow-inventories`` — forces a refresh of only Airflow package
inventories, without
+ cleaning build artifacts or external inventories.
+- ``--fail-on-missing-third-party-inventories`` — fails the build if any
third-party inventory cannot
+ be downloaded (useful for publishing workflows where complete
cross-references are important).
You can also use ``breeze build-docs --help`` to see available options and
head to
`breeze documentation <../dev/breeze/doc/03_developer_tasks.rst>`__ to learn
more about the ``breeze``
diff --git a/dev/breeze/doc/03_developer_tasks.rst
b/dev/breeze/doc/03_developer_tasks.rst
index 0e0cbd7388e..0db3d77fed2 100644
--- a/dev/breeze/doc/03_developer_tasks.rst
+++ b/dev/breeze/doc/03_developer_tasks.rst
@@ -278,6 +278,19 @@ package names and can be used to select more than one
package with single filter
breeze build-docs --package-filter apache-airflow-providers-*
+Inventory cache handling
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+When building documentation, Sphinx downloads intersphinx inventories to
enable cross-references
+between documentation sets. By default, missing third-party inventories (e.g.,
Pandas, SQLAlchemy)
+produce warnings but do **not** fail the build — third-party servers can be
temporarily unavailable.
+If a cached version exists, it will be used with a warning.
+
+Use ``--clean-inventory-cache`` to force a fresh download of all inventories,
or
+``--fail-on-missing-third-party-inventories`` to fail the build when any
third-party inventory
+is missing (useful for publishing). Note that ``--clean-build`` cleans build
artifacts but
+preserves the inventory cache.
+
Often errors during documentation generation come from the docstrings of
auto-api generated classes.
During the docs building auto-api generated files are stored in the
``generated`` folder. This helps you
easily identify the location the problems with documentation originated from.
diff --git a/dev/breeze/doc/09_release_management_tasks.rst
b/dev/breeze/doc/09_release_management_tasks.rst
index 24f53ea2194..657b523f0bb 100644
--- a/dev/breeze/doc/09_release_management_tasks.rst
+++ b/dev/breeze/doc/09_release_management_tasks.rst
@@ -974,6 +974,10 @@ These are all available flags of ``workflow-run`` command:
``--site-env`` specifies the environment to use for the site (e.g., auto,
live, staging). the default is auto, based on the ref it decides live or
staging.
``--refresh-site`` specifies whether to refresh the site after publishing the
documentation. This triggers workflow on apache/airflow-site repository to
refresh the site.
``--skip-write-to-stable-folder`` specifies the documentation packages to skip
writing to the stable folder.
+``--ignore-missing-inventories`` when set, the publish workflow will not fail
if third-party intersphinx
+inventories cannot be downloaded. By default, the publish workflow fails on
missing inventories to ensure
+complete cross-references in published documentation. Use this flag only when
you need to publish despite
+temporary third-party inventory outages.
These are all available flags of ``workflow-run publish-docs`` command:
diff --git a/dev/breeze/doc/images/output_build-docs.svg
b/dev/breeze/doc/images/output_build-docs.svg
index f81ca88c66d..37bc3a4990d 100644
--- a/dev/breeze/doc/images/output_build-docs.svg
+++ b/dev/breeze/doc/images/output_build-docs.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 1343.1999999999998"
xmlns="http://www.w3.org/2000/svg";>
+<svg class="rich-terminal" viewBox="0 0 1482 1440.8"
xmlns="http://www.w3.org/2000/svg";>
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath id="breeze-build-docs-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="1292.1999999999998" />
+ <rect x="0" y="0" width="1463.0" height="1389.8" />
</clipPath>
<clipPath id="breeze-build-docs-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -201,9 +201,21 @@
<clipPath id="breeze-build-docs-line-51">
<rect x="0" y="1245.9" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-build-docs-line-52">
+ <rect x="0" y="1270.3" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-build-docs-line-53">
+ <rect x="0" y="1294.7" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-build-docs-line-54">
+ <rect x="0" y="1319.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-build-docs-line-55">
+ <rect x="0" y="1343.5" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="1341.2" rx="8"/><text
class="breeze-build-docs-title" fill="#c5c8c6" text-anchor="middle" x="740"
y="27">Command: build-docs</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="1438.8" rx="8"/><text
class="breeze-build-docs-title" fill="#c5c8c6" text-anchor="middle" x="740"
y="27">Command: build-docs</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -238,8 +250,8 @@
</text><text class="breeze-build-docs-r5" x="0" y="556.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-22)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="556.8" textLength="183"
clip-path="url(#breeze-build-docs-line-22)">--one-pass-only</text><text
class="breeze-build-docs-r1" x="231.8" y="556.8" textLength="1000.4"
clip-path="url(#breeze-build-docs-line-22)">Builds documentation in one pass only. This is useful for 
[...]
</text><text class="breeze-build-docs-r5" x="0" y="581.2" textLength="1464"
clip-path="url(#breeze-build-docs-line-23)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="581.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-23)">
</text><text class="breeze-build-docs-r5" x="0" y="605.6" textLength="24.4"
clip-path="url(#breeze-build-docs-line-24)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="605.6" textLength="268.4"
clip-path="url(#breeze-build-docs-line-24)"> Cleaning inventories </text><text
class="breeze-build-docs-r5" x="292.8" y="605.6" textLength="1146.8"
clip-path="url(#breeze-build-docs-line-24)">───────────────────────────────────────────────────────────────────────────────────
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="630" textLength="12.2"
clip-path="url(#breeze-build-docs-line-25)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="630" textLength="353.8"
clip-path="url(#breeze-build-docs-line-25)">--clean-build                </text><text
class="breeze-build-docs-r1" x="402.6" y="630" textLength="1037"
clip-path="url(#breeze-build-docs-line-25)">Cleans the  [...]
-</text><text class="breeze-build-docs-r5" x="0" y="654.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-26)">│</text><text
class="breeze-build-docs-r1" x="402.6" y="654.4" textLength="1037"
clip-path="url(#breeze-build-docs-line-26)">inventory cache (including external inventories).                             
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="630" textLength="12.2"
clip-path="url(#breeze-build-docs-line-25)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="630" textLength="353.8"
clip-path="url(#breeze-build-docs-line-25)">--clean-build                </text><text
class="breeze-build-docs-r1" x="402.6" y="630" textLength="1037"
clip-path="url(#breeze-build-docs-line-25)">Cleans the  [...]
+</text><text class="breeze-build-docs-r5" x="0" y="654.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-26)">│</text><text
class="breeze-build-docs-r1" x="402.6" y="654.4" textLength="256.2"
clip-path="url(#breeze-build-docs-line-26)">inventory cache (use </text><text
class="breeze-build-docs-r4" x="658.8" y="654.4" textLength="280.6"
clip-path="url(#breeze-build-docs-line-26)">--clean-inventory-cache</text><text
class="breeze-build-docs-r1" x="939.4" y="654.4" [...]
</text><text class="breeze-build-docs-r5" x="0" y="678.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-27)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="678.8" textLength="353.8"
clip-path="url(#breeze-build-docs-line-27)">--refresh-airflow-inventories</text><text
class="breeze-build-docs-r1" x="402.6" y="678.8" textLength="1037"
clip-path="url(#breeze-build-docs-line-27)">When set, only airflow package inventories will be ref
[...]
</text><text class="breeze-build-docs-r5" x="0" y="703.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-28)">│</text><text
class="breeze-build-docs-r1" x="402.6" y="703.2" textLength="317.2"
clip-path="url(#breeze-build-docs-line-28)">already downloaded. With `</text><text
class="breeze-build-docs-r4" x="719.8" y="703.2" textLength="158.6"
clip-path="url(#breeze-build-docs-line-28)">--clean-build</text><text
class="breeze-build-docs-r1" x="878.4" y="703.2" textL [...]
</text><text class="breeze-build-docs-r5" x="0" y="727.6" textLength="1464"
clip-path="url(#breeze-build-docs-line-29)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="727.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-29)">
@@ -261,11 +273,15 @@
</text><text class="breeze-build-docs-r5" x="0" y="1118" textLength="12.2"
clip-path="url(#breeze-build-docs-line-45)">│</text><text
class="breeze-build-docs-r1" x="341.6" y="1118" textLength="134.2"
clip-path="url(#breeze-build-docs-line-45)">arguments. </text><text
class="breeze-build-docs-r7" x="475.8" y="1118" textLength="73.2"
clip-path="url(#breeze-build-docs-line-45)">(TEXT)</text><text
class="breeze-build-docs-r5" x="1451.8" y="1118" textLength="12.2"
clip-path="url(#breeze- [...]
</text><text class="breeze-build-docs-r5" x="0" y="1142.4" textLength="1464"
clip-path="url(#breeze-build-docs-line-46)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="1142.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-46)">
</text><text class="breeze-build-docs-r5" x="0" y="1166.8" textLength="24.4"
clip-path="url(#breeze-build-docs-line-47)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="1166.8" textLength="195.2"
clip-path="url(#breeze-build-docs-line-47)"> Common options </text><text
class="breeze-build-docs-r5" x="219.6" y="1166.8" textLength="1220"
clip-path="url(#breeze-build-docs-line-47)">────────────────────────────────────────────────────────────────────────────────────────
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="1191.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-48)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1191.2" textLength="109.8"
clip-path="url(#breeze-build-docs-line-48)">--dry-run</text><text
class="breeze-build-docs-r6" x="158.6" y="1191.2" textLength="24.4"
clip-path="url(#breeze-build-docs-line-48)">-D</text><text
class="breeze-build-docs-r1" x="207.4" y="1191.2" textLength="719.8"
clip-path="url(#breeze-buil [...]
-</text><text class="breeze-build-docs-r5" x="0" y="1215.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-49)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1215.6" textLength="109.8"
clip-path="url(#breeze-build-docs-line-49)">--verbose</text><text
class="breeze-build-docs-r6" x="158.6" y="1215.6" textLength="24.4"
clip-path="url(#breeze-build-docs-line-49)">-v</text><text
class="breeze-build-docs-r1" x="207.4" y="1215.6" textLength="585.6"
clip-path="url(#breeze-buil [...]
-</text><text class="breeze-build-docs-r5" x="0" y="1240" textLength="12.2"
clip-path="url(#breeze-build-docs-line-50)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1240" textLength="109.8"
clip-path="url(#breeze-build-docs-line-50)">--answer </text><text
class="breeze-build-docs-r6" x="158.6" y="1240" textLength="24.4"
clip-path="url(#breeze-build-docs-line-50)">-a</text><text
class="breeze-build-docs-r1" x="207.4" y="1240" textLength="329.4"
clip-path="url(#breeze-build-d [...]
-</text><text class="breeze-build-docs-r5" x="0" y="1264.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-51)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1264.4" textLength="109.8"
clip-path="url(#breeze-build-docs-line-51)">--help   </text><text
class="breeze-build-docs-r6" x="158.6" y="1264.4" textLength="24.4"
clip-path="url(#breeze-build-docs-line-51)">-h</text><text
class="breeze-build-docs-r1" x="207.4" y="1264.4" textLength="329.4"
clip-path="u [...]
-</text><text class="breeze-build-docs-r5" x="0" y="1288.8" textLength="1464"
clip-path="url(#breeze-build-docs-line-52)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="1288.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-52)">
+</text><text class="breeze-build-docs-r5" x="0" y="1191.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-48)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1191.2" textLength="500.2"
clip-path="url(#breeze-build-docs-line-48)">--clean-inventory-cache                  </text><text
class="breeze-build-docs-r1" x="597.8" y="1191.2" textLength="671"
clip-path="url(#breeze-build-docs- [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1215.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-49)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1215.6" textLength="500.2"
clip-path="url(#breeze-build-docs-line-49)">--fail-on-missing-third-party-inventories</text><text
class="breeze-build-docs-r1" x="597.8" y="1215.6" textLength="841.8"
clip-path="url(#breeze-build-docs-line-49)">Fail the build if any third-party inventory 
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="1240" textLength="12.2"
clip-path="url(#breeze-build-docs-line-50)">│</text><text
class="breeze-build-docs-r1" x="597.8" y="1240" textLength="841.8"
clip-path="url(#breeze-build-docs-line-50)">default, missing third-party inventories are warned about but do not </text><text
class="breeze-build-docs-r5" x="1451.8" y="1240" textLength="12.2"
clip-path="url(#breeze-build-docs-line-50)">│</te [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1264.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-51)">│</text><text
class="breeze-build-docs-r1" x="597.8" y="1264.4" textLength="841.8"
clip-path="url(#breeze-build-docs-line-51)">fail the build.                                    
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="1288.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-52)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1288.8" textLength="500.2"
clip-path="url(#breeze-build-docs-line-52)">--dry-run                                </text><text
class="breeze-build-docs-r6" x= [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1313.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-53)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1313.2" textLength="500.2"
clip-path="url(#breeze-build-docs-line-53)">--verbose                                </text><text
class="breeze-build-docs-r6" x= [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1337.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-54)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1337.6" textLength="500.2"
clip-path="url(#breeze-build-docs-line-54)">--answer                                 </text><text
class="breeze-build-docs-r [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1362" textLength="12.2"
clip-path="url(#breeze-build-docs-line-55)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1362" textLength="500.2"
clip-path="url(#breeze-build-docs-line-55)">--help                                   </text><text
class="breeze-build- [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1386.4" textLength="1464"
clip-path="url(#breeze-build-docs-line-56)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="1386.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-56)">
</text>
</g>
</g>
diff --git a/dev/breeze/doc/images/output_build-docs.txt
b/dev/breeze/doc/images/output_build-docs.txt
index abc5a5bb68a..adbeb62222d 100644
--- a/dev/breeze/doc/images/output_build-docs.txt
+++ b/dev/breeze/doc/images/output_build-docs.txt
@@ -1 +1 @@
-6f8557a503649e72517ecd140dadf2f6
+566d00220dbc02e70c18e942a52aceb3
diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg
b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg
index 845a7b6917b..d57016971c4 100644
--- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg
+++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 1050.4"
xmlns="http://www.w3.org/2000/svg";>
+<svg class="rich-terminal" viewBox="0 0 1482 1123.6"
xmlns="http://www.w3.org/2000/svg";>
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -45,7 +45,7 @@
<defs>
<clipPath id="breeze-workflow-run-publish-docs-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="999.4" />
+ <rect x="0" y="0" width="1463.0" height="1072.6" />
</clipPath>
<clipPath id="breeze-workflow-run-publish-docs-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -167,9 +167,18 @@
<clipPath id="breeze-workflow-run-publish-docs-line-39">
<rect x="0" y="953.1" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-workflow-run-publish-docs-line-40">
+ <rect x="0" y="977.5" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-workflow-run-publish-docs-line-41">
+ <rect x="0" y="1001.9" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-workflow-run-publish-docs-line-42">
+ <rect x="0" y="1026.3" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="1048.4" rx="8"/><text
class="breeze-workflow-run-publish-docs-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: workflow-run publish-docs</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="1121.6" rx="8"/><text
class="breeze-workflow-run-publish-docs-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: workflow-run publish-docs</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -217,9 +226,12 @@
</text><text class="breeze-workflow-run-publish-docs-r5" x="0" y="874"
textLength="12.2"
clip-path="url(#breeze-workflow-run-publish-docs-line-35)">│</text><text
class="breeze-workflow-run-publish-docs-r4" x="24.4" y="874" textLength="353.8"
clip-path="url(#breeze-workflow-run-publish-docs-line-35)">--site-env                   </text><text
class="breeze-workflow-run-publish-docs-r1" x="402.6" [...]
</text><text class="breeze-workflow-run-publish-docs-r5" x="0" y="898.4"
textLength="12.2"
clip-path="url(#breeze-workflow-run-publish-docs-line-36)">│</text><text
class="breeze-workflow-run-publish-docs-r4" x="24.4" y="898.4"
textLength="353.8"
clip-path="url(#breeze-workflow-run-publish-docs-line-36)">--skip-write-to-stable-folder</text><text
class="breeze-workflow-run-publish-docs-r1" x="402.6" y="898.4"
textLength="366"
clip-path="url(#breeze-workflow-run-publish-docs-line-36)">Skip& [...]
</text><text class="breeze-workflow-run-publish-docs-r5" x="0" y="922.8"
textLength="1464"
clip-path="url(#breeze-workflow-run-publish-docs-line-37)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-workflow-run-publish-docs-r1" x="1464" y="922.8"
textLength="12.2" clip-path="url(#breeze-workflow-run-publish-docs-line-37)">
-</text><text class="breeze-workflow-run-publish-docs-r5" x="0" y="947.2"
textLength="24.4"
clip-path="url(#breeze-workflow-run-publish-docs-line-38)">╭─</text><text
class="breeze-workflow-run-publish-docs-r5" x="24.4" y="947.2"
textLength="195.2"
clip-path="url(#breeze-workflow-run-publish-docs-line-38)"> Common options </text><text
class="breeze-workflow-run-publish-docs-r5" x="219.6" y="947.2"
textLength="1220" clip-path="url(#breeze-workflow-run-publish-docs-line-38)">─
[...]
-</text><text class="breeze-workflow-run-publish-docs-r5" x="0" y="971.6"
textLength="12.2"
clip-path="url(#breeze-workflow-run-publish-docs-line-39)">│</text><text
class="breeze-workflow-run-publish-docs-r4" x="24.4" y="971.6"
textLength="73.2"
clip-path="url(#breeze-workflow-run-publish-docs-line-39)">--help</text><text
class="breeze-workflow-run-publish-docs-r9" x="122" y="971.6" textLength="24.4"
clip-path="url(#breeze-workflow-run-publish-docs-line-39)">-h</text><text
class="breeze-w [...]
+</text><text class="breeze-workflow-run-publish-docs-r5" x="0" y="947.2"
textLength="24.4"
clip-path="url(#breeze-workflow-run-publish-docs-line-38)">╭─</text><text
class="breeze-workflow-run-publish-docs-r5" x="24.4" y="947.2" textLength="244"
clip-path="url(#breeze-workflow-run-publish-docs-line-38)"> Inventory handling </text><text
class="breeze-workflow-run-publish-docs-r5" x="268.4" y="947.2"
textLength="1171.2" clip-path="url(#breeze-workflow-run-publish-docs-line-38
[...]
+</text><text class="breeze-workflow-run-publish-docs-r5" x="0" y="971.6"
textLength="12.2"
clip-path="url(#breeze-workflow-run-publish-docs-line-39)">│</text><text
class="breeze-workflow-run-publish-docs-r4" x="24.4" y="971.6"
textLength="341.6"
clip-path="url(#breeze-workflow-run-publish-docs-line-39)">--ignore-missing-inventories</text><text
class="breeze-workflow-run-publish-docs-r1" x="390.4" y="971.6"
textLength="695.4"
clip-path="url(#breeze-workflow-run-publish-docs-line-39)">Do&# [...]
</text><text class="breeze-workflow-run-publish-docs-r5" x="0" y="996"
textLength="1464"
clip-path="url(#breeze-workflow-run-publish-docs-line-40)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-workflow-run-publish-docs-r1" x="1464" y="996" textLength="12.2"
clip-path="url(#breeze-workflow-run-publish-docs-line-40)">
+</text><text class="breeze-workflow-run-publish-docs-r5" x="0" y="1020.4"
textLength="24.4"
clip-path="url(#breeze-workflow-run-publish-docs-line-41)">╭─</text><text
class="breeze-workflow-run-publish-docs-r5" x="24.4" y="1020.4"
textLength="195.2"
clip-path="url(#breeze-workflow-run-publish-docs-line-41)"> Common options </text><text
class="breeze-workflow-run-publish-docs-r5" x="219.6" y="1020.4"
textLength="1220" clip-path="url(#breeze-workflow-run-publish-docs-line-41)
[...]
+</text><text class="breeze-workflow-run-publish-docs-r5" x="0" y="1044.8"
textLength="12.2"
clip-path="url(#breeze-workflow-run-publish-docs-line-42)">│</text><text
class="breeze-workflow-run-publish-docs-r4" x="24.4" y="1044.8"
textLength="73.2"
clip-path="url(#breeze-workflow-run-publish-docs-line-42)">--help</text><text
class="breeze-workflow-run-publish-docs-r9" x="122" y="1044.8"
textLength="24.4"
clip-path="url(#breeze-workflow-run-publish-docs-line-42)">-h</text><text
class="breez [...]
+</text><text class="breeze-workflow-run-publish-docs-r5" x="0" y="1069.2"
textLength="1464"
clip-path="url(#breeze-workflow-run-publish-docs-line-43)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-workflow-run-publish-docs-r1" x="1464" y="1069.2"
textLength="12.2" clip-path="url(#breeze-workflow-run-publish-docs-line-43)">
</text>
</g>
</g>
diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt
b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt
index 4d64c05aefd..3a57f0bb6d6 100644
--- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt
+++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt
@@ -1 +1 @@
-46e32fc6adb71c667231a207e063e291
+7915334135094723635d68ce396033bf
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 71733e54215..ce95383b69f 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -745,8 +745,13 @@ def start_airflow(
@click.option(
"--clean-build",
is_flag=True,
- help="Cleans the build directory before building the documentation and
removes all inventory "
- "cache (including external inventories).",
+ help="Cleans the build directory before building the documentation. "
+ "Does not delete inventory cache (use --clean-inventory-cache for that).",
+)
[email protected](
+ "--clean-inventory-cache",
+ is_flag=True,
+ help="Cleans the inventory cache before fetching inventories.",
)
@click.option(
"--refresh-airflow-inventories",
@@ -754,6 +759,12 @@ def start_airflow(
help="When set, only airflow package inventories will be refreshed,
regardless "
"if they are already downloaded. With `--clean-build` - everything is
cleaned..",
)
[email protected](
+ "--fail-on-missing-third-party-inventories",
+ is_flag=True,
+ help="Fail the build if any third-party inventory cannot be downloaded. "
+ "By default, missing third-party inventories are warned about but do not
fail the build.",
+)
@click.option("-d", "--docs-only", help="Only build documentation.",
is_flag=True)
@click.option(
"--include-commits", help="Include commits in the documentation.",
is_flag=True, envvar="INCLUDE_COMMITS"
@@ -790,7 +801,9 @@ def start_airflow(
def build_docs(
builder: str,
clean_build: bool,
+ clean_inventory_cache: bool,
refresh_airflow_inventories: bool,
+ fail_on_missing_third_party_inventories: bool,
docs_only: bool,
github_repository: str,
include_not_ready_providers: bool,
@@ -815,7 +828,7 @@ def build_docs(
)
rebuild_or_pull_ci_image_if_needed(command_params=build_params)
if clean_build:
- directories_to_clean = ["_build", "_doctrees", "_inventory_cache",
"apis"]
+ directories_to_clean = ["_build", "_doctrees", "apis"]
else:
directories_to_clean = ["apis"]
generated_path = AIRFLOW_ROOT_PATH / "generated"
@@ -824,6 +837,11 @@ def build_docs(
for directory in generated_path.rglob(dir_name):
console_print(f"[info]Removing {directory}")
shutil.rmtree(directory, ignore_errors=True)
+ if clean_inventory_cache:
+ inventory_cache_dir = generated_path / "_inventory_cache"
+ if inventory_cache_dir.exists():
+ console_print(f"[info]Removing inventory cache:
{inventory_cache_dir}")
+ shutil.rmtree(inventory_cache_dir, ignore_errors=True)
if refresh_airflow_inventories and not clean_build:
console_print("Removing airflow inventories.")
package_globs = ["helm-chart", "docker-stack", "apache-airflow*"]
@@ -849,6 +867,8 @@ def build_docs(
spellcheck_only=spellcheck_only,
one_pass_only=one_pass_only,
include_commits=include_commits,
+
fail_on_missing_third_party_inventories=fail_on_missing_third_party_inventories,
+ clean_inventory_cache=clean_inventory_cache,
short_doc_packages=expand_all_provider_distributions(
short_doc_packages=doc_packages,
include_removed=include_removed_providers,
diff --git
a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
index e6216a176d8..154d1e742c7 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
@@ -347,7 +347,12 @@ DEVELOPER_PARAMETERS: dict[str, list[dict[str, str |
list[str]]]] = {
},
{
"name": "Cleaning inventories",
- "options": ["--clean-build", "--refresh-airflow-inventories"],
+ "options": [
+ "--clean-build",
+ "--clean-inventory-cache",
+ "--refresh-airflow-inventories",
+ "--fail-on-missing-third-party-inventories",
+ ],
},
{
"name": "Filtering options",
diff --git a/dev/breeze/src/airflow_breeze/commands/workflow_commands.py
b/dev/breeze/src/airflow_breeze/commands/workflow_commands.py
index 632fdb52d7b..20952b74bee 100644
--- a/dev/breeze/src/airflow_breeze/commands/workflow_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/workflow_commands.py
@@ -102,6 +102,11 @@ def workflow_run_group():
default="main",
type=str,
)
[email protected](
+ "--ignore-missing-inventories",
+ help="Do not fail the build on missing third-party inventories.",
+ is_flag=True,
+)
@argument_doc_packages
def workflow_run_publish(
ref: str,
@@ -114,6 +119,7 @@ def workflow_run_publish(
airflow_base_version: str | None = None,
apply_commits: str | None = None,
workflow_branch: str = "main",
+ ignore_missing_inventories: bool = False,
):
if len(doc_packages) == 0:
console_print(
@@ -191,6 +197,7 @@ def workflow_run_publish(
"skip-write-to-stable-folder": skip_write_to_stable_folder,
"build-sboms": "true" if "apache-airflow" in doc_packages else "false",
"apply-commits": apply_commits if apply_commits else "",
+ "ignore-missing-inventories": str(ignore_missing_inventories).lower(),
}
if airflow_version:
diff --git a/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py
b/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py
index 2856e722ae4..239a5e46a6a 100644
--- a/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/workflow_commands_config.py
@@ -47,5 +47,11 @@ WORKFLOW_RUN_PARAMETERS: dict[str, list[dict[str, str |
list[str]]]] = {
"--skip-write-to-stable-folder",
],
},
+ {
+ "name": "Inventory handling",
+ "options": [
+ "--ignore-missing-inventories",
+ ],
+ },
],
}
diff --git a/dev/breeze/src/airflow_breeze/params/doc_build_params.py
b/dev/breeze/src/airflow_breeze/params/doc_build_params.py
index eb0100a3359..fc5e1df9674 100644
--- a/dev/breeze/src/airflow_breeze/params/doc_build_params.py
+++ b/dev/breeze/src/airflow_breeze/params/doc_build_params.py
@@ -30,6 +30,8 @@ class DocBuildParams:
short_doc_packages: tuple[str, ...]
one_pass_only: bool = False
include_commits: bool = False
+ fail_on_missing_third_party_inventories: bool = False
+ clean_inventory_cache: bool = False
github_actions = os.environ.get("GITHUB_ACTIONS", "false")
@property
@@ -43,6 +45,10 @@ class DocBuildParams:
doc_args.append("--one-pass-only")
if self.include_commits:
doc_args.append("--include-commits")
+ if self.fail_on_missing_third_party_inventories:
+ doc_args.append("--fail-on-missing-third-party-inventories")
+ if self.clean_inventory_cache:
+ doc_args.append("--clean-inventory-cache")
if self.package_filter:
for filter in self.package_filter:
doc_args.extend(["--package-filter", filter])
diff --git a/devel-common/src/docs/build_docs.py
b/devel-common/src/docs/build_docs.py
index 4b6d90e5ea2..d8cba3a377b 100755
--- a/devel-common/src/docs/build_docs.py
+++ b/devel-common/src/docs/build_docs.py
@@ -138,8 +138,9 @@ def _promote_new_flags():
console.print()
console.print("You can run clean build - refreshing inter-sphinx
inventories or refresh airflow ones.\n")
console.print(
- " [bright_blue]--clean-build - Refresh inventories
and build files for all inter-sphinx references (including external ones)[/]"
+ " [bright_blue]--clean-build - Clean build files
(without cleaning inventory cache)[/]"
)
+ console.print(" [bright_blue]--clean-inventory-cache - Clean
inventory cache before fetching[/]")
console.print(
" [bright_blue]--refresh-airflow-inventories - Force refresh only
airflow inventories (without cleaning build files or external inventories).[/]"
)
@@ -467,7 +468,12 @@ click.rich_click.OPTION_GROUPS = {
},
{
"name": "Cleaning inventories",
- "options": ["--clean-build", "--refresh-airflow-inventories"],
+ "options": [
+ "--clean-build",
+ "--clean-inventory-cache",
+ "--refresh-airflow-inventories",
+ "--fail-on-missing-third-party-inventories",
+ ],
},
{
"name": "Filtering options",
@@ -530,8 +536,13 @@ click.rich_click.OPTION_GROUPS = {
@click.option(
"--clean-build",
is_flag=True,
- help="Cleans the build directory before building the documentation and
removes all inventory "
- "cache (including external inventories).",
+ help="Cleans the build directory before building the documentation. "
+ "Does not delete inventory cache (use --clean-inventory-cache for that).",
+)
[email protected](
+ "--clean-inventory-cache",
+ is_flag=True,
+ help="Cleans the inventory cache before fetching inventories.",
)
@click.option(
"--refresh-airflow-inventories",
@@ -539,6 +550,12 @@ click.rich_click.OPTION_GROUPS = {
help="When set, only airflow package inventories will be refreshed,
regardless "
"if they are already downloaded. With `--clean-build` - everything is
cleaned..",
)
[email protected](
+ "--fail-on-missing-third-party-inventories",
+ is_flag=True,
+ help="Fail the build if any third-party inventory cannot be downloaded. "
+ "By default, missing third-party inventories are warned about but do not
fail the build.",
+)
@click.option(
"-v",
"--verbose",
@@ -556,12 +573,14 @@ def build_docs(
one_pass_only,
package_filters,
clean_build,
+ clean_inventory_cache,
docs_only,
spellcheck_only,
include_commits,
jobs,
list_packages,
refresh_airflow_inventories,
+ fail_on_missing_third_party_inventories,
verbose,
packages,
):
@@ -605,7 +624,10 @@ def build_docs(
# Inventories that could not be retrieved should be built first. This
may mean this is a
# new package.
packages_without_inventories = fetch_inventories(
- clean_build=clean_build,
refresh_airflow_inventories=refresh_airflow_inventories
+ clean_build=clean_build,
+ refresh_airflow_inventories=refresh_airflow_inventories,
+
fail_on_missing_third_party=fail_on_missing_third_party_inventories,
+ clean_inventory_cache=clean_inventory_cache,
)
normal_packages, priority_packages = partition(
lambda d: d in packages_without_inventories, packages_to_build
diff --git a/devel-common/src/docs/utils/conf_constants.py
b/devel-common/src/docs/utils/conf_constants.py
index 688fcf6fd6d..1bd89b3e8d6 100644
--- a/devel-common/src/docs/utils/conf_constants.py
+++ b/devel-common/src/docs/utils/conf_constants.py
@@ -274,10 +274,23 @@ def get_autodoc_mock_imports() -> list[str]:
]
+def _get_third_party_mapping(pkg_names: list[str]) -> dict[str, tuple[str,
tuple[str]]]:
+ """Build intersphinx mapping for third-party packages, skipping those
without cached inventories."""
+ mapping: dict[str, tuple[str, tuple[str]]] = {}
+ for pkg_name in pkg_names:
+ inv_path = INVENTORY_CACHE_DIR / pkg_name / "objects.inv"
+ if not inv_path.exists():
+ continue
+ mapping[pkg_name] = (
+ f"{THIRD_PARTY_INDEXES[pkg_name]}/",
+ (str(inv_path),),
+ )
+ return mapping
+
+
def get_intersphinx_mapping() -> dict[str, tuple[str, tuple[str]]]:
- return {
- pkg_name: (f"{THIRD_PARTY_INDEXES[pkg_name]}/",
(f"{INVENTORY_CACHE_DIR}/{pkg_name}/objects.inv",))
- for pkg_name in [
+ return _get_third_party_mapping(
+ [
"boto3",
"celery",
"docker",
@@ -289,16 +302,12 @@ def get_intersphinx_mapping() -> dict[str, tuple[str,
tuple[str]]]:
"requests",
"sqlalchemy",
]
- }
+ )
def get_google_intersphinx_mapping() -> dict[str, tuple[str, tuple[str]]]:
- return {
- pkg_name: (
- f"{THIRD_PARTY_INDEXES[pkg_name]}/",
- (f"{INVENTORY_CACHE_DIR}/{pkg_name}/objects.inv",),
- )
- for pkg_name in [
+ return _get_third_party_mapping(
+ [
"google-api-core",
"google-cloud-automl",
"google-cloud-bigquery",
@@ -324,7 +333,7 @@ def get_google_intersphinx_mapping() -> dict[str,
tuple[str, tuple[str]]]:
"google-cloud-videointelligence",
"google-cloud-vision",
]
- }
+ )
BASIC_AUTOAPI_IGNORE_PATTERNS = [
diff --git a/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
b/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
index ce472095b4a..a35e8b3cbfb 100644
--- a/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
+++ b/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
@@ -106,10 +106,15 @@ def is_airflow_package(pkg_name: str) -> bool:
return pkg_name.startswith("apache-airflow") or pkg_name in ["helm-chart",
"docker-stack", "task-sdk"]
-def fetch_inventories(clean_build: bool, refresh_airflow_inventories: bool =
False) -> list[str]:
+def fetch_inventories(
+ clean_build: bool,
+ refresh_airflow_inventories: bool = False,
+ fail_on_missing_third_party: bool = False,
+ clean_inventory_cache: bool = False,
+) -> list[str]:
"""Fetch all inventories for Airflow documentation packages and store in
cache."""
- if clean_build:
- shutil.rmtree(CACHE_PATH)
+ if clean_inventory_cache:
+ shutil.rmtree(CACHE_PATH, ignore_errors=True)
print()
print("[yellow]Inventory Cache cleaned!")
print()
@@ -172,24 +177,40 @@ def fetch_inventories(clean_build: bool,
refresh_airflow_inventories: bool = Fal
failed, success = list(failed), list(success)
print(f"Result: {len(success)} success, {len(failed)} failed")
if failed:
- terminate = False
+ missing_third_party: list[str] = []
missing_airflow_packages = False
print("Failed packages:")
for pkg_no, (pkg_name, _) in enumerate(failed, start=1):
print(f"{pkg_no}. {pkg_name}")
if not is_airflow_package(pkg_name):
- print(
- f"[yellow]Missing {pkg_name} inventory is not for an
Airflow package: "
- f"it will terminate the execution."
- )
- terminate = True
+ cached_path = CACHE_PATH / pkg_name / "objects.inv"
+ if fail_on_missing_third_party:
+ print(
+ f"[yellow]Missing {pkg_name} inventory is not for an
Airflow package: "
+ f"it will terminate the execution."
+ )
+ missing_third_party.append(pkg_name)
+ elif cached_path.exists():
+ print(
+ f"[yellow]Using cached inventory for {pkg_name} "
+ f"(download failed but cached version exists)."
+ )
+ else:
+ print(f"[yellow]No inventory available for {pkg_name},
cross-references will be broken.")
+ missing_third_party.append(pkg_name)
else:
print(
f"[yellow]Missing {pkg_name} inventory is for an Airflow
package, "
f"it will be built from sources."
)
missing_airflow_packages = True
- if terminate:
+ # Write marker file for CI to detect missing third-party inventories
+ marker_file = CACHE_PATH / ".missing_third_party_inventories"
+ if missing_third_party:
+ marker_file.write_text("\n".join(missing_third_party) + "\n")
+ elif marker_file.exists():
+ marker_file.unlink()
+ if fail_on_missing_third_party and missing_third_party:
print(
"[red]Terminate execution[/]\n\n"
"[yellow]Some non-airflow inventories are missing. If missing
inventory is really "
@@ -204,8 +225,8 @@ def fetch_inventories(clean_build: bool,
refresh_airflow_inventories: bool = Fal
if __name__ == "__main__":
if len(sys.argv) > 1 and sys.argv[1] == "clean":
print("[bright_blue]Cleaning inventory cache before fetching
inventories")
- clean_build = True
+ clean_inventory_cache = True
else:
print("[bright_blue]Just fetching inventories without cleaning cache")
- clean_build = False
- fetch_inventories(clean_build=clean_build)
+ clean_inventory_cache = False
+ fetch_inventories(clean_build=False,
clean_inventory_cache=clean_inventory_cache)
