This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f3acdd28984 Improve caching strategy for inventory cache (#49693)
f3acdd28984 is described below
commit f3acdd2898445052210e2d3ad08093ea97d84fd1
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu Apr 24 14:10:20 2025 +0200
Improve caching strategy for inventory cache (#49693)
* Improve caching strategy for inventory cache
Inventory cache is used to decrease the potential failures coming
from not-available (Tempoararily) external inventories of intersphinx.
This includes both - airflow package inventory and external intersphinx
docs as well.
We are more concerned about availability of third-party inventories
than ours, so we should use the cache mostly for the external packages
rather than for ours and we should refresh it every time any dependency
changes.
The caching strategy implemented originally is to keep inventories for
12 hours, but this might cause 12 hours of failures in CI since we are
using latest cached inventories for stability.
This change adds --refresh-airflow-inventories flag and uses it in
the CI to always refresh airflow inventories and leave the other
cached inventories.
* fixup! Improve caching strategy for inventory cache
---
.github/workflows/ci-image-checks.yml | 7 +-
dev/breeze/doc/images/output_build-docs.svg | 98 +++++++++++++++-------
dev/breeze/doc/images/output_build-docs.txt | 2 +-
.../airflow_breeze/commands/developer_commands.py | 19 ++++-
.../commands/developer_commands_config.py | 23 +++--
devel-common/src/docs/build_docs.py | 35 ++++++--
.../sphinx_exts/docs_build/fetch_inventories.py | 14 +++-
7 files changed, 147 insertions(+), 51 deletions(-)
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index 215e587ae8e..ab87ebe1a3b 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -281,19 +281,18 @@ jobs:
uses:
apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
path: ./generated/_inventory_cache/
- # TODO(potiuk): do better with determining the key
- key: cache-docs-inventory-v1-${{ hashFiles('pyproject.toml') }}
+ key: cache-docs-inventory-v1-${{ hashFiles('**/pyproject.toml') }}
id: restore-docs-inventory-cache
- name: "Building docs with ${{ matrix.flag }} flag"
env:
DOCS_LIST_AS_STRING: ${{ inputs.docs-list-as-string }}
run: >
- breeze build-docs ${DOCS_LIST_AS_STRING} ${{ matrix.flag }}
+ breeze build-docs ${DOCS_LIST_AS_STRING} ${{ matrix.flag }}
--refresh-airflow-inventories
- name: "Save docs inventory cache"
uses:
apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
path: ./generated/_inventory_cache/
- key: cache-docs-inventory-v1-${{ hashFiles('pyproject.toml') }}
+ key: cache-docs-inventory-v1-${{ hashFiles('**/pyproject.toml') }}
if-no-files-found: 'error'
retention-days: '2'
# If we upload from multiple matrix jobs we could end up with a race
condition. so just pick one job
diff --git a/dev/breeze/doc/images/output_build-docs.svg
b/dev/breeze/doc/images/output_build-docs.svg
index f4421e33dc2..3dc40b5fe79 100644
--- a/dev/breeze/doc/images/output_build-docs.svg
+++ b/dev/breeze/doc/images/output_build-docs.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 1148.0"
xmlns="http://www.w3.org/2000/svg">
+<svg class="rich-terminal" viewBox="0 0 1482 1367.6"
xmlns="http://www.w3.org/2000/svg">
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath id="breeze-build-docs-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="1097.0" />
+ <rect x="0" y="0" width="1463.0" height="1316.6" />
</clipPath>
<clipPath id="breeze-build-docs-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -177,9 +177,36 @@
<clipPath id="breeze-build-docs-line-43">
<rect x="0" y="1050.7" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-build-docs-line-44">
+ <rect x="0" y="1075.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-build-docs-line-45">
+ <rect x="0" y="1099.5" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-build-docs-line-46">
+ <rect x="0" y="1123.9" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-build-docs-line-47">
+ <rect x="0" y="1148.3" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-build-docs-line-48">
+ <rect x="0" y="1172.7" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-build-docs-line-49">
+ <rect x="0" y="1197.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-build-docs-line-50">
+ <rect x="0" y="1221.5" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-build-docs-line-51">
+ <rect x="0" y="1245.9" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-build-docs-line-52">
+ <rect x="0" y="1270.3" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="1146" rx="8"/><text
class="breeze-build-docs-title" fill="#c5c8c6" text-anchor="middle" x="740"
y="27">Command: build-docs</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="1365.6" rx="8"/><text
class="breeze-build-docs-title" fill="#c5c8c6" text-anchor="middle" x="740"
y="27">Command: build-docs</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -206,34 +233,43 @@
</text><text class="breeze-build-docs-r1" x="1464" y="361.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-14)">
</text><text class="breeze-build-docs-r1" x="12.2" y="386" textLength="195.2"
clip-path="url(#breeze-build-docs-line-15)">Build documents.</text><text
class="breeze-build-docs-r1" x="1464" y="386" textLength="12.2"
clip-path="url(#breeze-build-docs-line-15)">
</text><text class="breeze-build-docs-r1" x="1464" y="410.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-16)">
-</text><text class="breeze-build-docs-r5" x="0" y="434.8" textLength="24.4"
clip-path="url(#breeze-build-docs-line-17)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="434.8" textLength="134.2"
clip-path="url(#breeze-build-docs-line-17)"> Doc flags </text><text
class="breeze-build-docs-r5" x="158.6" y="434.8" textLength="1281"
clip-path="url(#breeze-build-docs-line-17)">────────────────────────────────────────────────────────────────────────────────────────────────
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="459.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-18)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="459.2" textLength="134.2"
clip-path="url(#breeze-build-docs-line-18)">--docs-only</text><text
class="breeze-build-docs-r6" x="402.6" y="459.2" textLength="24.4"
clip-path="url(#breeze-build-docs-line-18)">-d</text><text
class="breeze-build-docs-r1" x="451.4" y="459.2" textLength="305"
clip-path="url(#breeze-build-do [...]
-</text><text class="breeze-build-docs-r5" x="0" y="483.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-19)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="483.6" textLength="207.4"
clip-path="url(#breeze-build-docs-line-19)">--spellcheck-only</text><text
class="breeze-build-docs-r6" x="402.6" y="483.6" textLength="24.4"
clip-path="url(#breeze-build-docs-line-19)">-s</text><text
class="breeze-build-docs-r1" x="451.4" y="483.6" textLength="292.8"
clip-path="url(#breeze- [...]
-</text><text class="breeze-build-docs-r5" x="0" y="508" textLength="12.2"
clip-path="url(#breeze-build-docs-line-20)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="508" textLength="158.6"
clip-path="url(#breeze-build-docs-line-20)">--clean-build</text><text
class="breeze-build-docs-r1" x="451.4" y="508" textLength="988.2"
clip-path="url(#breeze-build-docs-line-20)">Clean inventories of Inter-Sphinx documentation and generated APIs and 
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="532.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-21)">│</text><text
class="breeze-build-docs-r1" x="451.4" y="532.4" textLength="988.2"
clip-path="url(#breeze-build-docs-line-21)">artifacts before the build - useful for a clean build.                        
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="556.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-22)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="556.8" textLength="183"
clip-path="url(#breeze-build-docs-line-22)">--one-pass-only</text><text
class="breeze-build-docs-r1" x="451.4" y="556.8" textLength="988.2"
clip-path="url(#breeze-build-docs-line-22)">Builds documentation in one pass only. This is useful for d
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="581.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-23)">│</text><text
class="breeze-build-docs-r1" x="451.4" y="581.2" textLength="988.2"
clip-path="url(#breeze-build-docs-line-23)">errors.                                        
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="605.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-24)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="605.6" textLength="195.2"
clip-path="url(#breeze-build-docs-line-24)">--package-filter</text><text
class="breeze-build-docs-r1" x="451.4" y="605.6" textLength="988.2"
clip-path="url(#breeze-build-docs-line-24)">Filter(s) to use more than one can be specified. You can
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="630" textLength="12.2"
clip-path="url(#breeze-build-docs-line-25)">│</text><text
class="breeze-build-docs-r1" x="451.4" y="630" textLength="988.2"
clip-path="url(#breeze-build-docs-line-25)">matching the full package name, for example `apache-airflow-providers-*`. Useful </text><text
class="breeze-build-docs-r5" x="1451.8" y="630" textLength="12.2"
clip-path="url(#breeze-build-docs-line-25)">│ [...]
-</text><text class="breeze-build-docs-r5" x="0" y="654.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-26)">│</text><text
class="breeze-build-docs-r1" x="451.4" y="654.4" textLength="988.2"
clip-path="url(#breeze-build-docs-line-26)">when you want to selectseveral similarly named packages together.                </text><text
class="breeze-build-docs-r5" x=" [...]
-</text><text class="breeze-build-docs-r5" x="0" y="678.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-27)">│</text><text
class="breeze-build-docs-r7" x="451.4" y="678.8" textLength="988.2"
clip-path="url(#breeze-build-docs-line-27)">(TEXT)                                        
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="703.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-28)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="703.2" textLength="353.8"
clip-path="url(#breeze-build-docs-line-28)">--include-not-ready-providers</text><text
class="breeze-build-docs-r1" x="451.4" y="703.2" textLength="817.4"
clip-path="url(#breeze-build-docs-line-28)">Whether to include providers that are not yet ready&#
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="727.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-29)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="727.6" textLength="329.4"
clip-path="url(#breeze-build-docs-line-29)">--include-removed-providers</text><text
class="breeze-build-docs-r1" x="451.4" y="727.6" textLength="561.2"
clip-path="url(#breeze-build-docs-line-29)">Whether to include providers that are removed.</text><text
class= [...]
-</text><text class="breeze-build-docs-r5" x="0" y="752" textLength="12.2"
clip-path="url(#breeze-build-docs-line-30)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="752" textLength="231.8"
clip-path="url(#breeze-build-docs-line-30)">--github-repository</text><text
class="breeze-build-docs-r6" x="402.6" y="752" textLength="24.4"
clip-path="url(#breeze-build-docs-line-30)">-g</text><text
class="breeze-build-docs-r1" x="451.4" y="752" textLength="585.6"
clip-path="url(#breeze-build- [...]
-</text><text class="breeze-build-docs-r5" x="0" y="776.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-31)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="776.4" textLength="109.8"
clip-path="url(#breeze-build-docs-line-31)">--builder</text><text
class="breeze-build-docs-r1" x="451.4" y="776.4" textLength="756.4"
clip-path="url(#breeze-build-docs-line-31)">Buildx builder used to perform `docker buildx build` commands.</text><tex
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="800.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-32)">│</text><text
class="breeze-build-docs-r5" x="451.4" y="800.8" textLength="756.4"
clip-path="url(#breeze-build-docs-line-32)">[default: autodetect]                                    
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="825.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-33)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="825.2" textLength="244"
clip-path="url(#breeze-build-docs-line-33)">--distributions-list</text><text
class="breeze-build-docs-r1" x="451.4" y="825.2" textLength="988.2"
clip-path="url(#breeze-build-docs-line-33)">Optional, contains comma-separated list of package ids that are&
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="849.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-34)">│</text><text
class="breeze-build-docs-r1" x="451.4" y="849.6" textLength="988.2"
clip-path="url(#breeze-build-docs-line-34)">documentation building, and document publishing. It is an easier alternative to  </text><text
class="breeze-build-docs-r5" x="1451.8" y="849.6" textLength="12.2"
clip-path="url(#breeze-b [...]
-</text><text class="breeze-build-docs-r5" x="0" y="874" textLength="12.2"
clip-path="url(#breeze-build-docs-line-35)">│</text><text
class="breeze-build-docs-r1" x="451.4" y="874" textLength="988.2"
clip-path="url(#breeze-build-docs-line-35)">adding individual packages as arguments to every command. This overrides the     </text><text
class="breeze-build-docs-r5" x="1451.8" y="874" textLength="12.2"
clip-path="url( [...]
-</text><text class="breeze-build-docs-r5" x="0" y="898.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-36)">│</text><text
class="breeze-build-docs-r1" x="451.4" y="898.4" textLength="988.2"
clip-path="url(#breeze-build-docs-line-36)">packages passed as arguments.                                 
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="922.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-37)">│</text><text
class="breeze-build-docs-r7" x="451.4" y="922.8" textLength="988.2"
clip-path="url(#breeze-build-docs-line-37)">(TEXT)                                        
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="947.2" textLength="1464"
clip-path="url(#breeze-build-docs-line-38)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="947.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-38)">
-</text><text class="breeze-build-docs-r5" x="0" y="971.6" textLength="24.4"
clip-path="url(#breeze-build-docs-line-39)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="971.6" textLength="195.2"
clip-path="url(#breeze-build-docs-line-39)"> Common options </text><text
class="breeze-build-docs-r5" x="219.6" y="971.6" textLength="1220"
clip-path="url(#breeze-build-docs-line-39)">───────────────────────────────────────────────────────────────────────────────────────────
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="996" textLength="12.2"
clip-path="url(#breeze-build-docs-line-40)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="996" textLength="109.8"
clip-path="url(#breeze-build-docs-line-40)">--dry-run</text><text
class="breeze-build-docs-r6" x="158.6" y="996" textLength="24.4"
clip-path="url(#breeze-build-docs-line-40)">-D</text><text
class="breeze-build-docs-r1" x="207.4" y="996" textLength="719.8"
clip-path="url(#breeze-build-docs-line- [...]
-</text><text class="breeze-build-docs-r5" x="0" y="1020.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-41)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1020.4" textLength="109.8"
clip-path="url(#breeze-build-docs-line-41)">--verbose</text><text
class="breeze-build-docs-r6" x="158.6" y="1020.4" textLength="24.4"
clip-path="url(#breeze-build-docs-line-41)">-v</text><text
class="breeze-build-docs-r1" x="207.4" y="1020.4" textLength="585.6"
clip-path="url(#breeze-buil [...]
-</text><text class="breeze-build-docs-r5" x="0" y="1044.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-42)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1044.8" textLength="97.6"
clip-path="url(#breeze-build-docs-line-42)">--answer</text><text
class="breeze-build-docs-r6" x="158.6" y="1044.8" textLength="24.4"
clip-path="url(#breeze-build-docs-line-42)">-a</text><text
class="breeze-build-docs-r1" x="207.4" y="1044.8" textLength="317.2"
clip-path="url(#breeze-build- [...]
-</text><text class="breeze-build-docs-r5" x="0" y="1069.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-43)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1069.2" textLength="73.2"
clip-path="url(#breeze-build-docs-line-43)">--help</text><text
class="breeze-build-docs-r6" x="158.6" y="1069.2" textLength="24.4"
clip-path="url(#breeze-build-docs-line-43)">-h</text><text
class="breeze-build-docs-r1" x="207.4" y="1069.2" textLength="329.4"
clip-path="url(#breeze-build-do [...]
-</text><text class="breeze-build-docs-r5" x="0" y="1093.6" textLength="1464"
clip-path="url(#breeze-build-docs-line-44)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="1093.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-44)">
+</text><text class="breeze-build-docs-r5" x="0" y="434.8" textLength="24.4"
clip-path="url(#breeze-build-docs-line-17)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="434.8" textLength="671"
clip-path="url(#breeze-build-docs-line-17)"> Build scope (default is to build docs and spellcheck) </text><text
class="breeze-build-docs-r5" x="695.4" y="434.8" textLength="744.2"
clip-path="url(#breeze-build-docs-line-17)">────────────────── [...]
+</text><text class="breeze-build-docs-r5" x="0" y="459.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-18)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="459.2" textLength="134.2"
clip-path="url(#breeze-build-docs-line-18)">--docs-only</text><text
class="breeze-build-docs-r6" x="256.2" y="459.2" textLength="24.4"
clip-path="url(#breeze-build-docs-line-18)">-d</text><text
class="breeze-build-docs-r1" x="305" y="459.2" textLength="305"
clip-path="url(#breeze-build-docs [...]
+</text><text class="breeze-build-docs-r5" x="0" y="483.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-19)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="483.6" textLength="207.4"
clip-path="url(#breeze-build-docs-line-19)">--spellcheck-only</text><text
class="breeze-build-docs-r6" x="256.2" y="483.6" textLength="24.4"
clip-path="url(#breeze-build-docs-line-19)">-s</text><text
class="breeze-build-docs-r1" x="305" y="483.6" textLength="292.8"
clip-path="url(#breeze-bu [...]
+</text><text class="breeze-build-docs-r5" x="0" y="508" textLength="1464"
clip-path="url(#breeze-build-docs-line-20)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="508" textLength="12.2"
clip-path="url(#breeze-build-docs-line-20)">
+</text><text class="breeze-build-docs-r5" x="0" y="532.4" textLength="24.4"
clip-path="url(#breeze-build-docs-line-21)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="532.4" textLength="183"
clip-path="url(#breeze-build-docs-line-21)"> Type of build </text><text
class="breeze-build-docs-r5" x="207.4" y="532.4" textLength="1232.2"
clip-path="url(#breeze-build-docs-line-21)">───────────────────────────────────────────────────────────────────────────────────────
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="556.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-22)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="556.8" textLength="183"
clip-path="url(#breeze-build-docs-line-22)">--one-pass-only</text><text
class="breeze-build-docs-r1" x="256.2" y="556.8" textLength="1000.4"
clip-path="url(#breeze-build-docs-line-22)">Builds documentation in one pass only. This is useful for 
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="581.2" textLength="1464"
clip-path="url(#breeze-build-docs-line-23)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="581.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-23)">
+</text><text class="breeze-build-docs-r5" x="0" y="605.6" textLength="24.4"
clip-path="url(#breeze-build-docs-line-24)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="605.6" textLength="268.4"
clip-path="url(#breeze-build-docs-line-24)"> Cleaning inventories </text><text
class="breeze-build-docs-r5" x="292.8" y="605.6" textLength="1146.8"
clip-path="url(#breeze-build-docs-line-24)">───────────────────────────────────────────────────────────────────────────────────
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="630" textLength="12.2"
clip-path="url(#breeze-build-docs-line-25)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="630" textLength="158.6"
clip-path="url(#breeze-build-docs-line-25)">--clean-build</text><text
class="breeze-build-docs-r1" x="427" y="630" textLength="1012.6"
clip-path="url(#breeze-build-docs-line-25)">Cleans the build directory before building the documentation and remove
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="654.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-26)">│</text><text
class="breeze-build-docs-r1" x="427" y="654.4" textLength="1012.6"
clip-path="url(#breeze-build-docs-line-26)">inventory cache (including external inventories).                             
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="678.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-27)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="678.8" textLength="353.8"
clip-path="url(#breeze-build-docs-line-27)">--refresh-airflow-inventories</text><text
class="breeze-build-docs-r1" x="427" y="678.8" textLength="1012.6"
clip-path="url(#breeze-build-docs-line-27)">When set, only airflow package inventories will be ref
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="703.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-28)">│</text><text
class="breeze-build-docs-r1" x="427" y="703.2" textLength="366"
clip-path="url(#breeze-build-docs-line-28)">are already downloaded. With `</text><text
class="breeze-build-docs-r4" x="793" y="703.2" textLength="158.6"
clip-path="url(#breeze-build-docs-line-28)">--clean-build</text><text
class="breeze-build-docs-r1" x="951.6" y="703.2" te [...]
+</text><text class="breeze-build-docs-r5" x="0" y="727.6" textLength="1464"
clip-path="url(#breeze-build-docs-line-29)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="727.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-29)">
+</text><text class="breeze-build-docs-r5" x="0" y="752" textLength="24.4"
clip-path="url(#breeze-build-docs-line-30)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="752" textLength="231.8"
clip-path="url(#breeze-build-docs-line-30)"> Filtering options </text><text
class="breeze-build-docs-r5" x="256.2" y="752" textLength="1183.4"
clip-path="url(#breeze-build-docs-line-30)">────────────────────────────────────────────────────────────────────────────────────────────
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="776.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-31)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="776.4" textLength="195.2"
clip-path="url(#breeze-build-docs-line-31)">--package-filter</text><text
class="breeze-build-docs-r1" x="427" y="776.4" textLength="1012.6"
clip-path="url(#breeze-build-docs-line-31)">Filter(s) to use more than one can be specified. You can&
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="800.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-32)">│</text><text
class="breeze-build-docs-r1" x="427" y="800.8" textLength="1012.6"
clip-path="url(#breeze-build-docs-line-32)">the full package name, for example `apache-airflow-providers-*`. Useful when you   </text><text
class="breeze-build-docs-r5" x="1451.8" y="800.8" textLength="12.2"
clip-path="url(#breeze- [...]
+</text><text class="breeze-build-docs-r5" x="0" y="825.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-33)">│</text><text
class="breeze-build-docs-r1" x="427" y="825.2" textLength="1012.6"
clip-path="url(#breeze-build-docs-line-33)">want to selectseveral similarly named packages together.                           
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="849.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-34)">│</text><text
class="breeze-build-docs-r7" x="427" y="849.6" textLength="1012.6"
clip-path="url(#breeze-build-docs-line-34)">(TEXT)                                        &
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="874" textLength="12.2"
clip-path="url(#breeze-build-docs-line-35)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="874" textLength="353.8"
clip-path="url(#breeze-build-docs-line-35)">--include-not-ready-providers</text><text
class="breeze-build-docs-r1" x="427" y="874" textLength="817.4"
clip-path="url(#breeze-build-docs-line-35)">Whether to include providers that are not yet ready to&#
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="898.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-36)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="898.4" textLength="329.4"
clip-path="url(#breeze-build-docs-line-36)">--include-removed-providers</text><text
class="breeze-build-docs-r1" x="427" y="898.4" textLength="561.2"
clip-path="url(#breeze-build-docs-line-36)">Whether to include providers that are removed.</text><text
class="b [...]
+</text><text class="breeze-build-docs-r5" x="0" y="922.8" textLength="1464"
clip-path="url(#breeze-build-docs-line-37)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="922.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-37)">
+</text><text class="breeze-build-docs-r5" x="0" y="947.2" textLength="24.4"
clip-path="url(#breeze-build-docs-line-38)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="947.2" textLength="170.8"
clip-path="url(#breeze-build-docs-line-38)"> Misc options </text><text
class="breeze-build-docs-r5" x="195.2" y="947.2" textLength="1244.4"
clip-path="url(#breeze-build-docs-line-38)">───────────────────────────────────────────────────────────────────────────────────────────
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="971.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-39)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="971.6" textLength="231.8"
clip-path="url(#breeze-build-docs-line-39)">--github-repository</text><text
class="breeze-build-docs-r6" x="292.8" y="971.6" textLength="24.4"
clip-path="url(#breeze-build-docs-line-39)">-g</text><text
class="breeze-build-docs-r1" x="341.6" y="971.6" textLength="585.6"
clip-path="url(#breez [...]
+</text><text class="breeze-build-docs-r5" x="0" y="996" textLength="12.2"
clip-path="url(#breeze-build-docs-line-40)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="996" textLength="109.8"
clip-path="url(#breeze-build-docs-line-40)">--builder</text><text
class="breeze-build-docs-r1" x="341.6" y="996" textLength="756.4"
clip-path="url(#breeze-build-docs-line-40)">Buildx builder used to perform `docker buildx build` commands.</text><text
clas [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1020.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-41)">│</text><text
class="breeze-build-docs-r5" x="341.6" y="1020.4" textLength="756.4"
clip-path="url(#breeze-build-docs-line-41)">[default: autodetect]                                    &#
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="1044.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-42)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1044.8" textLength="244"
clip-path="url(#breeze-build-docs-line-42)">--distributions-list</text><text
class="breeze-build-docs-r1" x="341.6" y="1044.8" textLength="1098"
clip-path="url(#breeze-build-docs-line-42)">Optional, contains comma-separated list of package ids that ar
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="1069.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-43)">│</text><text
class="breeze-build-docs-r1" x="341.6" y="1069.2" textLength="1098"
clip-path="url(#breeze-build-docs-line-43)">documentation building, and document publishing. It is an easier alternative to adding    </text><text
class="breeze-build-docs-r5" x="1451.8" y="1069.2" textLength="12.2 [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1093.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-44)">│</text><text
class="breeze-build-docs-r1" x="341.6" y="1093.6" textLength="1098"
clip-path="url(#breeze-build-docs-line-44)">individual packages as arguments to every command. This overrides the packages passed as  </text><text
class="breeze-build-docs-r5" x="1451.8" y="1093.6" textLength="12.2" cli [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1118" textLength="12.2"
clip-path="url(#breeze-build-docs-line-45)">│</text><text
class="breeze-build-docs-r1" x="341.6" y="1118" textLength="1098"
clip-path="url(#breeze-build-docs-line-45)">arguments.                                        
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="1142.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-46)">│</text><text
class="breeze-build-docs-r7" x="341.6" y="1142.4" textLength="1098"
clip-path="url(#breeze-build-docs-line-46)">(TEXT)                                        
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="1166.8" textLength="1464"
clip-path="url(#breeze-build-docs-line-47)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="1166.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-47)">
+</text><text class="breeze-build-docs-r5" x="0" y="1191.2" textLength="24.4"
clip-path="url(#breeze-build-docs-line-48)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="1191.2" textLength="195.2"
clip-path="url(#breeze-build-docs-line-48)"> Common options </text><text
class="breeze-build-docs-r5" x="219.6" y="1191.2" textLength="1220"
clip-path="url(#breeze-build-docs-line-48)">────────────────────────────────────────────────────────────────────────────────────────
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="1215.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-49)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1215.6" textLength="109.8"
clip-path="url(#breeze-build-docs-line-49)">--dry-run</text><text
class="breeze-build-docs-r6" x="158.6" y="1215.6" textLength="24.4"
clip-path="url(#breeze-build-docs-line-49)">-D</text><text
class="breeze-build-docs-r1" x="207.4" y="1215.6" textLength="719.8"
clip-path="url(#breeze-buil [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1240" textLength="12.2"
clip-path="url(#breeze-build-docs-line-50)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1240" textLength="109.8"
clip-path="url(#breeze-build-docs-line-50)">--verbose</text><text
class="breeze-build-docs-r6" x="158.6" y="1240" textLength="24.4"
clip-path="url(#breeze-build-docs-line-50)">-v</text><text
class="breeze-build-docs-r1" x="207.4" y="1240" textLength="585.6"
clip-path="url(#breeze-build-docs-l [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1264.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-51)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1264.4" textLength="97.6"
clip-path="url(#breeze-build-docs-line-51)">--answer</text><text
class="breeze-build-docs-r6" x="158.6" y="1264.4" textLength="24.4"
clip-path="url(#breeze-build-docs-line-51)">-a</text><text
class="breeze-build-docs-r1" x="207.4" y="1264.4" textLength="317.2"
clip-path="url(#breeze-build- [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1288.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-52)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="1288.8" textLength="73.2"
clip-path="url(#breeze-build-docs-line-52)">--help</text><text
class="breeze-build-docs-r6" x="158.6" y="1288.8" textLength="24.4"
clip-path="url(#breeze-build-docs-line-52)">-h</text><text
class="breeze-build-docs-r1" x="207.4" y="1288.8" textLength="329.4"
clip-path="url(#breeze-build-do [...]
+</text><text class="breeze-build-docs-r5" x="0" y="1313.2" textLength="1464"
clip-path="url(#breeze-build-docs-line-53)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="1313.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-53)">
</text>
</g>
</g>
diff --git a/dev/breeze/doc/images/output_build-docs.txt
b/dev/breeze/doc/images/output_build-docs.txt
index b8ce7331d5e..7089fd41168 100644
--- a/dev/breeze/doc/images/output_build-docs.txt
+++ b/dev/breeze/doc/images/output_build-docs.txt
@@ -1 +1 @@
-8ae2b68e8981f5fc57c32b7e064211bd
+0da87f68c7956d7611372a00164b6c5c
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 67927a27883..8d821f88194 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -667,9 +667,15 @@ def start_airflow(
@option_builder
@click.option(
"--clean-build",
- help="Clean inventories of Inter-Sphinx documentation and generated APIs
and sphinx artifacts "
- "before the build - useful for a clean build.",
is_flag=True,
+ help="Cleans the build directory before building the documentation and
removes all inventory "
+ "cache (including external inventories).",
+)
[email protected](
+ "--refresh-airflow-inventories",
+ is_flag=True,
+ help="When set, only airflow package inventories will be refreshed,
regardless "
+ "if they are already downloaded. With `--clean-build` - everything is
cleaned..",
)
@click.option("-d", "--docs-only", help="Only build documentation.",
is_flag=True)
@option_dry_run
@@ -704,6 +710,7 @@ def start_airflow(
def build_docs(
builder: str,
clean_build: bool,
+ refresh_airflow_inventories: bool,
docs_only: bool,
github_repository: str,
include_not_ready_providers: bool,
@@ -730,9 +737,17 @@ def build_docs(
directories_to_clean = ["apis"]
generated_path = AIRFLOW_ROOT_PATH / "generated"
for dir_name in directories_to_clean:
+ get_console().print("Removing all generated dirs.")
for directory in generated_path.rglob(dir_name):
get_console().print(f"[info]Removing {directory}")
shutil.rmtree(directory, ignore_errors=True)
+ if refresh_airflow_inventories and not clean_build:
+ get_console().print("Removing airflow inventories.")
+ package_globs = ["helm-chart", "docker-stack", "apache-airflow*"]
+ for package_glob in package_globs:
+ for directory in (generated_path /
"_inventory_cache").rglob(package_glob):
+ get_console().print(f"[info]Removing {directory}")
+ shutil.rmtree(directory, ignore_errors=True)
docs_list_as_tuple: tuple[str, ...] = ()
if distributions_list and len(distributions_list):
diff --git
a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
index 042bfc5bcfc..06b772fe62e 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
@@ -304,15 +304,28 @@ DEVELOPER_PARAMETERS: dict[str, list[dict[str, str |
list[str]]]] = {
],
"breeze build-docs": [
{
- "name": "Doc flags",
+ "name": "Build scope (default is to build docs and spellcheck)",
+ "options": ["--docs-only", "--spellcheck-only"],
+ },
+ {
+ "name": "Type of build",
+ "options": ["--one-pass-only"],
+ },
+ {
+ "name": "Cleaning inventories",
+ "options": ["--clean-build", "--refresh-airflow-inventories"],
+ },
+ {
+ "name": "Filtering options",
"options": [
- "--docs-only",
- "--spellcheck-only",
- "--clean-build",
- "--one-pass-only",
"--package-filter",
"--include-not-ready-providers",
"--include-removed-providers",
+ ],
+ },
+ {
+ "name": "Misc options",
+ "options": [
"--github-repository",
"--builder",
"--distributions-list",
diff --git a/devel-common/src/docs/build_docs.py
b/devel-common/src/docs/build_docs.py
index 2ff27b2183d..2ea402d42fd 100755
--- a/devel-common/src/docs/build_docs.py
+++ b/devel-common/src/docs/build_docs.py
@@ -132,12 +132,19 @@ def _promote_new_flags():
console.print("You can also use other extra flags to iterate faster:")
console.print(" [bright_blue]--docs-only - Only build
documentation[/]")
console.print(" [bright_blue]--spellcheck-only - Only perform
spellchecking[/]")
- console.print(" [bright_blue]--clean-build - Refresh inventories for
inter-sphinx references[/]")
console.print()
console.print("You can list all packages you can build:")
console.print()
console.print(" [bright_blue]--list-packages - Shows the list of
packages you can build[/]")
console.print()
+ console.print("You can run clean build - refreshing inter-sphinx
inventories or refresh airflow ones.\n")
+ console.print(
+ " [bright_blue]--clean-build - Refresh inventories
and build files for all inter-sphinx references (including external ones)[/]"
+ )
+ console.print(
+ " [bright_blue]--refresh-airflow-inventories - Force refresh only
airflow inventories (without cleaning build files or external inventories).[/]"
+ )
+ console.print()
console.print("For more info:")
console.print(" [bright_blue]uv run build-docs --help[/]")
console.print()
@@ -457,7 +464,11 @@ click.rich_click.OPTION_GROUPS = {
},
{
"name": "Type of build",
- "options": ["--autobuild", "--one-pass-only", "--clean-build"],
+ "options": ["--autobuild", "--one-pass-only"],
+ },
+ {
+ "name": "Cleaning inventories",
+ "options": ["--clean-build", "--refresh-airflow-inventories"],
},
{
"name": "Filtering options",
@@ -497,9 +508,6 @@ click.rich_click.OPTION_GROUPS = {
"only that package is selected to build. "
"If the command is run in the root of the Airflow repo, all packages are
selected to be built.",
)
[email protected](
- "--clean-build", is_flag=True, help="Cleans the build directory before
building the documentation."
-)
@click.option("--docs-only", is_flag=True, help="Only build documentation")
@click.option("--spellcheck-only", is_flag=True, help="Only perform
spellchecking")
@click.option(
@@ -516,6 +524,18 @@ click.rich_click.OPTION_GROUPS = {
is_flag=True,
help="Lists all available packages. You can use it to check the names of
the packages you want to build.",
)
[email protected](
+ "--clean-build",
+ is_flag=True,
+ help="Cleans the build directory before building the documentation and
removes all inventory "
+ "cache (including external inventories).",
+)
[email protected](
+ "--refresh-airflow-inventories",
+ is_flag=True,
+ help="When set, only airflow package inventories will be refreshed,
regardless "
+ "if they are already downloaded. With `--clean-build` - everything is
cleaned..",
+)
@click.option(
"-v",
"--verbose",
@@ -537,6 +557,7 @@ def build_docs(
spellcheck_only,
jobs,
list_packages,
+ refresh_airflow_inventories,
verbose,
packages,
):
@@ -579,7 +600,9 @@ def build_docs(
with with_group("Fetching inventories"):
# Inventories that could not be retrieved should be built first. This
may mean this is a
# new package.
- packages_without_inventories =
fetch_inventories(clean_build=clean_build)
+ packages_without_inventories = fetch_inventories(
+ clean_build=clean_build,
refresh_airflow_inventories=refresh_airflow_inventories
+ )
normal_packages, priority_packages = partition(
lambda d: d in packages_without_inventories, packages_to_build
)
diff --git a/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
b/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
index f9aa771bda7..2114e3e7393 100644
--- a/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
+++ b/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
@@ -93,7 +93,13 @@ def _is_outdated(path: str):
return delta > datetime.timedelta(hours=12)
-def fetch_inventories(clean_build: bool) -> list[str]:
+def should_be_refreshed(pkg_name: str, refresh_airflow_inventories: bool) ->
bool:
+ if pkg_name in ["helm-chart", "docker-stack"] or
pkg_name.startswith("apache-airflow"):
+ return refresh_airflow_inventories
+ return False
+
+
+def fetch_inventories(clean_build: bool, refresh_airflow_inventories: bool =
False) -> list[str]:
"""Fetch all inventories for Airflow documentation packages and store in
cache."""
if clean_build:
shutil.rmtree(CACHE_PATH)
@@ -136,7 +142,11 @@ def fetch_inventories(clean_build: bool) -> list[str]:
for pkg_name, doc_url in THIRD_PARTY_INDEXES.items()
)
- to_download = [(pkg_name, url, path) for pkg_name, url, path in
to_download if _is_outdated(path)]
+ to_download = [
+ (pkg_name, url, path)
+ for pkg_name, url, path in to_download
+ if _is_outdated(path) or should_be_refreshed(pkg_name,
refresh_airflow_inventories)
+ ]
if not to_download:
print("Nothing to do")
return []