This is an automated email from the ASF dual-hosted git repository.
jason810496 pushed a commit to branch v3-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-4-test by this push:
new cdc3d09f6c2 [v3-4-test] Add YAML Dag schema publishing infra (#75055)
(#75241)
cdc3d09f6c2 is described below
commit cdc3d09f6c2ab76b79ef2b08a2f88edad1bc5c80
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Oct 21 10:20:47 2026 +0800
[v3-4-test] Add YAML Dag schema publishing infra (#75055) (#75241)
* Add YAML Dag schema publishing infra
* Tighten YAML might_contain_dag sniff
* Only store the version instead of full URL in schem
* Remove unneeded Click argument name
(cherry picked from commit 5fa997cf002c824eefc6f9a893943e080f8ad4b4)
Co-authored-by: Tzu-ping Chung <[email protected]>
---
.github/workflows/publish-docs-to-s3.yml | 20 ++++++--
.../docs/authoring-and-scheduling/yaml-dags.rst | 4 +-
dev/breeze/doc/09_release_management_tasks.rst | 10 ++--
.../doc/images/output_release-management.svg | 2 +-
.../doc/images/output_release-management.txt | 2 +-
...ut_release-management_publish-schemas-to-s3.svg | 34 +++++++------
...ut_release-management_publish-schemas-to-s3.txt | 2 +-
.../commands/release_management_commands.py | 15 ++++--
.../commands/release_management_commands_config.py | 1 +
.../src/airflow_breeze/utils/publish_docs_to_s3.py | 6 ++-
scripts/ci/prek/generate_yaml_importer_schema.py | 1 +
.../sdk/importers/yaml_importer/importer.py | 20 ++++----
.../sdk/importers/yaml_importer/schema.json | 3 +-
.../importers/yaml_importer/test_importer.py | 55 +++++++++++++++++-----
14 files changed, 122 insertions(+), 53 deletions(-)
diff --git a/.github/workflows/publish-docs-to-s3.yml
b/.github/workflows/publish-docs-to-s3.yml
index 84d5292d0df..4e3c053366c 100644
--- a/.github/workflows/publish-docs-to-s3.yml
+++ b/.github/workflows/publish-docs-to-s3.yml
@@ -96,6 +96,7 @@ jobs:
airflow-version: ${{ steps.parameters.outputs.airflow-version }}
publish-execution-api-schema: ${{
steps.parameters.outputs.publish-execution-api-schema }}
publish-supervisor-schema: ${{
steps.parameters.outputs.publish-supervisor-schema }}
+ publish-dag-schema: ${{ steps.parameters.outputs.publish-dag-schema }}
# yamllint disable rule:line-length
skip-write-to-stable-folder: ${{ inputs.skip-write-to-stable-folder &&
'--skip-write-to-stable-folder' || '' }}
# Docs and the registry each build the image they need rather than
sharing one. Sharing
@@ -244,15 +245,19 @@ jobs:
fi
PUBLISH_EXEC="false"
PUBLISH_SUP="false"
+ PUBLISH_DAG="false"
if [[ "${INCLUDE_DOCS}" == "all" || "${INCLUDE_DOCS}" == "" ]]; then
PUBLISH_EXEC="true"
PUBLISH_SUP="true"
+ PUBLISH_DAG="true"
else
[[ " ${INCLUDE_DOCS} " == *" apache-airflow "* ]] &&
PUBLISH_EXEC="true"
[[ " ${INCLUDE_DOCS} " == *" task-sdk "* ]] && PUBLISH_SUP="true"
+ [[ " ${INCLUDE_DOCS} " == *" task-sdk "* ]] && PUBLISH_DAG="true"
fi
echo "publish-execution-api-schema=${PUBLISH_EXEC}" >>
${GITHUB_OUTPUT}
echo "publish-supervisor-schema=${PUBLISH_SUP}" >> ${GITHUB_OUTPUT}
+ echo "publish-dag-schema=${PUBLISH_DAG}" >> ${GITHUB_OUTPUT}
build-docs:
needs: [build-info]
@@ -409,8 +414,13 @@ jobs:
breeze shell --quiet --backend=none \
"python scripts/ci/prek/dump_supervisor_schemas.py >
/files/supervisor-schema.json"
cp files/supervisor-schema.json schemas/supervisor-schema.json
+ - name: "Copy YAML Dag schema artifact"
+ if: needs.build-info.outputs.publish-dag-schema == 'true'
+ run: |
+ mkdir -p schemas
+ cp task-sdk/src/airflow/sdk/importers/yaml_importer/schema.json
schemas/dag.json
- name: "Upload schema artifacts"
- if: needs.build-info.outputs.publish-execution-api-schema == 'true' ||
needs.build-info.outputs.publish-supervisor-schema == 'true'
+ if: needs.build-info.outputs.publish-execution-api-schema == 'true' ||
needs.build-info.outputs.publish-supervisor-schema == 'true' ||
needs.build-info.outputs.publish-dag-schema == 'true'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
# v7.0.2
with:
name: airflow-schemas
@@ -696,17 +706,18 @@ jobs:
--destination-location ${DESTINATION_LOCATION} --stable-versions \
--exclude-docs "${EXCLUDE_DOCS}" --overwrite
${SKIP_WRITE_TO_STABLE_FOLDER}
- name: "Download schema artifacts"
- if: needs.build-info.outputs.publish-execution-api-schema == 'true' ||
needs.build-info.outputs.publish-supervisor-schema == 'true'
+ if: needs.build-info.outputs.publish-execution-api-schema == 'true' ||
needs.build-info.outputs.publish-supervisor-schema == 'true' ||
needs.build-info.outputs.publish-dag-schema == 'true'
uses:
actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: airflow-schemas
path: schemas
- name: "Publish schemas to S3"
- if: needs.build-info.outputs.publish-execution-api-schema == 'true' ||
needs.build-info.outputs.publish-supervisor-schema == 'true'
+ if: needs.build-info.outputs.publish-execution-api-schema == 'true' ||
needs.build-info.outputs.publish-supervisor-schema == 'true' ||
needs.build-info.outputs.publish-dag-schema == 'true'
env:
DESTINATION_LOCATION: "${{
needs.build-info.outputs.destination-location }}"
PUBLISH_EXEC: "${{
needs.build-info.outputs.publish-execution-api-schema }}"
PUBLISH_SUP: "${{ needs.build-info.outputs.publish-supervisor-schema
}}"
+ PUBLISH_DAG: "${{ needs.build-info.outputs.publish-dag-schema }}"
run: |
# Reuse the docs bucket but publish under the schemas/ prefix.
SCHEMAS_DESTINATION="${DESTINATION_LOCATION%docs/}schemas/"
@@ -717,6 +728,9 @@ jobs:
if [[ "${PUBLISH_SUP}" == "true" ]]; then
args+=(--supervisor schemas/supervisor-schema.json)
fi
+ if [[ "${PUBLISH_DAG}" == "true" ]]; then
+ args+=(--dag schemas/dag.json)
+ fi
breeze release-management publish-schemas-to-s3 \
--destination-location "${SCHEMAS_DESTINATION}" "${args[@]}"
diff --git a/airflow-core/docs/authoring-and-scheduling/yaml-dags.rst
b/airflow-core/docs/authoring-and-scheduling/yaml-dags.rst
index 2d563b5d4bd..ceef4e9d4b1 100644
--- a/airflow-core/docs/authoring-and-scheduling/yaml-dags.rst
+++ b/airflow-core/docs/authoring-and-scheduling/yaml-dags.rst
@@ -102,8 +102,8 @@ The shape of a document
parsed with. ``$schema`` is also how Airflow recognizes that a file
contains
Dags. By default (see :ref:`config:core__dag_discovery_safe_mode`), a file
that does not carry the canonical ``$schema`` URL is skipped, so a mistyped
- URL makes the file disappear the same way. Editors that understand
- ``$schema`` will offer validation and completion with no extra setup.
+ URL makes the file disappear the same way. An editor that understands
+ ``$schema`` can offer validation and completion from it.
Dag-level keys
Dag attributes sit at the top level, such as ``dag_id``, ``schedule``,
diff --git a/dev/breeze/doc/09_release_management_tasks.rst
b/dev/breeze/doc/09_release_management_tasks.rst
index 4d22d9a61ca..b24c38e9ebe 100644
--- a/dev/breeze/doc/09_release_management_tasks.rst
+++ b/dev/breeze/doc/09_release_management_tasks.rst
@@ -1142,15 +1142,16 @@ These are all available flags of ``workflow-run
release-constraints`` command:
Publishing the schema files to S3
"""""""""""""""""""""""""""""""""
-Alongside the documentation, the ``Publish Docs to S3`` workflow also
publishes two generated schema artifacts
+Alongside the documentation, the ``Publish Docs to S3`` workflow also
publishes generated schema artifacts
to the same docs bucket, under the ``schemas/`` prefix (served at
``https://airflow.apache.org/schemas/``):
* The Execution API OpenAPI spec ``schemas/execution-api/<version>.json``
* The Supervisor JSON Schema ``schemas/supervisor-schema/<version>.json``
+* The YAML Dag JSON Schema ``schemas/dag/<version>.json``
Each dated file is immutable, so the command uploads individual objects and
skips a date that already exists
-unless ``--overwrite`` is given. Publishing is gated on the package set. The
schema files are only published
-when ``apache-airflow`` or ``task-sdk`` is built.
+unless ``--overwrite`` is given. Publishing is gated on the package set. The
Execution API schema is published
+when ``apache-airflow`` is built; the Supervisor and YAML Dag schemas when
``task-sdk`` is built.
To publish the schema files to S3, use the ``release-management
publish-schemas-to-s3`` command:
@@ -1159,11 +1160,12 @@ To publish the schema files to S3, use the
``release-management publish-schemas-
breeze release-management publish-schemas-to-s3 \
--execution-api execution-api.json \
--supervisor supervisor-schema.json \
+ --dag dag.json \
--destination-location s3://live-docs-airflow-apache-org/schemas/
``--destination-location`` is the ``s3://<bucket>/schemas/`` location to
publish under; each schema is
written to ``<location>/<schema-type>/<version>.json``. Pass at least one of
-``--execution-api`` / ``--supervisor``.
+``--execution-api`` / ``--supervisor`` / ``--dag``.
These are all available flags of ``release-management publish-schemas-to-s3``
command:
diff --git a/dev/breeze/doc/images/output_release-management.svg
b/dev/breeze/doc/images/output_release-management.svg
index ab02bee7fd7..e85b6fa1574 100644
--- a/dev/breeze/doc/images/output_release-management.svg
+++ b/dev/breeze/doc/images/output_release-management.svg
@@ -320,7 +320,7 @@
</text><text class="breeze-release-management-r5" x="0" y="1410.8"
textLength="12.2"
clip-path="url(#breeze-release-management-line-57)">│</text><text
class="breeze-release-management-r4" x="24.4" y="1410.8" textLength="280.6"
clip-path="url(#breeze-release-management-line-57)">generate-constraints   </text><text
class="breeze-release-management-r1" x="329.4" y="1410.8" textLength="1110.2"
clip-path="url(#breeze-release-management-line-57)">Generates pinned const
[...]
</text><text class="breeze-release-management-r5" x="0" y="1435.2"
textLength="12.2"
clip-path="url(#breeze-release-management-line-58)">│</text><text
class="breeze-release-management-r4" x="24.4" y="1435.2" textLength="280.6"
clip-path="url(#breeze-release-management-line-58)">update-constraints     </text><text
class="breeze-release-management-r1" x="329.4" y="1435.2" textLength="1110.2"
clip-path="url(#breeze-release-management-line-58)">Update released&# [...]
</text><text class="breeze-release-management-r5" x="0" y="1459.6"
textLength="12.2"
clip-path="url(#breeze-release-management-line-59)">│</text><text
class="breeze-release-management-r4" x="24.4" y="1459.6" textLength="280.6"
clip-path="url(#breeze-release-management-line-59)">publish-docs-to-s3     </text><text
class="breeze-release-management-r1" x="329.4" y="1459.6" textLength="1110.2"
clip-path="url(#breeze-release-management-line-59)">Publishes docs [...]
-</text><text class="breeze-release-management-r5" x="0" y="1484"
textLength="12.2"
clip-path="url(#breeze-release-management-line-60)">│</text><text
class="breeze-release-management-r4" x="24.4" y="1484" textLength="280.6"
clip-path="url(#breeze-release-management-line-60)">publish-schemas-to-s3  </text><text
class="breeze-release-management-r1" x="329.4" y="1484" textLength="1110.2"
clip-path="url(#breeze-release-management-line-60)">Publishes generated JSON sch
[...]
+</text><text class="breeze-release-management-r5" x="0" y="1484"
textLength="12.2"
clip-path="url(#breeze-release-management-line-60)">│</text><text
class="breeze-release-management-r4" x="24.4" y="1484" textLength="280.6"
clip-path="url(#breeze-release-management-line-60)">publish-schemas-to-s3  </text><text
class="breeze-release-management-r1" x="329.4" y="1484" textLength="1110.2"
clip-path="url(#breeze-release-management-line-60)">Publishes generated JSON sch
[...]
</text><text class="breeze-release-management-r5" x="0" y="1508.4"
textLength="12.2"
clip-path="url(#breeze-release-management-line-61)">│</text><text
class="breeze-release-management-r4" x="24.4" y="1508.4" textLength="280.6"
clip-path="url(#breeze-release-management-line-61)">verify-rc-by-pmc       </text><text
class="breeze-release-management-r1" x="329.4" y="1508.4" textLength="1110.2"
clip-path="url(#breeze-release-management-line-61)">[EXPERIMENTA [...]
</text><text class="breeze-release-management-r5" x="0" y="1532.8"
textLength="12.2"
clip-path="url(#breeze-release-management-line-62)">│</text><text
class="breeze-release-management-r4" x="24.4" y="1532.8" textLength="280.6"
clip-path="url(#breeze-release-management-line-62)">check-release-files    </text><text
class="breeze-release-management-r1" x="329.4" y="1532.8" textLength="1110.2"
clip-path="url(#breeze-release-management-line-62)">Verify that all&#
[...]
</text><text class="breeze-release-management-r5" x="0" y="1557.2"
textLength="12.2"
clip-path="url(#breeze-release-management-line-63)">│</text><text
class="breeze-release-management-r4" x="24.4" y="1557.2" textLength="280.6"
clip-path="url(#breeze-release-management-line-63)">mirror-base-images     </text><text
class="breeze-release-management-r1" x="329.4" y="1557.2" textLength="1110.2"
clip-path="url(#breeze-release-management-line-63)">Mirror the h [...]
diff --git a/dev/breeze/doc/images/output_release-management.txt
b/dev/breeze/doc/images/output_release-management.txt
index 98674687d9e..87687852912 100644
--- a/dev/breeze/doc/images/output_release-management.txt
+++ b/dev/breeze/doc/images/output_release-management.txt
@@ -1 +1 @@
-0b7200fc4be8b419c11762aff2314daf
+5fcf99342971550d24114e8346522643
diff --git
a/dev/breeze/doc/images/output_release-management_publish-schemas-to-s3.svg
b/dev/breeze/doc/images/output_release-management_publish-schemas-to-s3.svg
index 1f4a0c7ee77..ada760982a4 100644
--- a/dev/breeze/doc/images/output_release-management_publish-schemas-to-s3.svg
+++ b/dev/breeze/doc/images/output_release-management_publish-schemas-to-s3.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 513.5999999999999"
xmlns="http://www.w3.org/2000/svg";>
+<svg class="rich-terminal" viewBox="0 0 1482 538.0"
xmlns="http://www.w3.org/2000/svg";>
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -46,7 +46,7 @@
<defs>
<clipPath
id="breeze-release-management-publish-schemas-to-s3-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="462.59999999999997" />
+ <rect x="0" y="0" width="1463.0" height="487.0" />
</clipPath>
<clipPath id="breeze-release-management-publish-schemas-to-s3-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -102,9 +102,12 @@
<clipPath id="breeze-release-management-publish-schemas-to-s3-line-17">
<rect x="0" y="416.3" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-release-management-publish-schemas-to-s3-line-18">
+ <rect x="0" y="440.7" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="511.6" rx="8"/><text
class="breeze-release-management-publish-schemas-to-s3-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: release-management publish-schemas-to-s3</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="536" rx="8"/><text
class="breeze-release-management-publish-schemas-to-s3-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: release-management publish-schemas-to-s3</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -117,22 +120,23 @@
<text class="breeze-release-management-publish-schemas-to-s3-r1" x="1464"
y="20" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-0)">
</text><text class="breeze-release-management-publish-schemas-to-s3-r2"
x="12.2" y="44.4" textLength="73.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-1)">Usage:</text><text
class="breeze-release-management-publish-schemas-to-s3-r3" x="97.6" y="44.4"
textLength="573.4"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-1)">breeze release-management publish-schemas-to-s3</text><text
class="breeze-release-management-publish-schemas-to-s [...]
</text><text class="breeze-release-management-publish-schemas-to-s3-r1"
x="1464" y="68.8" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-2)">
-</text><text class="breeze-release-management-publish-schemas-to-s3-r1"
x="12.2" y="93.2" textLength="927.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-3)">Publishes generated JSON schema artifacts (Execution API, Supervisor) to S3.</text><text
class="breeze-release-management-publish-schemas-to-s3-r1" x="1464" y="93.2"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-3)">
+</text><text class="breeze-release-management-publish-schemas-to-s3-r1"
x="12.2" y="93.2" textLength="1049.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-3)">Publishes generated JSON schema artifacts (Execution API, Supervisor, YAML Dag) to S3.</text><text
class="breeze-release-management-publish-schemas-to-s3-r1" x="1464" y="93.2"
textLength="12.2" clip-path="url(#breeze-release-management-publish-schemas-to
[...]
</text><text class="breeze-release-management-publish-schemas-to-s3-r1"
x="1464" y="117.6" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-4)">
</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="142" textLength="24.4"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-5)">╭─</text><text
class="breeze-release-management-publish-schemas-to-s3-r5" x="24.4" y="142"
textLength="280.6"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-5)"> Publish schemas to S3 </text><text
class="breeze-release-management-publish-schemas-to-s3-r5" x="305" y="1 [...]
</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="166.4" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-6)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r4" x="61" y="166.4"
textLength="268.4"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-6)">--execution-api       </text><text
class="breeze-release-management-publish-schemas-to-s3-r1" x= [...]
</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="190.8" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-7)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r4" x="61" y="190.8"
textLength="268.4"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-7)">--supervisor          </text><text
class="breeze-release-management-publish-schem [...]
-</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="215.2" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-8)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r7" x="24.4" y="215.2"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-8)">*</text><text
class="breeze-release-management-publish-schemas-to-s3-r4" x="61" y="215.2"
textLength="268.4" clip-path="url(#breez [...]
-</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="239.6" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-9)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r1" x="353.8" y="239.6"
textLength="1085.8"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-9)">s3://live-docs-airflow-apache-org/schemas/. Each schema is written to     
[...]
-</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="264" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-10)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r8" x="353.8" y="264"
textLength="122"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-10)"><location></text><text
class="breeze-release-management-publish-schemas-to-s3-r1" x="475.8" y="264"
textLength="12.2" clip-pat [...]
-</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="288.4" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-11)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r6" x="353.8" y="288.4"
textLength="561.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-11)">(s3://live-docs-airflow-apache-org/schemas/ | </text><text
class="breeze-release-management-publish-schemas-to-s3-r5 [...]
-</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="312.8" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-12)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r6" x="353.8" y="312.8"
textLength="561.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-12)">s3://staging-docs-airflow-apache-org/schemas/)</text><text
class="breeze-release-management-publish-schemas-to-s3-r5" x="1451. [...]
-</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="337.2" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-13)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r4" x="61" y="337.2"
textLength="268.4"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-13)">--overwrite           </text><text
class="breeze-release-management-publis [...]
-</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="361.6" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-14)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r4" x="61" y="361.6"
textLength="268.4"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-14)">--dry-run             </text><text
class="breeze-release-managem [...]
-</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="386" textLength="1464"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-publish-schemas-to-s3-r1" x="1464" y="386"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-15)">
-</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="410.4" textLength="24.4"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-16)">╭─</text><text
class="breeze-release-management-publish-schemas-to-s3-r5" x="24.4" y="410.4"
textLength="195.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-16)"> Common options </text><text
class="breeze-release-management-publish-schemas-to-s3-r5" x="219.6" y="410.4"
tex [...]
-</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="434.8" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-17)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r4" x="24.4" y="434.8"
textLength="73.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-17)">--help</text><text
class="breeze-release-management-publish-schemas-to-s3-r10" x="122" y="434.8"
textLength="24.4" clip-path="ur [...]
-</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="459.2" textLength="1464"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-18)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-publish-schemas-to-s3-r1" x="1464" y="459.2"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-18)">
+</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="215.2" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-8)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r4" x="61" y="215.2"
textLength="268.4"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-8)">--dag                 </text><text
class="bree [...]
+</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="239.6" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-9)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r7" x="24.4" y="239.6"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-9)">*</text><text
class="breeze-release-management-publish-schemas-to-s3-r4" x="61" y="239.6"
textLength="268.4" clip-path="url(#breez [...]
+</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="264" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-10)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r1" x="353.8" y="264"
textLength="1085.8"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-10)">s3://live-docs-airflow-apache-org/schemas/. Each schema is written to      
[...]
+</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="288.4" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-11)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r8" x="353.8" y="288.4"
textLength="122"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-11)"><location></text><text
class="breeze-release-management-publish-schemas-to-s3-r1" x="475.8" y="288.4"
textLength="12.2" cl [...]
+</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="312.8" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-12)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r6" x="353.8" y="312.8"
textLength="561.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-12)">(s3://live-docs-airflow-apache-org/schemas/ | </text><text
class="breeze-release-management-publish-schemas-to-s3-r5 [...]
+</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="337.2" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-13)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r6" x="353.8" y="337.2"
textLength="561.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-13)">s3://staging-docs-airflow-apache-org/schemas/)</text><text
class="breeze-release-management-publish-schemas-to-s3-r5" x="1451. [...]
+</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="361.6" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-14)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r4" x="61" y="361.6"
textLength="268.4"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-14)">--overwrite           </text><text
class="breeze-release-management-publis [...]
+</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="386" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-15)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r4" x="61" y="386"
textLength="268.4"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-15)">--dry-run             </text><text
class="breeze-release-management- [...]
+</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="410.4" textLength="1464"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-16)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-publish-schemas-to-s3-r1" x="1464" y="410.4"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-16)">
+</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="434.8" textLength="24.4"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-17)">╭─</text><text
class="breeze-release-management-publish-schemas-to-s3-r5" x="24.4" y="434.8"
textLength="195.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-17)"> Common options </text><text
class="breeze-release-management-publish-schemas-to-s3-r5" x="219.6" y="434.8"
tex [...]
+</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="459.2" textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-18)">│</text><text
class="breeze-release-management-publish-schemas-to-s3-r4" x="24.4" y="459.2"
textLength="73.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-18)">--help</text><text
class="breeze-release-management-publish-schemas-to-s3-r10" x="122" y="459.2"
textLength="24.4" clip-path="ur [...]
+</text><text class="breeze-release-management-publish-schemas-to-s3-r5" x="0"
y="483.6" textLength="1464"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-19)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-release-management-publish-schemas-to-s3-r1" x="1464" y="483.6"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-schemas-to-s3-line-19)">
</text>
</g>
</g>
diff --git
a/dev/breeze/doc/images/output_release-management_publish-schemas-to-s3.txt
b/dev/breeze/doc/images/output_release-management_publish-schemas-to-s3.txt
index 3217f6210b9..87cf6ff7e02 100644
--- a/dev/breeze/doc/images/output_release-management_publish-schemas-to-s3.txt
+++ b/dev/breeze/doc/images/output_release-management_publish-schemas-to-s3.txt
@@ -1 +1 @@
-60c42737a26b1a9363f13b85d7589a47
+2ed2141b68ce75e8ecbee93dba705535
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 39473330fb7..4cbd3a1f22a 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -4911,7 +4911,7 @@ def publish_docs_to_s3(
@release_management_group.command(
name="publish-schemas-to-s3",
- help="Publishes generated JSON schema artifacts (Execution API,
Supervisor) to S3.",
+ help="Publishes generated JSON schema artifacts (Execution API,
Supervisor, YAML Dag) to S3.",
)
@click.option(
"--execution-api",
@@ -4923,6 +4923,11 @@ def publish_docs_to_s3(
help="Path to the generated Supervisor schema JSON file.",
type=click.Path(exists=True, dir_okay=False, path_type=Path),
)
[email protected](
+ "--dag",
+ help="Path to the generated YAML Dag schema JSON file.",
+ type=click.Path(exists=True, dir_okay=False, path_type=Path),
+)
@click.option(
"--destination-location",
help="S3 location to publish the schemas under, e.g. "
@@ -4944,14 +4949,15 @@ def publish_docs_to_s3(
def publish_schemas_to_s3(
execution_api: Path | None,
supervisor: Path | None,
+ dag: Path | None,
destination_location: str,
overwrite: bool,
dry_run: bool,
):
from airflow_breeze.utils.publish_docs_to_s3 import publish_schemas_to_s3
as _publish_schemas_to_s3
- if not execution_api and not supervisor:
- console_print("[error]Provide at least one of --execution-api or
--supervisor[/]")
+ if not execution_api and not supervisor and not dag:
+ console_print("[error]Provide at least one of --execution-api,
--supervisor or --dag[/]")
sys.exit(1)
console_print("[info]Publishing schemas to S3[/]")
@@ -4960,11 +4966,14 @@ def publish_schemas_to_s3(
console_print(f"[info]Execution API schema: {execution_api}[/]")
if supervisor:
console_print(f"[info]Supervisor schema: {supervisor}[/]")
+ if dag:
+ console_print(f"[info]YAML Dag schema: {dag}[/]")
_publish_schemas_to_s3(
destination_location=destination_location.rstrip("/"),
execution_api_schema=execution_api,
supervisor_schema=supervisor,
+ dag_schema=dag,
overwrite=overwrite,
dry_run=dry_run,
)
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
index 42a9761c2af..8c2a3850935 100644
---
a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
+++
b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
@@ -593,6 +593,7 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str
| list[str]]]] = {
"options": [
"--execution-api",
"--supervisor",
+ "--dag",
"--destination-location",
"--overwrite",
"--dry-run",
diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
b/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
index 507983583ce..3158c2d42b0 100644
--- a/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
+++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py
@@ -398,6 +398,7 @@ class S3DocsPublish:
SCHEMA_VERSION_FIELDS: dict[str, str] = {
"execution-api": "info.version",
"supervisor-schema": "api_version",
+ "dag": "version",
}
@@ -479,6 +480,7 @@ def publish_schemas_to_s3(
destination_location: str,
execution_api_schema: pathlib.Path | None,
supervisor_schema: pathlib.Path | None,
+ dag_schema: pathlib.Path | None = None,
overwrite: bool = False,
dry_run: bool = False,
) -> None:
@@ -494,9 +496,11 @@ def publish_schemas_to_s3(
to_publish.append(("execution-api", execution_api_schema))
if supervisor_schema:
to_publish.append(("supervisor-schema", supervisor_schema))
+ if dag_schema:
+ to_publish.append(("dag", dag_schema))
if not to_publish:
- console_print("[error]No schema files provided. Pass --execution-api
and/or --supervisor.\n")
+ console_print("[error]No schema files provided. Pass --execution-api,
--supervisor and/or --dag.\n")
sys.exit(1)
uploaded_any = False
diff --git a/scripts/ci/prek/generate_yaml_importer_schema.py
b/scripts/ci/prek/generate_yaml_importer_schema.py
index e785ea9826d..3eb12e14100 100755
--- a/scripts/ci/prek/generate_yaml_importer_schema.py
+++ b/scripts/ci/prek/generate_yaml_importer_schema.py
@@ -49,6 +49,7 @@ def build_schema() -> str:
head = get_bundle().versions[0].value # newest-first
schema = DagDocument.model_json_schema(by_alias=True)
schema["$schema"] = "https://json-schema.org/draft/2020-12/schema";
+ schema["version"] = str(head)
schema["title"] = f"Airflow YAML Dag ({head})"
return json.dumps(schema, indent=2) + "\n"
diff --git a/task-sdk/src/airflow/sdk/importers/yaml_importer/importer.py
b/task-sdk/src/airflow/sdk/importers/yaml_importer/importer.py
index 080e0fc97af..a6daea67ad3 100644
--- a/task-sdk/src/airflow/sdk/importers/yaml_importer/importer.py
+++ b/task-sdk/src/airflow/sdk/importers/yaml_importer/importer.py
@@ -28,6 +28,7 @@ objects they describe. It is not turned on by default; enable
with::
from __future__ import annotations
import logging
+import re
import traceback
from typing import TYPE_CHECKING
@@ -55,10 +56,10 @@ if TYPE_CHECKING:
log = logging.getLogger(__name__)
-# The sniff matches the canonical Dag ``$schema`` URL, not a bare ``$schema``,
so a JSON-Schema
-# file (or any other YAML declaring a $schema) in the bundle is not queued and
then stuck as a
-# parse error. Reuses the migrator's public prefix to stay in sync.
-_DAG_SCHEMA_PREFIX = DAG_SCHEMA_PREFIX.encode()
+# Sniff a file for the canonical $schema declaration. The URL is matched only
as
+# a valid YAML or JSON key-value format (e.g. $schema: <url>) to reduce false
+# positives. The migrator's public prefix is reused to keep checks in sync.
+_DAG_SCHEMA_SNIFF_RE = re.compile(rb'\$schema["\']?\s*:\s*["\']?' +
re.escape(DAG_SCHEMA_PREFIX.encode()))
class YamlDagImporter(AbstractDagImporter[FileDagDefinition]):
@@ -154,11 +155,12 @@ class
YamlDagImporter(AbstractDagImporter[FileDagDefinition]):
def might_contain_dag(self, definition: DagDefinition, safe_mode: bool) ->
bool:
"""
- Cheap sniff: the file declares a ``$schema`` set to the canonical Dag
URL.
+ Cheap snif ensuring the file declares a reasonable ``$schema``.
- Both are required so a file that merely mentions the URL (without a
``$schema`` declaration)
- is skipped. The sniff stays lenient past that -- a malformed Dag still
reaches the parser and
- gets a clear error rather than silently disappearing. Keep the file if
the read fails.
+ This matches the URL as the ``$schema`` value, and skips a file that
+ does not have such a key or only mentions the URL in another field. A
+ malformed Dag may still reach the parser and get a clear error, rather
+ than silently disappearing.
"""
if not safe_mode:
return True
@@ -166,4 +168,4 @@ class
YamlDagImporter(AbstractDagImporter[FileDagDefinition]):
data = definition.read_bytes()
except OSError:
return True
- return b"$schema" in data and _DAG_SCHEMA_PREFIX in data
+ return _DAG_SCHEMA_SNIFF_RE.search(data) is not None
diff --git a/task-sdk/src/airflow/sdk/importers/yaml_importer/schema.json
b/task-sdk/src/airflow/sdk/importers/yaml_importer/schema.json
index 67686d202ae..59825504670 100644
--- a/task-sdk/src/airflow/sdk/importers/yaml_importer/schema.json
+++ b/task-sdk/src/airflow/sdk/importers/yaml_importer/schema.json
@@ -413,5 +413,6 @@
],
"title": "Airflow YAML Dag (2026-10-30)",
"type": "object",
- "$schema": "https://json-schema.org/draft/2020-12/schema";
+ "$schema": "https://json-schema.org/draft/2020-12/schema";,
+ "version": "2026-10-30"
}
diff --git a/task-sdk/tests/task_sdk/importers/yaml_importer/test_importer.py
b/task-sdk/tests/task_sdk/importers/yaml_importer/test_importer.py
index 092c47c7f3d..b3d237a6c80 100644
--- a/task-sdk/tests/task_sdk/importers/yaml_importer/test_importer.py
+++ b/task-sdk/tests/task_sdk/importers/yaml_importer/test_importer.py
@@ -160,19 +160,50 @@ def test_get_source_code(mock_bundle):
assert "$schema" in src.source_code
-def test_might_contain_dag(mock_bundle):
+_DRAFT_SCHEMA = "https://json-schema.org/draft/2020-12/schema";
+
+
[email protected](
+ ("case", "body", "expected"),
+ [
+ # Recognized: $schema declares the canonical Dag URL, in any of its
spellings.
+ ("yaml", f"$schema: {HEAD_SCHEMA}\ndag_id: d\ntasks: []\n", True),
+ ("yaml_quoted_value", f'$schema: "{HEAD_SCHEMA}"\n', True),
+ ("yaml_quoted_key", f'"$schema": {HEAD_SCHEMA}\n', True),
+ ("yaml_space_before_colon", f"$schema : {HEAD_SCHEMA}\n", True),
+ ("yaml_not_first_line", f"dag_id: d\n$schema: {HEAD_SCHEMA}\n", True),
+ ("json", f'{{"$schema": "{HEAD_SCHEMA}", "dag_id": "d", "tasks":
[]}}', True),
+ # Skipped: no $schema declaration naming the canonical Dag URL.
+ ("no_schema", "just: config\n", False),
+ ("other_schema_url", f"$schema: {_DRAFT_SCHEMA}\n", False),
+ ("url_only_no_schema", f"note: see {HEAD_SCHEMA}\n", False),
+ # The URL sits in another field while $schema is something else -- the
published snapshot's shape.
+ ("url_in_other_field", f"$schema: {_DRAFT_SCHEMA}\n$id:
{HEAD_SCHEMA}\n", False),
+ # The word "$schema" appears in prose, not as a declaration whose
value is the URL.
+ ("schema_word_in_prose", f"description: set $schema at
{HEAD_SCHEMA}\n", False),
+ # A different key that merely starts with "$schema".
+ ("different_schema_key", f"$schema_version: {HEAD_SCHEMA}\n", False),
+ ],
+)
+def test_might_contain_dag_sniff(mock_bundle, case, body, expected):
+ importer = YamlDagImporter()
+ definition = _def(mock_bundle, f"{case}.yaml", body)
+ assert importer.might_contain_dag(definition, safe_mode=True) is expected
+
+
+def test_might_contain_dag_skips_sniff_without_safe_mode(mock_bundle):
+ # Without safe mode the sniff is bypassed and every file is kept; the
parser reports non-Dags.
+ importer = YamlDagImporter()
+ definition = _def(mock_bundle, "config.yaml", "just: config\n")
+ assert importer.might_contain_dag(definition, safe_mode=False)
+
+
+def test_might_contain_dag_keeps_unreadable_file(mock_bundle):
+ # A file that cannot be read is kept rather than silently dropped.
importer = YamlDagImporter()
- has = _def(mock_bundle, "has.yaml", f"$schema: {HEAD_SCHEMA}\ndag_id:
d\ntasks: []\n")
- nope = _def(mock_bundle, "nope.yaml", "just: config\n")
- # A $schema that is not the canonical Dag URL (e.g. a JSON-Schema file)
must not be picked up.
- other = _def(mock_bundle, "schema.yaml", "$schema:
https://json-schema.org/draft/2020-12/schema\n";)
- # The canonical URL merely mentioned, without a $schema declaration, must
not be picked up either.
- mention = _def(mock_bundle, "mention.yaml", f"note: see {HEAD_SCHEMA}\n")
- assert importer.might_contain_dag(has, safe_mode=True)
- assert not importer.might_contain_dag(nope, safe_mode=True)
- assert not importer.might_contain_dag(other, safe_mode=True)
- assert not importer.might_contain_dag(mention, safe_mode=True)
- assert importer.might_contain_dag(nope, safe_mode=False) # sniff skipped
+ definition = _def(mock_bundle, "gone.yaml", f"$schema: {HEAD_SCHEMA}\n")
+ definition.path.unlink()
+ assert importer.might_contain_dag(definition, safe_mode=True)
def test_list_dag_definitions_skips_non_dag_yaml(mock_bundle):
