This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new 7e5c827472 Improve breeze command image generation and consistency check (#35169) 7e5c827472 is described below commit 7e5c827472732c6dba2bb55a80b21617c59d88a5 Author: Jarek Potiuk <ja...@potiuk.com> AuthorDate: Wed Oct 25 11:47:17 2023 +0200 Improve breeze command image generation and consistency check (#35169) * Improve breeze command image generation and consistency check There is no more need to regenerate the images manually via `breeze setup regenerate-command-images`. Also check for consistency of command configuration vs command parameters in Breeze is now done in pre-commit. This means that anyone making changes to Breeze and having pre-commit installed will have two steps less to keep the --help images generated and to keep consistency of flags in breeze commands. * Update .pre-commit-config.yaml --- .github/workflows/ci.yml | 1 - .../commands/setup_commands_config.py | 1 + images/breeze/output-commands-hash.txt | 2 +- images/breeze/output_setup.svg | 28 ++++++--------- .../ci/pre_commit/pre_commit_breeze_cmd_line.py | 41 +++++++++++++++++++--- 5 files changed, 48 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 779b3a46bb..3dd00e2094 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -455,7 +455,6 @@ jobs: echo "Please run 'breeze setup version' and commit the changes." && false) - run: python -m pytest -n auto --color=yes working-directory: ./dev/breeze/ - - run: breeze setup check-all-params-in-groups tests-www: timeout-minutes: 10 diff --git a/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py b/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py index 463305a716..1da86bb56a 100644 --- a/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/setup_commands_config.py @@ -23,6 +23,7 @@ SETUP_COMMANDS: dict[str, str | list[str]] = { "self-upgrade", "cleanup", "config", + "check-all-params-in-groups", "regenerate-command-images", "command-hash-export", "version", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 790bab4e7a..283ca94693 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -62,7 +62,7 @@ setup:config:0b95b129703e11a99b8ddd05a0dd6b35 setup:regenerate-command-images:ea2fba3440bd4e84311a53abe6e8dc56 setup:self-upgrade:4af905a147fcd6670a0e33d3d369a94b setup:version:be116d90a21c2afe01087f7609774e1e -setup:000b1d18baa7f1a740a1afb6a6bfe246 +setup:9dff88972f7c2ba4da2ad47f77ba2d06 shell:619e9de6bb9e6eb26acab06c8aedb183 start-airflow:c238330495a57851030415e34042173b static-checks:19926b8fcea5784b28d4a0d99865363c diff --git a/images/breeze/output_setup.svg b/images/breeze/output_setup.svg index 0864a65f00..9b165e4524 100644 --- a/images/breeze/output_setup.svg +++ b/images/breeze/output_setup.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 489.2" xmlns="http://www.w3.org/2000/svg"> +<svg class="rich-terminal" viewBox="0 0 1482 440.4" xmlns="http://www.w3.org/2000/svg"> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -42,7 +42,7 @@ <defs> <clipPath id="breeze-setup-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="438.2" /> + <rect x="0" y="0" width="1463.0" height="389.4" /> </clipPath> <clipPath id="breeze-setup-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -89,15 +89,9 @@ <clipPath id="breeze-setup-line-14"> <rect x="0" y="343.1" width="1464" height="24.65"/> </clipPath> -<clipPath id="breeze-setup-line-15"> - <rect x="0" y="367.5" width="1464" height="24.65"/> - </clipPath> -<clipPath id="breeze-setup-line-16"> - <rect x="0" y="391.9" width="1464" height="24.65"/> - </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="487.2" rx="8"/><text class="breeze-setup-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: setup</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="438.4" rx="8"/><text class="breeze-setup-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: setup</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -116,15 +110,13 @@ </text><text class="breeze-setup-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-setup-line-6)">│</text><text class="breeze-setup-r4" x="24.4" y="166.4" textLength="12.2" clip-path="url(#breeze-setup-line-6)">-</text><text class="breeze-setup-r4" x="36.6" y="166.4" textLength="61" clip-path="url(#breeze-setup-line-6)">-help</text><text class="breeze-setup-r6" x="122" y="166.4" textLength="24.4" clip-path="url(#breeze-setup-line-6)">-h</text><text class="breeze-setup-r1" x="1 [...] </text><text class="breeze-setup-r5" x="0" y="190.8" textLength="1464" clip-path="url(#breeze-setup-line-7)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-setup-r1" x="1464" y="190.8" textLength="12.2" clip-path="url(#breeze-setup-line-7)"> </text><text class="breeze-setup-r5" x="0" y="215.2" textLength="24.4" clip-path="url(#breeze-setup-line-8)">╭─</text><text class="breeze-setup-r5" x="24.4" y="215.2" textLength="85.4" clip-path="url(#breeze-setup-line-8)"> Setup </text><text class="breeze-setup-r5" x="109.8" y="215.2" textLength="1329.8" clip-path="url(#breeze-setup-line-8)">─────────────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class="breeze-s [...] -</text><text class="breeze-setup-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-setup-line-9)">│</text><text class="breeze-setup-r4" x="24.4" y="239.6" textLength="390.4" clip-path="url(#breeze-setup-line-9)">autocomplete                    </text><text class="breeze-setup-r1" x="439.2" y="239.6" textLength="1000.4" clip-path="url(#breeze-setup-line-9)">Enables autocompl [...] -</text><text class="breeze-setup-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-setup-line-10)">│</text><text class="breeze-setup-r4" x="24.4" y="264" textLength="390.4" clip-path="url(#breeze-setup-line-10)">self-upgrade                    </text><text class="breeze-setup-r1" x="439.2" y="264" textLength="1000.4" clip-path="url(#breeze-setup-line-10)">Self upgrade Br [...] -</text><text class="breeze-setup-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-setup-line-11)">│</text><text class="breeze-setup-r4" x="24.4" y="288.4" textLength="390.4" clip-path="url(#breeze-setup-line-11)">config                          </text><text class="breeze-setup-r1" x="439.2" y="288.4" textLength="1000.4" clip-path="url(#breeze-setup [...] -</text><text class="breeze-setup-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-setup-line-12)">│</text><text class="breeze-setup-r4" x="24.4" y="312.8" textLength="390.4" clip-path="url(#breeze-setup-line-12)">regenerate-command-images       </text><text class="breeze-setup-r1" x="439.2" y="312.8" textLength="1000.4" clip-path="url(#breeze-setup-line-12)">Regenerate breeze command images.       [...] -</text><text class="breeze-setup-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-setup-line-13)">│</text><text class="breeze-setup-r4" x="24.4" y="337.2" textLength="390.4" clip-path="url(#breeze-setup-line-13)">version                         </text><text class="breeze-setup-r1" x="439.2" y="337.2" textLength="1000.4" clip-path="url(#breeze-setup-line [...] -</text><text class="breeze-setup-r5" x="0" y="361.6" textLength="1464" clip-path="url(#breeze-setup-line-14)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-setup-r1" x="1464" y="361.6" textLength="12.2" clip-path="url(#breeze-setup-line-14)"> -</text><text class="breeze-setup-r5" x="0" y="386" textLength="24.4" clip-path="url(#breeze-setup-line-15)">╭─</text><text class="breeze-setup-r5" x="24.4" y="386" textLength="122" clip-path="url(#breeze-setup-line-15)"> Commands </text><text class="breeze-setup-r5" x="146.4" y="386" textLength="1293.2" clip-path="url(#breeze-setup-line-15)">──────────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class="breeze-setup [...] -</text><text class="breeze-setup-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-setup-line-16)">│</text><text class="breeze-setup-r4" x="24.4" y="410.4" textLength="524.6" clip-path="url(#breeze-setup-line-16)">check-all-params-in-groups                 </text><text class="breeze-setup-r1" x="573.4" y="410.4" textLength="866.2" clip-path="url(#breeze-setup-line-16)">Check that all [...] -</text><text class="breeze-setup-r5" x="0" y="434.8" textLength="1464" clip-path="url(#breeze-setup-line-17)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-setup-r1" x="1464" y="434.8" textLength="12.2" clip-path="url(#breeze-setup-line-17)"> +</text><text class="breeze-setup-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-setup-line-9)">│</text><text class="breeze-setup-r4" x="24.4" y="239.6" textLength="402.6" clip-path="url(#breeze-setup-line-9)">autocomplete                     </text><text class="breeze-setup-r1" x="451.4" y="239.6" textLength="988.2" clip-path="url(#breeze-setup-line-9)">Enables auto [...] +</text><text class="breeze-setup-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-setup-line-10)">│</text><text class="breeze-setup-r4" x="24.4" y="264" textLength="402.6" clip-path="url(#breeze-setup-line-10)">self-upgrade                     </text><text class="breeze-setup-r1" x="451.4" y="264" textLength="988.2" clip-path="url(#breeze-setup-line-10)">Self upgrade [...] +</text><text class="breeze-setup-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-setup-line-11)">│</text><text class="breeze-setup-r4" x="24.4" y="288.4" textLength="402.6" clip-path="url(#breeze-setup-line-11)">config                           </text><text class="breeze-setup-r1" x="451.4" y="288.4" textLength="988.2" clip-path="url(#breeze- [...] +</text><text class="breeze-setup-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-setup-line-12)">│</text><text class="breeze-setup-r4" x="24.4" y="312.8" textLength="402.6" clip-path="url(#breeze-setup-line-12)">check-all-params-in-groups       </text><text class="breeze-setup-r1" x="451.4" y="312.8" textLength="988.2" clip-path="url(#breeze-setup-line-12)">Check that all parameters are put in groups.  [...] +</text><text class="breeze-setup-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-setup-line-13)">│</text><text class="breeze-setup-r4" x="24.4" y="337.2" textLength="402.6" clip-path="url(#breeze-setup-line-13)">regenerate-command-images        </text><text class="breeze-setup-r1" x="451.4" y="337.2" textLength="988.2" clip-path="url(#breeze-setup-line-13)">Regenerate breeze command images.     & [...] +</text><text class="breeze-setup-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-setup-line-14)">│</text><text class="breeze-setup-r4" x="24.4" y="361.6" textLength="402.6" clip-path="url(#breeze-setup-line-14)">version                          </text><text class="breeze-setup-r1" x="451.4" y="361.6" textLength="988.2" clip-path="url(#breeze-setup [...] +</text><text class="breeze-setup-r5" x="0" y="386" textLength="1464" clip-path="url(#breeze-setup-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-setup-r1" x="1464" y="386" textLength="12.2" clip-path="url(#breeze-setup-line-15)"> </text> </g> </g> diff --git a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py index 198488a119..63c9429189 100755 --- a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py +++ b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py @@ -19,6 +19,8 @@ from __future__ import annotations import os +import shutil +import subprocess import sys from pathlib import Path from subprocess import call @@ -76,11 +78,40 @@ def is_regeneration_needed() -> bool: if __name__ == "__main__": + return_code = 0 verify_all_commands_described_in_docs() if is_regeneration_needed(): - console.print("\n[bright_blue]Some of the commands changed since last time images were generated.\n") - console.print( - "\n[red]Image generation is needed. Please run this command:\n\n" - "[magenta]breeze setup regenerate-command-images\n" + return_code = 1 + if shutil.which("breeze") is None: + console.print("\n[red]Breeze command configuration has changed.\n") + console.print( + "[yellow]The `breeze` command is not on path. " + "Skipping regeneration of images and consistency check." + ) + console.print( + "\n[bright_blue]Some of the commands changed since last time images were generated.\n" + ) + console.print("\n[yellow]You should install it and run those commands manually:\n") + console.print("\n[magenta]breeze setup regenerate-command images\n") + console.print("\n[magenta]breeze setup check-all-params-in-groups\n") + sys.exit(return_code) + res = subprocess.run( + ["breeze", "setup", "regenerate-command-images"], + check=False, ) - sys.exit(1) + if res.returncode != 0: + console.print("\n[red]Breeze command configuration has changed.\n") + console.print("\n[bright_blue]Images have been regenerated.\n") + console.print("\n[bright_blue]You might want to run it manually:\n") + console.print("\n[magenta]]breeze setup regenerate-command images\n") + res = subprocess.run( + ["breeze", "setup", "check-all-params-in-groups"], + check=False, + ) + if res.returncode != 0: + return_code = 1 + console.print("\n[red]Breeze command configuration has changed.\n") + console.print("\n[yellow]Please fix it in the appropriate command_*_config.py file\n") + console.print("\n[bright_blue]You can run consistency check manually by running:\n") + console.print("\nbreeze setup check-all-params-in-groups\n") + sys.exit(return_code)