This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 53f2a9b1b817bcd3a9e83e3d5e2f5053a0f9b598 Author: Jarek Potiuk <ja...@potiuk.com> AuthorDate: Mon Feb 19 13:33:18 2024 +0100 Synchronize missing changes in breeze with main and regenerate images --- Dockerfile.ci | 2 +- ...release-management_verify-provider-packages.svg | 94 ++++++++++++---------- ...release-management_verify-provider-packages.txt | 2 +- dev/breeze/src/airflow_breeze/breeze.py | 2 - .../src/airflow_breeze/commands/ci_commands.py | 2 + .../airflow_breeze/commands/ci_image_commands.py | 2 +- .../common_package_installation_options.py | 8 ++ .../airflow_breeze/commands/developer_commands.py | 6 +- .../commands/production_image_commands.py | 2 +- .../commands/release_candidate_command.py | 28 +------ .../commands/release_management_commands.py | 27 ++++--- .../commands/release_management_commands_config.py | 1 + dev/breeze/src/airflow_breeze/global_constants.py | 10 +-- .../src/airflow_breeze/params/shell_params.py | 2 + .../prepare_providers/provider_documentation.py | 6 +- .../prepare_providers/provider_packages.py | 6 +- .../airflow_breeze/utils/docker_command_utils.py | 1 + dev/breeze/src/airflow_breeze/utils/packages.py | 4 +- .../src/airflow_breeze/utils/reproducible.py | 2 + dev/breeze/src/airflow_breeze/utils/run_utils.py | 24 ------ .../src/airflow_breeze/utils/selective_checks.py | 2 - pyproject.toml | 1 - scripts/docker/entrypoint_ci.sh | 2 +- .../in_container/install_airflow_and_providers.py | 13 ++- 24 files changed, 117 insertions(+), 132 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 09c4fe45c5..bc4533c38f 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -908,7 +908,7 @@ function check_pydantic() { echo echo "${COLOR_YELLOW}Remove pydantic and 3rd party libraries that depend on it${COLOR_RESET}" echo - pip uninstall --root-user-action ignore pydantic aws-sam-translator openai pyiceberg qdrant-client cfn-lint -y + pip uninstall --root-user-action ignore pydantic aws-sam-translator openai pyiceberg cfn-lint -y pip check elif [[ ${PYDANTIC=} == "v1" ]]; then echo diff --git a/dev/breeze/doc/images/output_release-management_verify-provider-packages.svg b/dev/breeze/doc/images/output_release-management_verify-provider-packages.svg index 71011b5580..85e55ff94d 100644 --- a/dev/breeze/doc/images/output_release-management_verify-provider-packages.svg +++ b/dev/breeze/doc/images/output_release-management_verify-provider-packages.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 1343.1999999999998" xmlns="http://www.w3.org/2000/svg"> +<svg class="rich-terminal" viewBox="0 0 1482 1416.3999999999999" xmlns="http://www.w3.org/2000/svg"> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -43,7 +43,7 @@ <defs> <clipPath id="breeze-release-management-verify-provider-packages-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="1292.1999999999998" /> + <rect x="0" y="0" width="1463.0" height="1365.3999999999999" /> </clipPath> <clipPath id="breeze-release-management-verify-provider-packages-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -201,9 +201,18 @@ <clipPath id="breeze-release-management-verify-provider-packages-line-51"> <rect x="0" y="1245.9" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-release-management-verify-provider-packages-line-52"> + <rect x="0" y="1270.3" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-verify-provider-packages-line-53"> + <rect x="0" y="1294.7" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-release-management-verify-provider-packages-line-54"> + <rect x="0" y="1319.1" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="1341.2" rx="8"/><text class="breeze-release-management-verify-provider-packages-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management verify-provider-packages</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="1414.4" rx="8"/><text class="breeze-release-management-verify-provider-packages-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management verify-provider-packages</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -228,44 +237,47 @@ </text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-12)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-12)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="312.8" textLength="85.4" cli [...] </text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="337.2" textLength="1464" clip-path="url(#breeze-release-management-verify-provider-packages-line-13)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-verify-provider-packages-r1" x="1464" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-13)"> </text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="361.6" textLength="24.4" clip-path="url(#breeze-release-management-verify-provider-packages-line-14)">╭─</text><text class="breeze-release-management-verify-provider-packages-r5" x="24.4" y="361.6" textLength="512.4" clip-path="url(#breeze-release-management-verify-provider-packages-line-14)"> Installing packages after entering shell </text><text class="breeze-release-managem [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-15)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-15)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="386" textLength="97.6" clip-path [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-16)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="475.8" y="410.4" textLength="902.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-16)">(TEXT)                   [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-17)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-17)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="434.8" textLength="97.6" cli [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-18)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="475.8" y="459.2" textLength="866.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-18)">(constraints-source-providers | constraints | constraints-no-providers)</text><text class="bre [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-19)">│</text><text class="breeze-release-management-verify-provider-packages-r5" x="475.8" y="483.6" textLength="866.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-19)">[default: constraints-source-providers]            [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-20)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="508" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-20)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="508" textLength="97.6" clip-path [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-21)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="475.8" y="532.4" textLength="963.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-21)">constraints URL).               &# [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-22)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="475.8" y="556.8" textLength="963.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-22)">(TEXT)                   [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-23)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-23)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="581.2" textLength="97.6" cli [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-24)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-24)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="605.6" textLength="97.6" cli [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-25)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="630" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-25)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="630" textLength="97.6" clip-path [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-26)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="475.8" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-26)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="488" y="654.4" textLength="48.8" cli [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-27)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="475.8" y="678.8" textLength="963.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-27)">(TEXT)                   [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-28)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-28)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="703.2" textLength="97.6" cli [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-29)">│</text><text class="breeze-release-management-verify-provider-packages-r5" x="475.8" y="727.6" textLength="658.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-29)">[default: wheel]                [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="752" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-30)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="752" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-30)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="752" textLength="122" clip-path= [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="776.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-31)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="475.8" y="776.4" textLength="927.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-31)">(TEXT)                   [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-32)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-32)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="800.8" textLength="122" clip [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="825.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-33)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="475.8" y="825.2" textLength="866.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-33)">(constraints-source-providers | constraints | constraints-no-providers)</text><text class="bre [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="849.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-34)">│</text><text class="breeze-release-management-verify-provider-packages-r5" x="475.8" y="849.6" textLength="866.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-34)">[default: constraints-source-providers]            [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="874" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-35)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="874" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-35)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="874" textLength="122" clip-path= [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="898.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-36)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="475.8" y="898.4" textLength="963.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-36)">constraints URL). Can be 'default' in which case the defaul [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="922.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-37)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="475.8" y="922.8" textLength="963.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-37)">constraints-reference is used.            &# [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="947.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-38)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="475.8" y="947.2" textLength="963.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-38)">(TEXT)                   [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="971.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-39)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="971.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-39)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="971.6" textLength="122" clip [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="996" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-40)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="996" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-40)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="996" textLength="48.8" clip-path [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1020.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-41)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="475.8" y="1020.4" textLength="963.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-41)">install from PyPI), `none`, `wheel`, or `sdist` to install from [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1044.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-42)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="475.8" y="1044.8" textLength="963.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-42)">or VCS URL to install from (https://pip.pypa.io/en/stable/topics/vcs-support/).</t [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1069.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-43)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="475.8" y="1069.2" textLength="97.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-43)">Implies </text><text class="breeze-release-management-verify-provider-packages-r4" x="573.4" y="1069.2" text [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1093.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-44)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="475.8" y="1093.6" textLength="963.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-44)">(none | wheel | sdist | <airflow_version>)      [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1118" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-45)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="1118" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-45)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="1118" textLength="48.8" clip-p [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1142.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-46)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="475.8" y="1142.4" textLength="963.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-46)">folder when entering breeze.            [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1166.8" textLength="1464" clip-path="url(#breeze-release-management-verify-provider-packages-line-47)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-verify-provider-packages-r1" x="1464" y="1166.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-47)"> -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1191.2" textLength="24.4" clip-path="url(#breeze-release-management-verify-provider-packages-line-48)">╭─</text><text class="breeze-release-management-verify-provider-packages-r5" x="24.4" y="1191.2" textLength="195.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-48)"> Common options </text><text class="breeze-release-management-verify-provider-packages-r5" x="219 [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1215.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-49)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="1215.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-49)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="1215.6" textLength="48.8" [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1240" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-50)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="1240" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-50)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="1240" textLength="97.6" clip-p [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1264.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-51)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="1264.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-51)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="1264.4" textLength="61" cl [...] -</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1288.8" textLength="1464" clip-path="url(#breeze-release-management-verify-provider-packages-line-52)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-verify-provider-packages-r1" x="1464" y="1288.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-52)"> +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-15)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-15)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="386" textLength="97.6" clip-path [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-16)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="488" y="410.4" textLength="902.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-16)">(TEXT)                  &# [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-17)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-17)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="434.8" textLength="97.6" cli [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-18)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="488" y="459.2" textLength="866.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-18)">(constraints-source-providers | constraints | constraints-no-providers)</text><text class="breez [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-19)">│</text><text class="breeze-release-management-verify-provider-packages-r5" x="488" y="483.6" textLength="866.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-19)">[default: constraints-source-providers]             [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-20)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="508" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-20)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="508" textLength="97.6" clip-path [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-21)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="488" y="532.4" textLength="951.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-21)">constraints URL).                [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-22)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="488" y="556.8" textLength="951.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-22)">(TEXT)                  &# [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-23)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-23)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="581.2" textLength="97.6" cli [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-24)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-24)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="605.6" textLength="97.6" cli [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-25)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="630" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-25)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="630" textLength="97.6" clip-path [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-26)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="488" y="654.4" textLength="951.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-26)">or airflow version.               [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-27)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="678.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-27)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="678.8" textLength="97.6" cli [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-28)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="488" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-28)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="500.2" y="703.2" textLength="48.8" cli [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-29)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="488" y="727.6" textLength="951.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-29)">(TEXT)                  &# [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="752" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-30)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="752" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-30)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="752" textLength="97.6" clip-path [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="776.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-31)">│</text><text class="breeze-release-management-verify-provider-packages-r5" x="488" y="776.4" textLength="658.8" clip-path="url(#breeze-release-management-verify-provider-packages-line-31)">[default: wheel]                 [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-32)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-32)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="800.8" textLength="122" clip [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="825.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-33)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="488" y="825.2" textLength="927.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-33)">(TEXT)                  &# [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="849.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-34)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="849.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-34)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="849.6" textLength="122" clip [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="874" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-35)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="488" y="874" textLength="866.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-35)">(constraints-source-providers | constraints | constraints-no-providers)</text><text class="breeze-re [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="898.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-36)">│</text><text class="breeze-release-management-verify-provider-packages-r5" x="488" y="898.4" textLength="866.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-36)">[default: constraints-source-providers]             [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="922.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-37)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="922.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-37)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="922.8" textLength="122" clip [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="947.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-38)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="488" y="947.2" textLength="951.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-38)">constraints URL). Can be 'default' in which case the default& [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="971.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-39)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="488" y="971.6" textLength="951.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-39)">constraints-reference is used.             [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="996" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-40)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="488" y="996" textLength="951.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-40)">(TEXT)                    [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1020.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-41)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="1020.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-41)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="1020.4" textLength="122" c [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1044.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-42)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="1044.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-42)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="1044.8" textLength="48.8" [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1069.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-43)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="488" y="1069.2" textLength="951.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-43)">install from PyPI), `none`, `wheel`, or `sdist` to install from  [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1093.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-44)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="488" y="1093.6" textLength="951.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-44)">or VCS URL to install from           [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1118" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-45)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="488" y="1118" textLength="744.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-45)">(https://pip.pypa.io/en/stable/topics/vcs-support/). Implies </text><text class="breeze-release-management-v [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1142.4" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-46)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="488" y="1142.4" textLength="951.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-46)">`remove`.                  [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1166.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-47)">│</text><text class="breeze-release-management-verify-provider-packages-r7" x="488" y="1166.8" textLength="951.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-47)">(none | wheel | sdist | <airflow_version>)       [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1191.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-48)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="1191.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-48)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="1191.2" textLength="48.8" [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1215.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-49)">│</text><text class="breeze-release-management-verify-provider-packages-r1" x="488" y="1215.6" textLength="951.6" clip-path="url(#breeze-release-management-verify-provider-packages-line-49)">folder when entering breeze.             [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1240" textLength="1464" clip-path="url(#breeze-release-management-verify-provider-packages-line-50)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-verify-provider-packages-r1" x="1464" y="1240" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-50)"> +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1264.4" textLength="24.4" clip-path="url(#breeze-release-management-verify-provider-packages-line-51)">╭─</text><text class="breeze-release-management-verify-provider-packages-r5" x="24.4" y="1264.4" textLength="195.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-51)"> Common options </text><text class="breeze-release-management-verify-provider-packages-r5" x="219 [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1288.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-52)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="1288.8" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-52)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="1288.8" textLength="48.8" [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1313.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-53)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="1313.2" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-53)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="1313.2" textLength="97.6" [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1337.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-54)">│</text><text class="breeze-release-management-verify-provider-packages-r4" x="24.4" y="1337.6" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-54)">-</text><text class="breeze-release-management-verify-provider-packages-r4" x="36.6" y="1337.6" textLength="61" cl [...] +</text><text class="breeze-release-management-verify-provider-packages-r5" x="0" y="1362" textLength="1464" clip-path="url(#breeze-release-management-verify-provider-packages-line-55)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-verify-provider-packages-r1" x="1464" y="1362" textLength="12.2" clip-path="url(#breeze-release-management-verify-provider-packages-line-55)"> </text> </g> </g> diff --git a/dev/breeze/doc/images/output_release-management_verify-provider-packages.txt b/dev/breeze/doc/images/output_release-management_verify-provider-packages.txt index 4d34c31928..291f5e80ba 100644 --- a/dev/breeze/doc/images/output_release-management_verify-provider-packages.txt +++ b/dev/breeze/doc/images/output_release-management_verify-provider-packages.txt @@ -1 +1 @@ -fbada2db7324ba53ba7a30c7b75bbda2 +7242dbb1309e32f1bb5f606da8c62ed1 diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py index 069c347525..b0b8879303 100755 --- a/dev/breeze/src/airflow_breeze/breeze.py +++ b/dev/breeze/src/airflow_breeze/breeze.py @@ -42,7 +42,6 @@ from airflow_breeze.commands.sbom_commands import sbom # noqa from airflow_breeze.commands.setup_commands import setup # noqa from airflow_breeze.commands.testing_commands import group_for_testing # noqa - main.add_command(group_for_testing) main.add_command(kubernetes_group) main.add_command(ci_group) @@ -52,6 +51,5 @@ main.add_command(setup) main.add_command(release_management) main.add_command(sbom) - if __name__ == "__main__": main() diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py index e7df1b51f0..5ed450f3c4 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py @@ -325,6 +325,8 @@ class WorkflowInfo(NamedTuple): and (self.ref_name == "main" or TEST_BRANCH_MATCHER.match(self.ref_name)) ): return "true" + if "canary" in self.pull_request_labels and self.head_repo == "apache/airflow": + return "true" return "false" def run_coverage(self) -> str: diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 188ea647ff..103d4dafd5 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -619,7 +619,7 @@ def verify( ) if (pull or image_name) and run_in_parallel: get_console().print( - "[error]You cannot use --pull,--image-name and --run-in-parallel at the same time. " "Exiting[/]" + "[error]You cannot use --pull,--image-name and --run-in-parallel at the same time. Exiting[/]" ) sys.exit(1) if run_in_parallel: diff --git a/dev/breeze/src/airflow_breeze/commands/common_package_installation_options.py b/dev/breeze/src/airflow_breeze/commands/common_package_installation_options.py index d0b85e82e8..d5cd788db4 100644 --- a/dev/breeze/src/airflow_breeze/commands/common_package_installation_options.py +++ b/dev/breeze/src/airflow_breeze/commands/common_package_installation_options.py @@ -61,6 +61,14 @@ option_airflow_skip_constraints = click.option( help="Do not use constraints when installing airflow.", envvar="AIRFLOW_SKIP_CONSTRAINTS", ) +option_install_airflow_with_constraints = click.option( + "--install-airflow-with-constraints", + is_flag=True, + default=False, + show_default=True, + envvar="INSTALL_AIRFLOW_WITH_CONSTRAINTS", + help="Install airflow in a separate step, with constraints determined from package or airflow version.", +) option_install_selected_providers = click.option( "--install-selected-providers", help="Comma-separated list of providers selected to be installed (implies --use-packages-from-dist).", diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 2d529a743c..cbf0c4ee54 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -669,7 +669,11 @@ def build_docs( fix_ownership_using_docker() if result.returncode == 0: get_console().print( - "[info]Start the webserver in breeze and view the built docs at http://localhost:28080/docs/[/]" + "[info]To view the built documentation, you have two options:\n\n" + "1. Start the webserver in breeze and access the built docs at " + "http://localhost:28080/docs/\n" + "2. Alternatively, you can run ./docs/start_doc_server.sh for a lighter resource option and view" + "the built docs at http://localhost:8000" ) sys.exit(result.returncode) diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 1266085c79..a98462c163 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -549,7 +549,7 @@ def verify( ) if (pull or image_name) and run_in_parallel: get_console().print( - "[error]You cannot use --pull,--image-name and --run-in-parallel at the same time. " "Exiting[/]" + "[error]You cannot use --pull,--image-name and --run-in-parallel at the same time. Exiting[/]" ) sys.exit(1) if run_in_parallel: diff --git a/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py b/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py index 4441943978..16f3dd18a9 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py +++ b/dev/breeze/src/airflow_breeze/commands/release_candidate_command.py @@ -274,31 +274,6 @@ def push_release_candidate_tag_to_github(version): console_print("[success]Release candidate tag pushed to GitHub") -def create_issue_for_testing(version, previous_version, github_token): - if confirm_action("Do you want to create issue for testing? Only applicable for patch release"): - console_print() - console_print("Create issue in github for testing the release using this subject:") - console_print() - console_print(f"Status of testing of Apache Airflow {version}") - console_print() - if CI: - run_command(["git", "fetch"], check=True) - if confirm_action("Print the issue body?"): - run_command( - [ - "./dev/prepare_release_issue.py", - "generate-issue-content", - "--previous-release", - f"{previous_version}", - "--current-release", - f"{version}", - "--github-token", - f"{github_token}", - ], - check=True, - ) - - def remove_old_releases(version, repo_root): if confirm_action("In beta release we do not remove old RCs. Is this a beta release?"): return @@ -414,7 +389,7 @@ def publish_release_candidate(version, previous_version, github_token): if confirm_action("Use docker to create artifacts?"): create_artifacts_with_docker() elif confirm_action("Use hatch to create artifacts?"): - create_artifacts_with_hatch() + create_artifacts_with_hatch(source_date_epoch) # Sign the release sign_the_release(airflow_repo_root) # Tag and push constraints @@ -442,7 +417,6 @@ def publish_release_candidate(version, previous_version, github_token): push_release_candidate_tag_to_github(version) # Create issue for testing os.chdir(airflow_repo_root) - create_issue_for_testing(version, previous_version, github_token) console_print() console_print("Done!") diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index ec16ac6bc0..5a966b25d4 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -68,6 +68,7 @@ from airflow_breeze.commands.common_package_installation_options import ( option_airflow_constraints_mode_update, option_airflow_constraints_reference, option_airflow_skip_constraints, + option_install_airflow_with_constraints, option_install_selected_providers, option_providers_constraints_location, option_providers_constraints_mode_ci, @@ -209,6 +210,7 @@ class VersionedFile(NamedTuple): suffix: str type: str comparable_version: Version + file_name: str AIRFLOW_PIP_VERSION = "24.0" @@ -217,12 +219,13 @@ GITPYTHON_VERSION = "3.1.40" RICH_VERSION = "13.7.0" NODE_VERSION = "21.2.0" PRE_COMMIT_VERSION = "3.5.0" +HATCH_VERSION = "1.9.1" PYYAML_VERSION = "6.0.1" AIRFLOW_BUILD_DOCKERFILE = f""" FROM python:{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}-slim-{ALLOWED_DEBIAN_VERSIONS[0]} RUN apt-get update && apt-get install -y --no-install-recommends git -RUN pip install pip=={AIRFLOW_PIP_VERSION} hatch==1.9.1 pyyaml=={PYYAML_VERSION}\ +RUN pip install pip=={AIRFLOW_PIP_VERSION} hatch=={HATCH_VERSION} pyyaml=={PYYAML_VERSION}\ gitpython=={GITPYTHON_VERSION} rich=={RICH_VERSION} pre-commit=={PRE_COMMIT_VERSION} COPY . /opt/airflow """ @@ -703,7 +706,7 @@ def basic_provider_checks(provider_package_id: str) -> dict[str, Any]: f"since you asked for it, it will be built [/]\n" ) elif provider_metadata.get("state") == "suspended": - get_console().print(f"[warning]The package: {provider_package_id} is suspended " f"skipping it [/]\n") + get_console().print(f"[warning]The package: {provider_package_id} is suspended skipping it [/]\n") raise PackageSuspendedException() return provider_metadata @@ -1018,7 +1021,7 @@ def generate_constraints( list_generated_constraints(output=None) -SDIST_FILENAME_PREFIX = "apache-airflow-providers-" +SDIST_FILENAME_PREFIX = "apache_airflow_providers_" WHEEL_FILENAME_PREFIX = "apache_airflow_providers-" SDIST_FILENAME_PATTERN = re.compile(rf"{SDIST_FILENAME_PREFIX}(.*)-[0-9].*\.tar\.gz") @@ -1032,7 +1035,7 @@ def _get_all_providers_in_dist( matched = filename_pattern.match(file.name) if not matched: raise Exception(f"Cannot parse provider package name from {file.name}") - provider_package_id = matched.group(1).replace("-", ".") + provider_package_id = matched.group(1).replace("_", ".") yield provider_package_id @@ -1164,7 +1167,7 @@ def install_provider_packages( provider_chunks = [chunk for chunk in provider_chunks if chunk] if not provider_chunks: get_console().print("[info]No providers to install") - return + sys.exit(1) total_num_providers = 0 for index, chunk in enumerate(provider_chunks): get_console().print(f"Chunk {index}: {chunk} ({len(chunk)} providers)") @@ -1233,6 +1236,7 @@ def install_provider_packages( @option_airflow_skip_constraints @option_dry_run @option_github_repository +@option_install_airflow_with_constraints @option_install_selected_providers @option_installation_package_format @option_mount_sources @@ -1250,6 +1254,7 @@ def verify_provider_packages( airflow_constraints_reference: str, airflow_extras: str, github_repository: str, + install_airflow_with_constraints: bool, install_selected_providers: str, mount_sources: str, package_format: str, @@ -1275,6 +1280,7 @@ def verify_provider_packages( airflow_extras=airflow_extras, airflow_skip_constraints=airflow_skip_constraints, github_repository=github_repository, + install_airflow_with_constraints=install_airflow_with_constraints, mount_sources=mount_sources, package_format=package_format, providers_constraints_location=providers_constraints_location, @@ -1572,10 +1578,10 @@ def clean_old_provider_artifacts( # Leave only last version from each type for versioned_file in package_types[:-1]: get_console().print( - f"""[warning]Removing {versioned_file.base + versioned_file.version + - versioned_file.suffix} as they are older than remaining file""" + f"[warning]Removing {versioned_file.file_name} as they are older than remaining file: " + f"{package_types[-1].file_name}[/]" ) - command = ["svn", "rm", versioned_file.base + versioned_file.version + versioned_file.suffix] + command = ["svn", "rm", versioned_file.file_name] run_command(command, check=False) @@ -1700,7 +1706,7 @@ def release_prod_images( "--push", ] run_command(docker_buildx_command) - if python == DEFAULT_PYTHON_MAJOR_MINOR_VERSION: + if python == ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[-1]: alias_image( slim_image_name, f"{dockerhub_repo}:slim-{airflow_version}", @@ -1732,7 +1738,7 @@ def release_prod_images( "--push", ] run_command(docker_buildx_command) - if python == DEFAULT_PYTHON_MAJOR_MINOR_VERSION: + if python == ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[-1]: alias_image(image_name, f"{dockerhub_repo}:{airflow_version}") # in case of re-tagging the images might need few seconds to refresh multi-platform images in DockerHub time.sleep(10) @@ -2267,6 +2273,7 @@ def split_version_and_suffix(file_name: str, suffix: str) -> VersionedFile: suffix=suffix, type=no_version_file + "-" + suffix, comparable_version=Version(version), + file_name=file_name, ) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index ac7bbecb13..7afbda1242 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -118,6 +118,7 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = { "--airflow-constraints-reference", "--airflow-extras", "--airflow-skip-constraints", + "--install-airflow-with-constraints", "--install-selected-providers", "--package-format", "--providers-constraints-location", diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 8398b51d11..33d11fecde 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -370,15 +370,7 @@ def get_airflow_extras(): # Initialize integrations -AVAILABLE_INTEGRATIONS = [ - "cassandra", - "kerberos", - "mongo", - "pinot", - "celery", - "statsd", - "trino", -] +AVAILABLE_INTEGRATIONS = ["cassandra", "kerberos", "mongo", "pinot", "celery", "statsd", "trino", "qdrant"] ALL_PROVIDER_YAML_FILES = Path(AIRFLOW_SOURCES_ROOT, "airflow", "providers").rglob("provider.yaml") PROVIDER_RUNTIME_DATA_SCHEMA_PATH = AIRFLOW_SOURCES_ROOT / "airflow" / "provider_info.schema.json" diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 1d1c785f08..b3ff492cec 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -162,6 +162,7 @@ class ShellParams: image_tag: str | None = None include_mypy_volume: bool = False install_airflow_version: str = "" + install_airflow_with_constraints: bool = False install_selected_providers: str | None = None integration: tuple[str, ...] = () issue_id: str = "" @@ -526,6 +527,7 @@ class ShellParams: _set_var(_env, "HOST_OS", self.host_os) _set_var(_env, "HOST_USER_ID", self.host_user_id) _set_var(_env, "INIT_SCRIPT_FILE", None, "init.sh") + _set_var(_env, "INSTALL_AIRFLOW_WITH_CONSTRAINTS", self.install_airflow_with_constraints) _set_var(_env, "INSTALL_AIRFLOW_VERSION", self.install_airflow_version) _set_var(_env, "INSTALL_SELECTED_PROVIDERS", self.install_selected_providers) _set_var(_env, "ISSUE_ID", self.issue_id) diff --git a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py index 00bb2203a0..34407d7c3e 100644 --- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py +++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py @@ -348,7 +348,7 @@ def _get_all_changes_for_package( ) else: get_console().print( - f"[info]New version of the '{provider_package_id}' " f"package is ready to be released!\n" + f"[info]New version of the '{provider_package_id}' package is ready to be released!\n" ) next_version_tag = f"{HTTPS_REMOTE}/{base_branch}" changes_table = "" @@ -413,7 +413,7 @@ def _ask_the_user_for_the_type_of_changes(non_interactive: bool) -> TypeOfChange if given_answer in type_of_changes_array: return TypeOfChange(given_answer) get_console().print( - f"[warning] Wrong answer given: '{given_answer}'. " f"Should be one of {display_answers}" + f"[warning] Wrong answer given: '{given_answer}'. Should be one of {display_answers}" ) @@ -680,7 +680,7 @@ def update_release_notes( answer = user_confirm(f"Provider {provider_package_id} marked for release. Proceed?") if answer == Answer.NO: get_console().print( - f"\n[warning]Skipping provider: {provider_package_id} " f"on user request![/]\n" + f"\n[warning]Skipping provider: {provider_package_id} on user request![/]\n" ) raise PrepareReleaseDocsUserSkippedException() elif answer == Answer.QUIT: diff --git a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py index fa3a1b6845..88ad3e8c8c 100644 --- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py +++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py @@ -86,7 +86,7 @@ def copy_provider_sources_to_target(provider_id: str) -> Path: relative_provider_path = source_provider_sources_path.relative_to(AIRFLOW_SOURCES_ROOT) target_providers_sub_folder = target_provider_root_path / relative_provider_path get_console().print( - f"[info]Copying provider sources: " f"{source_provider_sources_path} -> {target_providers_sub_folder}" + f"[info]Copying provider sources: {source_provider_sources_path} -> {target_providers_sub_folder}" ) copytree(source_provider_sources_path, target_providers_sub_folder) shutil.copy(AIRFLOW_SOURCES_ROOT / "LICENSE", target_providers_sub_folder / "LICENSE") @@ -214,9 +214,7 @@ def build_provider_package(provider_id: str, target_provider_root_sources_path: except subprocess.CalledProcessError as ex: get_console().print("[error]The command returned an error %s", ex) raise PrepareReleasePackageErrorBuildingPackageException() - get_console().print( - f"\n[info]Prepared provider package {provider_id} in " f"format {package_format}[/]\n" - ) + get_console().print(f"\n[info]Prepared provider package {provider_id} in format {package_format}[/]\n") def move_built_packages_and_cleanup( diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index b6f2cc37c2..8615abffc1 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -568,6 +568,7 @@ def fix_ownership_using_docker(quiet: bool = False): f"DOCKER_IS_ROOTLESS={is_docker_rootless()}", "-e", f"VERBOSE_COMMANDS={str(not quiet).lower()}", + "--rm", "-t", OWNERSHIP_CLEANUP_DOCKER_TAG, "/opt/airflow/scripts/in_container/run_fix_ownership.py", diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index a6eb386148..307b4cc9c0 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -310,7 +310,7 @@ def get_short_package_name(long_form_provider: str) -> str: else: if not long_form_provider.startswith(LONG_PROVIDERS_PREFIX): raise ValueError( - f"Invalid provider name: {long_form_provider}. " f"Should start with {LONG_PROVIDERS_PREFIX}" + f"Invalid provider name: {long_form_provider}. Should start with {LONG_PROVIDERS_PREFIX}" ) return long_form_provider[len(LONG_PROVIDERS_PREFIX) :].replace("-", ".") @@ -651,7 +651,7 @@ def make_sure_remote_apache_exists_and_fetch(github_repository: str = "apache/ai ) else: get_console().print( - f"[error]Error {ex}[/]\n" f"[error]When checking if {HTTPS_REMOTE} is set.[/]\n\n" + f"[error]Error {ex}[/]\n[error]When checking if {HTTPS_REMOTE} is set.[/]\n\n" ) sys.exit(1) get_console().print("[info]Fetching full history and tags from remote.") diff --git a/dev/breeze/src/airflow_breeze/utils/reproducible.py b/dev/breeze/src/airflow_breeze/utils/reproducible.py index e60e374648..418d5b2079 100644 --- a/dev/breeze/src/airflow_breeze/utils/reproducible.py +++ b/dev/breeze/src/airflow_breeze/utils/reproducible.py @@ -139,6 +139,8 @@ def repack_deterministically( arcname = entry if prepend_path is not None: arcname = os.path.normpath(os.path.join(prepend_path, arcname)) + if arcname == ".": + continue if arcname.startswith("./"): arcname = arcname[2:] tar_file.add(entry, filter=reset, recursive=False, arcname=arcname) diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index c4389a9962..f3956bc07d 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -33,8 +33,6 @@ from typing import Mapping, Union from rich.markup import escape -from airflow_breeze.branch_defaults import AIRFLOW_BRANCH -from airflow_breeze.global_constants import APACHE_AIRFLOW_GITHUB_REPOSITORY from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.console import Output, get_console from airflow_breeze.utils.path_utils import ( @@ -377,28 +375,6 @@ def check_if_image_exists(image: str) -> bool: return cmd_result.returncode == 0 -def get_ci_image_for_pre_commits() -> str: - github_repository = os.environ.get("GITHUB_REPOSITORY", APACHE_AIRFLOW_GITHUB_REPOSITORY) - python_version = "3.8" - airflow_image = f"ghcr.io/{github_repository}/{AIRFLOW_BRANCH}/ci/python{python_version}" - skip_image_pre_commits = os.environ.get("SKIP_IMAGE_PRE_COMMITS", "false") - if skip_image_pre_commits[0].lower() == "t": - get_console().print( - f"[info]Skipping image check as SKIP_IMAGE_PRE_COMMITS is set to {skip_image_pre_commits}[/]" - ) - sys.exit(0) - if not check_if_image_exists( - image=airflow_image, - ): - get_console().print(f"[red]The image {airflow_image} is not available.[/]\n") - get_console().print( - f"\n[yellow]Please run this to fix it:[/]\n\n" - f"breeze ci-image build --python {python_version}\n\n" - ) - sys.exit(1) - return airflow_image - - def _run_compile_internally(command_to_execute: list[str], dev: bool) -> RunCommandResult: from filelock import SoftFileLock, Timeout diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 4f9a383a36..a98478d137 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - from __future__ import annotations import difflib @@ -992,7 +991,6 @@ class SelectiveChecks: "check-airflow-provider-compatibility", "check-extra-packages-references", "check-provider-yaml-valid", - "validate-operators-init", "lint-helm-chart", "validate-operators-init", ) diff --git a/pyproject.toml b/pyproject.toml index fa264b9f1f..2f2badf5e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1302,7 +1302,6 @@ extend-select = [ "D412", "D419", "TCH", # Rules around TYPE_CHECKING blocks - "TID251", # Specific modules or module members that may not be imported or accessed "TID253", # Ban certain modules from being imported at module level ] ignore = [ diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 875d0b2291..d6a6b56540 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -236,7 +236,7 @@ function check_pydantic() { echo echo "${COLOR_YELLOW}Remove pydantic and 3rd party libraries that depend on it${COLOR_RESET}" echo - pip uninstall --root-user-action ignore pydantic aws-sam-translator openai pyiceberg qdrant-client cfn-lint -y + pip uninstall --root-user-action ignore pydantic aws-sam-translator openai pyiceberg cfn-lint -y pip check elif [[ ${PYDANTIC=} == "v1" ]]; then echo diff --git a/scripts/in_container/install_airflow_and_providers.py b/scripts/in_container/install_airflow_and_providers.py index cbbf87992c..d3bf247639 100755 --- a/scripts/in_container/install_airflow_and_providers.py +++ b/scripts/in_container/install_airflow_and_providers.py @@ -420,6 +420,14 @@ ALLOWED_CONSTRAINTS_MODE = ["constraints-source-providers", "constraints", "cons envvar="USE_PACKAGES_FROM_DIST", help="Should install packages from dist folder if set.", ) +@click.option( + "--install-airflow-with-constraints", + is_flag=True, + default=False, + show_default=True, + envvar="INSTALL_AIRFLOW_WITH_CONSTRAINTS", + help="Install airflow in a separate step, with constraints determined from package or airflow version.", +) def install_airflow_and_providers( airflow_constraints_mode: str, airflow_constraints_location: str, @@ -438,6 +446,7 @@ def install_airflow_and_providers( python_version: str, use_airflow_version: str, use_packages_from_dist: bool, + install_airflow_with_constraints: bool, ): console.print("[bright_blue]Installing Airflow and Providers") installation_spec = find_installation_spec( @@ -458,7 +467,7 @@ def install_airflow_and_providers( use_airflow_version=use_airflow_version, use_packages_from_dist=use_packages_from_dist, ) - if installation_spec.airflow_package: + if installation_spec.airflow_package and install_airflow_with_constraints: install_airflow_cmd = [ "pip", "install", @@ -474,6 +483,8 @@ def install_airflow_and_providers( run_command(install_airflow_cmd, github_actions=github_actions, check=True) if installation_spec.provider_packages or not install_airflow_with_constraints: install_providers_cmd = ["pip", "install", "--root-user-action", "ignore"] + if not install_airflow_with_constraints and installation_spec.airflow_package: + install_providers_cmd.append(installation_spec.airflow_package) console.print("\n[bright_blue]Installing provider packages:") for provider_package in sorted(installation_spec.provider_packages): console.print(f" {provider_package}")