This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-7-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 6d62f3febeaf213ffc11b1a47dcbeb7292ed711e Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Sep 12 11:49:56 2023 +0200 Update min-sqlalchemy version to account for latest features used (#34293) Some of the recent sqlalchemy changes are not working with minimum version of sqlalchemy of ours - for example `where` syntax does not allow moe than one clause and we are already passing more in _do_delete_old_records (added in #33527). This syntax however was added in SQL Alchemy 1.4.28 and our minimum version was 1.4.27. This change bumps the minimum SQLAlchemy version to 1.4.28 but it also adds a special test job that only runs on Postgres that downgrades the SQLAlchemy to the minimum supported version (retrieved from setup.cfg). This way, we will be able to detect such incompatible changes at the PR time. This is a new flag `--downgrade-sqlalchemy` on test command that works similar to earlier `--upgrade-boto`. We also enable the `--upgrade-boto` and `--downgrade-sqlalchemy` flags to be used for `breeze shell` command - thanks to that we can easily test both flags with `breeze shell` command. (cherry picked from commit efbead9fe7462b3634b6d9c842bd9a7ac78a0207) --- .github/workflows/ci.yml | 49 ++++++++++++++++++++++ Dockerfile.ci | 23 ++++++---- .../airflow_breeze/commands/developer_commands.py | 8 ++++ .../commands/developer_commands_config.py | 7 ++++ .../airflow_breeze/commands/testing_commands.py | 12 +++--- .../commands/testing_commands_config.py | 1 + .../src/airflow_breeze/params/shell_params.py | 1 + .../src/airflow_breeze/utils/common_options.py | 12 ++++++ .../airflow_breeze/utils/docker_command_utils.py | 2 + images/breeze/output-commands-hash.txt | 6 +-- images/breeze/output_shell.svg | 34 +++++++++++---- images/breeze/output_testing_tests.svg | 26 +++++++----- scripts/ci/docker-compose/_docker.env | 1 + scripts/ci/docker-compose/base.yml | 1 + scripts/ci/docker-compose/devcontainer.env | 1 + scripts/docker/entrypoint_ci.sh | 23 ++++++---- setup.cfg | 2 +- 17 files changed, 165 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ca7543136..39ed3ac08d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1021,6 +1021,55 @@ jobs: uses: ./.github/actions/post_tests_failure if: failure() + tests-postgres-min-sqlalchemy: + timeout-minutes: 130 + name: > + MinSQLAlchemy${{needs.build-info.outputs.default-postgres-version}}, + Py${{needs.build-info.outputs.default-python-version}}: + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} + runs-on: "${{needs.build-info.outputs.runs-on}}" + needs: [build-info, wait-for-ci-images] + env: + RUNS_ON: "${{needs.build-info.outputs.runs-on}}" + PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" + PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" + FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" + DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" + BACKEND: "postgres" + PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" + PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" + POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" + BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" + DOWNGRADE_SQLALCHEMY: "true" + JOB_ID: > + postgres-min-sqlalchemy-${{needs.build-info.outputs.default-python-version}}- + ${{needs.build-info.outputs.default-postgres-version}} + COVERAGE: "${{needs.build-info.outputs.run-coverage}}" + if: needs.build-info.outputs.run-tests == 'true' + steps: + - name: Cleanup repo + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v3 + with: + persist-credentials: false + - name: > + Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} + uses: ./.github/actions/prepare_breeze_and_image + - name: > + Tests: ${{needs.build-info.outputs.default-python-version}}: + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} + run: breeze testing tests --run-in-parallel + - name: > + Post Tests success: ${{needs.build-info.outputs.default-python-version}}:Boto" + uses: ./.github/actions/post_tests_success + if: success() + - name: > + Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:Boto" + uses: ./.github/actions/post_tests_failure + if: failure() + tests-postgres-in-progress-features-disabled: timeout-minutes: 130 name: > diff --git a/Dockerfile.ci b/Dockerfile.ci index 9092465c82..a0f47d86d3 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -937,6 +937,22 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then fi rm -f "${AIRFLOW_SOURCES}/pytest.ini" +if [[ ${UPGRADE_BOTO=} == "true" ]]; then + echo + echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" + echo + pip uninstall --root-user-action ignore aiobotocore -y || true + pip install --root-user-action ignore --upgrade boto3 botocore + pip check +fi +if [[ ${DOWNGRADE_SQLALCHEMY=} == "true" ]]; then + min_sqlalchemy_version=$(grep "sqlalchemy>=" setup.cfg | sed "s/.*>=\([0-9\.]*\).*/\1/") + echo + echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}" + echo + pip install --root-user-action ignore "sqlalchemy==${min_sqlalchemy_version}" + pip check +fi set +u if [[ "${RUN_TESTS}" != "true" ]]; then @@ -1166,13 +1182,6 @@ else exit 1 fi fi -if [[ ${UPGRADE_BOTO=} == "true" ]]; then - echo - echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" - echo - pip uninstall aiobotocore -y || true - pip install --upgrade boto3 botocore -fi readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ ALL_TESTS ALL_PRESELECTED_TESTS diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 92b3f7baa0..5c569ba811 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -51,6 +51,7 @@ from airflow_breeze.utils.common_options import ( option_celery_broker, option_celery_flower, option_db_reset, + option_downgrade_sqlalchemy, option_dry_run, option_executor, option_force_build, @@ -70,6 +71,7 @@ from airflow_breeze.utils.common_options import ( option_platform_single, option_postgres_version, option_python, + option_upgrade_boto, option_use_airflow_version, option_use_packages_from_dist, option_verbose, @@ -162,6 +164,8 @@ class TimerThread(threading.Thread): @option_image_tag_for_running @option_max_time @option_include_mypy_volume +@option_upgrade_boto +@option_downgrade_sqlalchemy @option_verbose @option_dry_run @option_github_repository @@ -197,6 +201,8 @@ def shell( celery_broker: str, celery_flower: bool, extra_args: tuple, + upgrade_boto: bool, + downgrade_sqlalchemy: bool, ): """Enter breeze environment. this is the default command use when no other is selected.""" if get_verbose() or get_dry_run(): @@ -234,6 +240,8 @@ def shell( executor=executor, celery_broker=celery_broker, celery_flower=celery_flower, + upgrade_boto=upgrade_boto, + downgrade_sqlalchemy=downgrade_sqlalchemy, ) sys.exit(result.returncode) diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py index 81c99e793f..9ef4f8ca96 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py @@ -99,6 +99,13 @@ DEVELOPER_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = { "--package-format", ], }, + { + "name": "Upgrading/downgrading selected packages", + "options": [ + "--upgrade-boto", + "--downgrade-sqlalchemy", + ], + }, ], "breeze compile-www-assets": [ { diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index d892727ee4..7acd6cffd6 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -38,6 +38,7 @@ from airflow_breeze.utils.common_options import ( option_backend, option_db_reset, option_debug_resources, + option_downgrade_sqlalchemy, option_dry_run, option_github_repository, option_image_name, @@ -52,6 +53,7 @@ from airflow_breeze.utils.common_options import ( option_python, option_run_in_parallel, option_skip_cleanup, + option_upgrade_boto, option_use_airflow_version, option_verbose, ) @@ -367,12 +369,8 @@ def run_tests_in_parallel( show_default=True, envvar="PARALLEL_TEST_TYPES", ) [email protected]( - "--upgrade-boto", - help="Remove aiobotocore and upgrade botocore and boto to the latest version.", - is_flag=True, - envvar="UPGRADE_BOTO", -) +@option_upgrade_boto +@option_downgrade_sqlalchemy @click.option( "--collect-only", help="Collect tests only, do not run them.", @@ -416,6 +414,7 @@ def command_for_tests( mount_sources: str, extra_pytest_args: tuple, upgrade_boto: bool, + downgrade_sqlalchemy: bool, collect_only: bool, remove_arm_packages: bool, github_repository: str, @@ -436,6 +435,7 @@ def command_for_tests( forward_ports=False, test_type=test_type, upgrade_boto=upgrade_boto, + downgrade_sqlalchemy=downgrade_sqlalchemy, collect_only=collect_only, remove_arm_packages=remove_arm_packages, github_repository=github_repository, diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py index 908843c224..84d0d2b6ec 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py @@ -56,6 +56,7 @@ TESTING_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = { "--use-airflow-version", "--mount-sources", "--upgrade-boto", + "--downgrade-sqlalchemy", "--remove-arm-packages", "--skip-docker-compose-down", ], diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 99bc8c4d3c..b2240a42c9 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -120,6 +120,7 @@ class ShellParams: dry_run: bool = False verbose: bool = False upgrade_boto: bool = False + downgrade_sqlalchemy: bool = False executor: str = START_AIRFLOW_DEFAULT_ALLOWED_EXECUTORS celery_broker: str = DEFAULT_CELERY_BROKER celery_flower: bool = False diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index f9a82064b8..5e8e50da61 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -601,3 +601,15 @@ option_eager_upgrade_additional_requirements = click.option( help="Optional additional requirements to upgrade eagerly to avoid backtracking " "(see `breeze ci find-backtracking-candidates`).", ) +option_upgrade_boto = click.option( + "--upgrade-boto", + help="Remove aiobotocore and upgrade botocore and boto to the latest version.", + is_flag=True, + envvar="UPGRADE_BOTO", +) +option_downgrade_sqlalchemy = click.option( + "--downgrade-sqlalchemy", + help="Downgrade SQLAlchemy to minimum supported version.", + is_flag=True, + envvar="DOWNGRADE_SQLALCHEMY", +) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index df6047c322..b0a5697b18 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -600,6 +600,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: set_value_to_default_if_not_set(env, "TEST_TYPE", "") set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60") set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false") + set_value_to_default_if_not_set(env, "DOWNGRADE_SQLALCHEMY", "false") set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false") set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false") set_value_to_default_if_not_set(env, "VERBOSE", "false") @@ -647,6 +648,7 @@ DERIVE_ENV_VARIABLES_FROM_ATTRIBUTES = { "SQLITE_URL": "sqlite_url", "START_AIRFLOW": "start_airflow", "UPGRADE_BOTO": "upgrade_boto", + "DOWNGRADE_SQLALCHEMY": "downgrade_sqlalchemy", "USE_AIRFLOW_VERSION": "use_airflow_version", "USE_PACKAGES_FROM_DIST": "use_packages_from_dist", "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index b15e25524a..09b6278bf9 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -62,11 +62,11 @@ setup:regenerate-command-images:d5e29ec6acb1a6af7d83772c2962f89d setup:self-upgrade:4af905a147fcd6670a0e33d3d369a94b setup:version:be116d90a21c2afe01087f7609774e1e setup:8de3ed2645928e8f9f89c58f0ccd2a60 -shell:30a9271ff59a0fe756dda402cc165683 +shell:1e901a677a6df6ba1d64d3fe79b42587 start-airflow:ff0f63e20b9ff454e5d3c7b9ba9080d7 static-checks:d319b1c7972a6623bc8ee1b174cacb48 testing:docker-compose-tests:0c810047fc66a0cfe91119e2d08b3507 testing:helm-tests:8e491da2e01ebd815322c37562059d77 testing:integration-tests:486e4d91449ecdb7630ef2a470d705a3 -testing:tests:3c202e65824e405269e78f58936980e0 -testing:68a089bc30e0e60f834f205df1e9f086 +testing:tests:b3b921fd5a7d3435a0ad34e90b75cb2f +testing:13325e047fc32d9e40b51bfd15212a91 diff --git a/images/breeze/output_shell.svg b/images/breeze/output_shell.svg index a8dd2ecd2e..7c8b47ed11 100644 --- a/images/breeze/output_shell.svg +++ b/images/breeze/output_shell.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 1611.6" xmlns="http://www.w3.org/2000/svg";> +<svg class="rich-terminal" viewBox="0 0 1482 1709.1999999999998" xmlns="http://www.w3.org/2000/svg";> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -43,7 +43,7 @@ <defs> <clipPath id="breeze-shell-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="1560.6" /> + <rect x="0" y="0" width="1463.0" height="1658.1999999999998" /> </clipPath> <clipPath id="breeze-shell-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -234,9 +234,21 @@ <clipPath id="breeze-shell-line-62"> <rect x="0" y="1514.3" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-shell-line-63"> + <rect x="0" y="1538.7" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-shell-line-64"> + <rect x="0" y="1563.1" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-shell-line-65"> + <rect x="0" y="1587.5" width="1464" height="24.65"/> + </clipPath> +<clipPath id="breeze-shell-line-66"> + <rect x="0" y="1611.9" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="1609.6" rx="8"/><text class="breeze-shell-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: shell</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="1707.2" rx="8"/><text class="breeze-shell-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: shell</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -304,12 +316,16 @@ </text><text class="breeze-shell-r5" x="0" y="1362" textLength="12.2" clip-path="url(#breeze-shell-line-55)">│</text><text class="breeze-shell-r4" x="24.4" y="1362" textLength="12.2" clip-path="url(#breeze-shell-line-55)">-</text><text class="breeze-shell-r4" x="36.6" y="1362" textLength="97.6" clip-path="url(#breeze-shell-line-55)">-package</text><text class="breeze-shell-r4" x="134.2" y="1362" textLength="85.4" clip-path="url(#breeze-shell-line-55)">-format</text><text class="breeze-sh [...] </text><text class="breeze-shell-r5" x="0" y="1386.4" textLength="12.2" clip-path="url(#breeze-shell-line-56)">│</text><text class="breeze-shell-r5" x="451.4" y="1386.4" textLength="658.8" clip-path="url(#breeze-shell-line-56)">[default: wheel]                                      </text><text cl [...] </text><text class="breeze-shell-r5" x="0" y="1410.8" textLength="1464" clip-path="url(#breeze-shell-line-57)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-shell-r2" x="1464" y="1410.8" textLength="12.2" clip-path="url(#breeze-shell-line-57)"> -</text><text class="breeze-shell-r5" x="0" y="1435.2" textLength="24.4" clip-path="url(#breeze-shell-line-58)">╭─</text><text class="breeze-shell-r5" x="24.4" y="1435.2" textLength="195.2" clip-path="url(#breeze-shell-line-58)"> Common options </text><text class="breeze-shell-r5" x="219.6" y="1435.2" textLength="1220" clip-path="url(#breeze-shell-line-58)">────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class [...] -</text><text class="breeze-shell-r5" x="0" y="1459.6" textLength="12.2" clip-path="url(#breeze-shell-line-59)">│</text><text class="breeze-shell-r4" x="24.4" y="1459.6" textLength="12.2" clip-path="url(#breeze-shell-line-59)">-</text><text class="breeze-shell-r4" x="36.6" y="1459.6" textLength="97.6" clip-path="url(#breeze-shell-line-59)">-verbose</text><text class="breeze-shell-r6" x="158.6" y="1459.6" textLength="24.4" clip-path="url(#breeze-shell-line-59)">-v</text><text class="breeze [...] -</text><text class="breeze-shell-r5" x="0" y="1484" textLength="12.2" clip-path="url(#breeze-shell-line-60)">│</text><text class="breeze-shell-r4" x="24.4" y="1484" textLength="12.2" clip-path="url(#breeze-shell-line-60)">-</text><text class="breeze-shell-r4" x="36.6" y="1484" textLength="48.8" clip-path="url(#breeze-shell-line-60)">-dry</text><text class="breeze-shell-r4" x="85.4" y="1484" textLength="48.8" clip-path="url(#breeze-shell-line-60)">-run</text><text class="breeze-shell-r6" [...] -</text><text class="breeze-shell-r5" x="0" y="1508.4" textLength="12.2" clip-path="url(#breeze-shell-line-61)">│</text><text class="breeze-shell-r4" x="24.4" y="1508.4" textLength="12.2" clip-path="url(#breeze-shell-line-61)">-</text><text class="breeze-shell-r4" x="36.6" y="1508.4" textLength="85.4" clip-path="url(#breeze-shell-line-61)">-answer</text><text class="breeze-shell-r6" x="158.6" y="1508.4" textLength="24.4" clip-path="url(#breeze-shell-line-61)">-a</text><text class="breeze- [...] -</text><text class="breeze-shell-r5" x="0" y="1532.8" textLength="12.2" clip-path="url(#breeze-shell-line-62)">│</text><text class="breeze-shell-r4" x="24.4" y="1532.8" textLength="12.2" clip-path="url(#breeze-shell-line-62)">-</text><text class="breeze-shell-r4" x="36.6" y="1532.8" textLength="61" clip-path="url(#breeze-shell-line-62)">-help</text><text class="breeze-shell-r6" x="158.6" y="1532.8" textLength="24.4" clip-path="url(#breeze-shell-line-62)">-h</text><text class="breeze-shel [...] -</text><text class="breeze-shell-r5" x="0" y="1557.2" textLength="1464" clip-path="url(#breeze-shell-line-63)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-shell-r2" x="1464" y="1557.2" textLength="12.2" clip-path="url(#breeze-shell-line-63)"> +</text><text class="breeze-shell-r5" x="0" y="1435.2" textLength="24.4" clip-path="url(#breeze-shell-line-58)">╭─</text><text class="breeze-shell-r5" x="24.4" y="1435.2" textLength="500.2" clip-path="url(#breeze-shell-line-58)"> Upgrading/downgrading selected packages </text><text class="breeze-shell-r5" x="524.6" y="1435.2" textLength="915" clip-path="url(#breeze-shell-line-58)">───────────────────────────────────────────────────────────────────────────</text><text c [...] +</text><text class="breeze-shell-r5" x="0" y="1459.6" textLength="12.2" clip-path="url(#breeze-shell-line-59)">│</text><text class="breeze-shell-r4" x="24.4" y="1459.6" textLength="12.2" clip-path="url(#breeze-shell-line-59)">-</text><text class="breeze-shell-r4" x="36.6" y="1459.6" textLength="97.6" clip-path="url(#breeze-shell-line-59)">-upgrade</text><text class="breeze-shell-r4" x="134.2" y="1459.6" textLength="61" clip-path="url(#breeze-shell-line-59)">-boto</text><text class="breez [...] +</text><text class="breeze-shell-r5" x="0" y="1484" textLength="12.2" clip-path="url(#breeze-shell-line-60)">│</text><text class="breeze-shell-r4" x="24.4" y="1484" textLength="12.2" clip-path="url(#breeze-shell-line-60)">-</text><text class="breeze-shell-r4" x="36.6" y="1484" textLength="122" clip-path="url(#breeze-shell-line-60)">-downgrade</text><text class="breeze-shell-r4" x="158.6" y="1484" textLength="134.2" clip-path="url(#breeze-shell-line-60)">-sqlalchemy</text><text class="bre [...] +</text><text class="breeze-shell-r5" x="0" y="1508.4" textLength="1464" clip-path="url(#breeze-shell-line-61)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-shell-r2" x="1464" y="1508.4" textLength="12.2" clip-path="url(#breeze-shell-line-61)"> +</text><text class="breeze-shell-r5" x="0" y="1532.8" textLength="24.4" clip-path="url(#breeze-shell-line-62)">╭─</text><text class="breeze-shell-r5" x="24.4" y="1532.8" textLength="195.2" clip-path="url(#breeze-shell-line-62)"> Common options </text><text class="breeze-shell-r5" x="219.6" y="1532.8" textLength="1220" clip-path="url(#breeze-shell-line-62)">────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class [...] +</text><text class="breeze-shell-r5" x="0" y="1557.2" textLength="12.2" clip-path="url(#breeze-shell-line-63)">│</text><text class="breeze-shell-r4" x="24.4" y="1557.2" textLength="12.2" clip-path="url(#breeze-shell-line-63)">-</text><text class="breeze-shell-r4" x="36.6" y="1557.2" textLength="97.6" clip-path="url(#breeze-shell-line-63)">-verbose</text><text class="breeze-shell-r6" x="158.6" y="1557.2" textLength="24.4" clip-path="url(#breeze-shell-line-63)">-v</text><text class="breeze [...] +</text><text class="breeze-shell-r5" x="0" y="1581.6" textLength="12.2" clip-path="url(#breeze-shell-line-64)">│</text><text class="breeze-shell-r4" x="24.4" y="1581.6" textLength="12.2" clip-path="url(#breeze-shell-line-64)">-</text><text class="breeze-shell-r4" x="36.6" y="1581.6" textLength="48.8" clip-path="url(#breeze-shell-line-64)">-dry</text><text class="breeze-shell-r4" x="85.4" y="1581.6" textLength="48.8" clip-path="url(#breeze-shell-line-64)">-run</text><text class="breeze-sh [...] +</text><text class="breeze-shell-r5" x="0" y="1606" textLength="12.2" clip-path="url(#breeze-shell-line-65)">│</text><text class="breeze-shell-r4" x="24.4" y="1606" textLength="12.2" clip-path="url(#breeze-shell-line-65)">-</text><text class="breeze-shell-r4" x="36.6" y="1606" textLength="85.4" clip-path="url(#breeze-shell-line-65)">-answer</text><text class="breeze-shell-r6" x="158.6" y="1606" textLength="24.4" clip-path="url(#breeze-shell-line-65)">-a</text><text class="breeze-shell-r2 [...] +</text><text class="breeze-shell-r5" x="0" y="1630.4" textLength="12.2" clip-path="url(#breeze-shell-line-66)">│</text><text class="breeze-shell-r4" x="24.4" y="1630.4" textLength="12.2" clip-path="url(#breeze-shell-line-66)">-</text><text class="breeze-shell-r4" x="36.6" y="1630.4" textLength="61" clip-path="url(#breeze-shell-line-66)">-help</text><text class="breeze-shell-r6" x="158.6" y="1630.4" textLength="24.4" clip-path="url(#breeze-shell-line-66)">-h</text><text class="breeze-shel [...] +</text><text class="breeze-shell-r5" x="0" y="1654.8" textLength="1464" clip-path="url(#breeze-shell-line-67)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-shell-r2" x="1464" y="1654.8" textLength="12.2" clip-path="url(#breeze-shell-line-67)"> </text> </g> </g> diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg index ec3da499fa..9cf14a8662 100644 --- a/images/breeze/output_testing_tests.svg +++ b/images/breeze/output_testing_tests.svg @@ -1,4 +1,4 @@ -<svg class="rich-terminal" viewBox="0 0 1482 1489.6" xmlns="http://www.w3.org/2000/svg";> +<svg class="rich-terminal" viewBox="0 0 1482 1514.0" xmlns="http://www.w3.org/2000/svg";> <!-- Generated with Rich https://www.textualize.io --> <style> @@ -43,7 +43,7 @@ <defs> <clipPath id="breeze-testing-tests-clip-terminal"> - <rect x="0" y="0" width="1463.0" height="1438.6" /> + <rect x="0" y="0" width="1463.0" height="1463.0" /> </clipPath> <clipPath id="breeze-testing-tests-line-0"> <rect x="0" y="1.5" width="1464" height="24.65"/> @@ -219,9 +219,12 @@ <clipPath id="breeze-testing-tests-line-57"> <rect x="0" y="1392.3" width="1464" height="24.65"/> </clipPath> +<clipPath id="breeze-testing-tests-line-58"> + <rect x="0" y="1416.7" width="1464" height="24.65"/> + </clipPath> </defs> - <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="1487.6" rx="8"/><text class="breeze-testing-tests-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: testing tests</text> + <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="1512" rx="8"/><text class="breeze-testing-tests-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: testing tests</text> <g transform="translate(26,22)"> <circle cx="0" cy="0" r="7" fill="#ff5f57"/> <circle cx="22" cy="0" r="7" fill="#febc2e"/> @@ -282,14 +285,15 @@ </text><text class="breeze-testing-tests-r5" x="0" y="1191.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-48)">│</text><text class="breeze-testing-tests-r7" x="414.8" y="1191.2" textLength="1024.8" clip-path="url(#breeze-testing-tests-line-48)">(selected | all | skip | remove)                            & [...] </text><text class="breeze-testing-tests-r5" x="0" y="1215.6" textLength="12.2" clip-path="url(#breeze-testing-tests-line-49)">│</text><text class="breeze-testing-tests-r5" x="414.8" y="1215.6" textLength="1024.8" clip-path="url(#breeze-testing-tests-line-49)">[default: selected]                                   [...] </text><text class="breeze-testing-tests-r5" x="0" y="1240" textLength="12.2" clip-path="url(#breeze-testing-tests-line-50)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1240" textLength="12.2" clip-path="url(#breeze-testing-tests-line-50)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1240" textLength="97.6" clip-path="url(#breeze-testing-tests-line-50)">-upgrade</text><text class="breeze-testing-tests-r4" x="134.2" y="1240" textLength="61" clip-path="url(#breez [...] -</text><text class="breeze-testing-tests-r5" x="0" y="1264.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-51)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1264.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-51)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1264.4" textLength="85.4" clip-path="url(#breeze-testing-tests-line-51)">-remove</text><text class="breeze-testing-tests-r4" x="122" y="1264.4" textLength="158.6" clip-path="ur [...] -</text><text class="breeze-testing-tests-r5" x="0" y="1288.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-52)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1288.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-52)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1288.8" textLength="61" clip-path="url(#breeze-testing-tests-line-52)">-skip</text><text class="breeze-testing-tests-r4" x="97.6" y="1288.8" textLength="244" clip-path="url(#br [...] -</text><text class="breeze-testing-tests-r5" x="0" y="1313.2" textLength="1464" clip-path="url(#breeze-testing-tests-line-53)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-testing-tests-r2" x="1464" y="1313.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-53)"> -</text><text class="breeze-testing-tests-r5" x="0" y="1337.6" textLength="24.4" clip-path="url(#breeze-testing-tests-line-54)">╭─</text><text class="breeze-testing-tests-r5" x="24.4" y="1337.6" textLength="195.2" clip-path="url(#breeze-testing-tests-line-54)"> Common options </text><text class="breeze-testing-tests-r5" x="219.6" y="1337.6" textLength="1220" clip-path="url(#breeze-testing-tests-line-54)">────────────────────────────────────────────────────────────────────── [...] -</text><text class="breeze-testing-tests-r5" x="0" y="1362" textLength="12.2" clip-path="url(#breeze-testing-tests-line-55)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1362" textLength="12.2" clip-path="url(#breeze-testing-tests-line-55)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1362" textLength="97.6" clip-path="url(#breeze-testing-tests-line-55)">-verbose</text><text class="breeze-testing-tests-r6" x="158.6" y="1362" textLength="24.4" clip-path="url(#bre [...] -</text><text class="breeze-testing-tests-r5" x="0" y="1386.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-56)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1386.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-56)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1386.4" textLength="48.8" clip-path="url(#breeze-testing-tests-line-56)">-dry</text><text class="breeze-testing-tests-r4" x="85.4" y="1386.4" textLength="48.8" clip-path="url(# [...] -</text><text class="breeze-testing-tests-r5" x="0" y="1410.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-57)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1410.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-57)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1410.8" textLength="61" clip-path="url(#breeze-testing-tests-line-57)">-help</text><text class="breeze-testing-tests-r6" x="158.6" y="1410.8" textLength="24.4" clip-path="url(# [...] -</text><text class="breeze-testing-tests-r5" x="0" y="1435.2" textLength="1464" clip-path="url(#breeze-testing-tests-line-58)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-testing-tests-r2" x="1464" y="1435.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-58)"> +</text><text class="breeze-testing-tests-r5" x="0" y="1264.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-51)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1264.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-51)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1264.4" textLength="122" clip-path="url(#breeze-testing-tests-line-51)">-downgrade</text><text class="breeze-testing-tests-r4" x="158.6" y="1264.4" textLength="134.2" clip-path [...] +</text><text class="breeze-testing-tests-r5" x="0" y="1288.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-52)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1288.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-52)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1288.8" textLength="85.4" clip-path="url(#breeze-testing-tests-line-52)">-remove</text><text class="breeze-testing-tests-r4" x="122" y="1288.8" textLength="158.6" clip-path="ur [...] +</text><text class="breeze-testing-tests-r5" x="0" y="1313.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-53)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1313.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-53)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1313.2" textLength="61" clip-path="url(#breeze-testing-tests-line-53)">-skip</text><text class="breeze-testing-tests-r4" x="97.6" y="1313.2" textLength="244" clip-path="url(#br [...] +</text><text class="breeze-testing-tests-r5" x="0" y="1337.6" textLength="1464" clip-path="url(#breeze-testing-tests-line-54)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-testing-tests-r2" x="1464" y="1337.6" textLength="12.2" clip-path="url(#breeze-testing-tests-line-54)"> +</text><text class="breeze-testing-tests-r5" x="0" y="1362" textLength="24.4" clip-path="url(#breeze-testing-tests-line-55)">╭─</text><text class="breeze-testing-tests-r5" x="24.4" y="1362" textLength="195.2" clip-path="url(#breeze-testing-tests-line-55)"> Common options </text><text class="breeze-testing-tests-r5" x="219.6" y="1362" textLength="1220" clip-path="url(#breeze-testing-tests-line-55)">──────────────────────────────────────────────────────────────────────────── [...] +</text><text class="breeze-testing-tests-r5" x="0" y="1386.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-56)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1386.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-56)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1386.4" textLength="97.6" clip-path="url(#breeze-testing-tests-line-56)">-verbose</text><text class="breeze-testing-tests-r6" x="158.6" y="1386.4" textLength="24.4" clip-path=" [...] +</text><text class="breeze-testing-tests-r5" x="0" y="1410.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-57)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1410.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-57)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1410.8" textLength="48.8" clip-path="url(#breeze-testing-tests-line-57)">-dry</text><text class="breeze-testing-tests-r4" x="85.4" y="1410.8" textLength="48.8" clip-path="url(# [...] +</text><text class="breeze-testing-tests-r5" x="0" y="1435.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-58)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1435.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-58)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1435.2" textLength="61" clip-path="url(#breeze-testing-tests-line-58)">-help</text><text class="breeze-testing-tests-r6" x="158.6" y="1435.2" textLength="24.4" clip-path="url(# [...] +</text><text class="breeze-testing-tests-r5" x="0" y="1459.6" textLength="1464" clip-path="url(#breeze-testing-tests-line-59)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-testing-tests-r2" x="1464" y="1459.6" textLength="12.2" clip-path="url(#breeze-testing-tests-line-59)"> </text> </g> </g> diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env index 2cd4bc0f7d..aa21804e9b 100644 --- a/scripts/ci/docker-compose/_docker.env +++ b/scripts/ci/docker-compose/_docker.env @@ -75,6 +75,7 @@ TEST_TIMEOUT TEST_TYPE UPGRADE_BOTO UPGRADE_TO_NEWER_DEPENDENCIES +DOWNGRADE_SQLALCHEMY VERBOSE VERBOSE_COMMANDS VERSION_SUFFIX_FOR_PYPI diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 160e8feec0..3b7417eab5 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -85,6 +85,7 @@ services: - TEST_TYPE=${TEST_TYPE} - TEST_TIMEOUT=${TEST_TIMEOUT} - UPGRADE_BOTO=${UPGRADE_BOTO} + - DOWNGRADE_SQLALCHEMY=${DOWNGRADE_SQLALCHEMY} - UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} - VERBOSE=${VERBOSE} - VERBOSE_COMMANDS=${VERBOSE_COMMANDS} diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env index d66767fa02..f71a0e1e39 100644 --- a/scripts/ci/docker-compose/devcontainer.env +++ b/scripts/ci/docker-compose/devcontainer.env @@ -69,6 +69,7 @@ START_AIRFLOW="false" SUSPENDED_PROVIDERS_FOLDERS="" TEST_TYPE= UPGRADE_BOTO="false" +DOWNGRADE_SQLALCHEMY="false" UPGRADE_TO_NEWER_DEPENDENCIES="false" VERBOSE="false" VERBOSE_COMMANDS="false" diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 73b8a20deb..87e573d114 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -326,6 +326,22 @@ fi # Remove pytest.ini from the current directory if it exists. It has been removed from the source tree # but may still be present in the local directory if the user has old breeze image rm -f "${AIRFLOW_SOURCES}/pytest.ini" +if [[ ${UPGRADE_BOTO=} == "true" ]]; then + echo + echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" + echo + pip uninstall --root-user-action ignore aiobotocore -y || true + pip install --root-user-action ignore --upgrade boto3 botocore + pip check +fi +if [[ ${DOWNGRADE_SQLALCHEMY=} == "true" ]]; then + min_sqlalchemy_version=$(grep "sqlalchemy>=" setup.cfg | sed "s/.*>=\([0-9\.]*\).*/\1/") + echo + echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}" + echo + pip install --root-user-action ignore "sqlalchemy==${min_sqlalchemy_version}" + pip check +fi set +u # If we do not want to run tests, we simply drop into bash @@ -558,13 +574,6 @@ else exit 1 fi fi -if [[ ${UPGRADE_BOTO=} == "true" ]]; then - echo - echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" - echo - pip uninstall aiobotocore -y || true - pip install --upgrade boto3 botocore -fi readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ ALL_TESTS ALL_PRESELECTED_TESTS diff --git a/setup.cfg b/setup.cfg index 367924d82d..26c8cb2e03 100644 --- a/setup.cfg +++ b/setup.cfg @@ -142,7 +142,7 @@ install_requires = # See https://sqlalche.me/e/b8d9 for details of deprecated features # you can set environment variable SQLALCHEMY_WARN_20=1 to show all deprecation warnings. # The issue tracking it is https://github.com/apache/airflow/issues/28723 - sqlalchemy>=1.4.24,<2.0 + sqlalchemy>=1.4.28,<2.0 sqlalchemy_jsonfield>=1.0 tabulate>=0.7.5 tenacity>=6.2.0,!=8.2.0
