This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-7-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 3fad890393ec0adf388bd7df059d88ce965e7472 Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Aug 21 17:49:17 2023 +0200 Fix dependencies for celery and opentelemetry for Python 3.8 (#33579) We used to have problems with `pip` backtracking when we relaxed too much open-telemetry dependencies. It turned out that the backtracting was only happening on Python 3.8 and that it was ultimately caused by conflict between importlib_metadata between Airflow and newer versions of opentelemetry (we had <5 for Python 3.8, they had >6 for all versions. The reason for limiting it in Airflow was Celery that was not working well with importlib 5. Since Celery 5.3 solved the problems (released 6th of June) we can now relax the importlib_metadata limit and set Celery to version >= 5.3.0) which nicely resolves the conflict and there is no more backtracking when trying to install newer versions of opentelemetry for Python 3.8. Fixes: #33577 (cherry picked from commit ae25a52ae342c9e0bc3afdb21d613447c3687f6c) --- airflow/providers/celery/provider.yaml | 2 +- dev/breeze/src/airflow_breeze/utils/path_utils.py | 2 +- generated/provider_dependencies.json | 2 +- setup.cfg | 8 ++------ setup.py | 2 +- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml index 0eccce6890..cecb52f933 100644 --- a/airflow/providers/celery/provider.yaml +++ b/airflow/providers/celery/provider.yaml @@ -43,7 +43,7 @@ dependencies: # Uses Celery for CeleryExecutor, and we also know that Kubernetes Python client follows SemVer # (https://docs.celeryq.dev/en/stable/contributing.html?highlight=semver#versions). # Make sure that the limit here is synchronized with [celery] extra in the airflow core - - celery>=5.2.3,<6 + - celery>=5.3.0,<6 - flower>=1.0.0 integrations: diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 2314f3aa9f..c9627c5ffb 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -87,7 +87,7 @@ def get_package_setup_metadata_hash() -> str: try: from importlib.metadata import distribution # type: ignore[attr-defined] except ImportError: - from importlib_metadata import distribution # type: ignore[no-redef] + from importlib_metadata import distribution # type: ignore[no-redef, assignment] prefix = "Package config hash: " diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 42146c2767..b029917e81 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -244,7 +244,7 @@ "celery": { "deps": [ "apache-airflow>=2.4.0", - "celery>=5.2.3,<6", + "celery>=5.3.0,<6", "flower>=1.0.0" ], "cross-providers-deps": [ diff --git a/setup.cfg b/setup.cfg index 399ff79bd9..c2ff2567f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -102,10 +102,7 @@ install_requires = graphviz>=0.12 gunicorn>=20.1.0 httpx - # Importlib-metadata 5 is breaking Celery import due to regression it introduced - # This was tracked and fixed in https://github.com/celery/celery/pull/7785 but it is not released yet - # We can remove the < 5.0.0 limitation when Celery 5.3.0 gets released and we bump celery to >= 5.3.0 - importlib_metadata>=1.7,<5.0.0;python_version<"3.9" + importlib_metadata>=1.7;python_version<"3.9" importlib_resources>=5.2;python_version<"3.9" itsdangerous>=2.0 jinja2>=3.0.0 @@ -118,8 +115,7 @@ install_requires = markupsafe>=1.1.1 marshmallow-oneofschema>=2.0.1 mdit-py-plugins>=0.3.0 - # Pip can not find a version that satisfies constraints if opentelemetry-api is not pinned. - opentelemetry-api==1.15.0 + opentelemetry-api>=1.15.0 opentelemetry-exporter-otlp packaging>=14.0 pathspec>=0.9.0 diff --git a/setup.py b/setup.py index 6f29dc1f47..00f4f7f947 100644 --- a/setup.py +++ b/setup.py @@ -275,7 +275,7 @@ celery = [ # limiting minimum airflow version supported in celery provider due to the # potential breaking changes in Airflow Core as well (celery is added as extra, so Airflow # core is not hard-limited via install-requires, only by extra). - "celery>=5.2.3,<6" + "celery>=5.3.0,<6" ] cgroups = [ # Cgroupspy 0.2.2 added Python 3.10 compatibility
