This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 5cd78cf425 Upgrade dependencies in order to avoid backtracking (#27531)
5cd78cf425 is described below
commit 5cd78cf425f6fedc380662ec9a9e37be51403ccb
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Nov 6 22:21:36 2022 +0100
Upgrade dependencies in order to avoid backtracking (#27531)
Seems that the new `pip` have become a little more relaxed when it
comes to looking for dependencies, and we need to make sure that
version constraint is specified in every extra otherwise it might
have hard time finding the right set of dependencies while
running "eager upgrade".
Over the last few days all our main builds failed because of that as pip
was backtracking when performing eager upgrade..
This PR updates various dependencies to get rid of the backtracking,
following the process described in the
https://github.com/apache/airflow/blob/main/dev/TRACKING_BACKTRACKING_ISSUES.md
---
Dockerfile | 4 ++--
Dockerfile.ci | 4 ++--
IMAGES.rst | 2 +-
airflow/providers/amazon/provider.yaml | 8 ++++----
airflow/providers/google/provider.yaml | 1 +
airflow/providers/microsoft/azure/provider.yaml | 1 +
dev/breeze/src/airflow_breeze/global_constants.py | 2 +-
dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py | 2 +-
docs/docker-stack/build-arg-ref.rst | 2 +-
generated/provider_dependencies.json | 10 ++++++----
scripts/docker/common.sh | 2 +-
setup.cfg | 1 +
setup.py | 6 ++++--
13 files changed, 26 insertions(+), 19 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 5f61dfa769..6b2d3d7484 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -48,7 +48,7 @@ ARG AIRFLOW_VERSION="2.4.2"
ARG PYTHON_BASE_IMAGE="python:3.7-slim-bullseye"
-ARG AIRFLOW_PIP_VERSION=22.3
+ARG AIRFLOW_PIP_VERSION=22.3.1
ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow";
ARG
AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md";
@@ -422,7 +422,7 @@ function common::get_airflow_version_specification() {
function common::override_pip_version_if_needed() {
if [[ -n ${AIRFLOW_VERSION} ]]; then
if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]];
then
- export AIRFLOW_PIP_VERSION="22.3"
+ export AIRFLOW_PIP_VERSION="22.3.1"
fi
fi
}
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 5cb13b312a..ea3766699d 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -382,7 +382,7 @@ function common::get_airflow_version_specification() {
function common::override_pip_version_if_needed() {
if [[ -n ${AIRFLOW_VERSION} ]]; then
if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]];
then
- export AIRFLOW_PIP_VERSION="22.3"
+ export AIRFLOW_PIP_VERSION="22.3.1"
fi
fi
}
@@ -1093,7 +1093,7 @@ ARG AIRFLOW_CI_BUILD_EPOCH="3"
ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
# By default in the image, we are installing all providers when installing
from sources
ARG INSTALL_PROVIDERS_FROM_SOURCES="true"
-ARG AIRFLOW_PIP_VERSION=22.3
+ARG AIRFLOW_PIP_VERSION=22.3.1
# Setup PIP
# By default PIP install run without cache to make image smaller
ARG PIP_NO_CACHE_DIR="true"
diff --git a/IMAGES.rst b/IMAGES.rst
index 2f9ff7c929..f6b78aee0f 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -457,7 +457,7 @@ The following build arguments (``--build-arg`` in docker
build command) can be u
| ``ADDITIONAL_DEV_APT_ENV`` |
| Additional env variables defined |
| |
| when installing dev deps |
+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_PIP_VERSION`` | ``22.3``
| PIP version used. |
+| ``AIRFLOW_PIP_VERSION`` | ``22.3.1``
| PIP version used. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``PIP_PROGRESS_BAR`` | ``on``
| Progress bar for PIP installation |
+------------------------------------------+------------------------------------------+------------------------------------------+
diff --git a/airflow/providers/amazon/provider.yaml
b/airflow/providers/amazon/provider.yaml
index aaa65a2718..6486290ef1 100644
--- a/airflow/providers/amazon/provider.yaml
+++ b/airflow/providers/amazon/provider.yaml
@@ -48,7 +48,7 @@ versions:
dependencies:
- apache-airflow>=2.3.0
- apache-airflow-providers-common-sql>=1.3.0
- - boto3>=1.15.0
+ - boto3>=1.24.0
# watchtower 3 has been released end Jan and introduced breaking change
across the board that might
# change logging behaviour:
#
https://github.com/kislyuk/watchtower/blob/develop/Changes.rst#changes-for-v300-2022-01-26
@@ -58,9 +58,9 @@ dependencies:
- redshift_connector>=2.0.888
- sqlalchemy_redshift>=0.8.6
- pandas>=0.17.1
- - mypy-boto3-rds>=1.21.0
- - mypy-boto3-redshift-data>=1.21.0
- - mypy-boto3-appflow>=1.21.0
+ - mypy-boto3-rds>=1.24.0
+ - mypy-boto3-redshift-data>=1.24.0
+ - mypy-boto3-appflow>=1.24.0
integrations:
- integration-name: Amazon Athena
diff --git a/airflow/providers/google/provider.yaml
b/airflow/providers/google/provider.yaml
index 3e105f13e1..dad775d8b3 100644
--- a/airflow/providers/google/provider.yaml
+++ b/airflow/providers/google/provider.yaml
@@ -66,6 +66,7 @@ dependencies:
# TODO: Upgrade all Google libraries that are limited to <2.0.0
- PyOpenSSL
- asgiref>=3.5.2
+ - gcloud_aio_auth>=4.0.0
- gcloud-aio-bigquery>=6.1.2
- gcloud-aio-storage
- google-ads>=15.1.1
diff --git a/airflow/providers/microsoft/azure/provider.yaml
b/airflow/providers/microsoft/azure/provider.yaml
index 63c670d4a8..88adf73cbc 100644
--- a/airflow/providers/microsoft/azure/provider.yaml
+++ b/airflow/providers/microsoft/azure/provider.yaml
@@ -64,6 +64,7 @@ dependencies:
# Limited due to https://github.com/Azure/azure-uamqp-python/issues/191
- azure-servicebus>=7.6.1; platform_machine != "aarch64"
- azure-synapse-spark
+ - adal>=.1.2.7
integrations:
- integration-name: Microsoft Azure Batch
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index f0ee7e92bf..115540ef40 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -74,7 +74,7 @@ ALLOWED_POSTGRES_VERSIONS = ["10", "11", "12", "13", "14"]
ALLOWED_MYSQL_VERSIONS = ["5.7", "8"]
ALLOWED_MSSQL_VERSIONS = ["2017-latest", "2019-latest"]
-PIP_VERSION = "22.3"
+PIP_VERSION = "22.3.1"
@lru_cache(maxsize=None)
diff --git a/dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py
b/dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py
index 68b7d1db51..f0cf6ab580 100644
--- a/dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py
+++ b/dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py
@@ -92,7 +92,7 @@ def find_newer_dependencies(
get_console().print("[info]Use the following pip install command (see the
doc above for details)\n")
get_console().print(
'pip install ".[devel_all]" --upgrade --upgrade-strategy eager '
- '"dill<0.3.3" "certifi<2021.0.0"' + constraint_string,
+ '"dill<0.3.3" "pyarrow>=6.0.0" "protobuf<4.21.0"' + constraint_string,
markup=False,
soft_wrap=True,
)
diff --git a/docs/docker-stack/build-arg-ref.rst
b/docs/docker-stack/build-arg-ref.rst
index 4626bf769e..e37e6f29b9 100644
--- a/docs/docker-stack/build-arg-ref.rst
+++ b/docs/docker-stack/build-arg-ref.rst
@@ -45,7 +45,7 @@ Those are the most common arguments that you use when you
want to build a custom
+------------------------------------------+------------------------------------------+---------------------------------------------+
| ``AIRFLOW_USER_HOME_DIR`` | ``/home/airflow``
| Home directory of the Airflow user. |
+------------------------------------------+------------------------------------------+---------------------------------------------+
-| ``AIRFLOW_PIP_VERSION`` | ``22.3``
| PIP version used. |
+| ``AIRFLOW_PIP_VERSION`` | ``22.3.1``
| PIP version used. |
+------------------------------------------+------------------------------------------+---------------------------------------------+
| ``ADDITIONAL_PIP_INSTALL_FLAGS`` |
| additional ``pip`` flags passed to the |
| |
| installation commands (except when |
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index 8b9a5957fa..d86f146dfe 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -19,11 +19,11 @@
"deps": [
"apache-airflow-providers-common-sql>=1.3.0",
"apache-airflow>=2.3.0",
- "boto3>=1.15.0",
+ "boto3>=1.24.0",
"jsonpath_ng>=1.5.3",
- "mypy-boto3-appflow>=1.21.0",
- "mypy-boto3-rds>=1.21.0",
- "mypy-boto3-redshift-data>=1.21.0",
+ "mypy-boto3-appflow>=1.24.0",
+ "mypy-boto3-rds>=1.24.0",
+ "mypy-boto3-redshift-data>=1.24.0",
"pandas>=0.17.1",
"redshift_connector>=2.0.888",
"sqlalchemy_redshift>=0.8.6",
@@ -305,6 +305,7 @@
"asgiref>=3.5.2",
"gcloud-aio-bigquery>=6.1.2",
"gcloud-aio-storage",
+ "gcloud_aio_auth>=4.0.0",
"google-ads>=15.1.1",
"google-api-core>=2.7.0,<3.0.0",
"google-api-python-client>=1.6.0,<2.0.0",
@@ -426,6 +427,7 @@
},
"microsoft.azure": {
"deps": [
+ "adal>=.1.2.7",
"apache-airflow>=2.3.0",
"azure-batch>=8.0.0",
"azure-cosmos>=4.0.0",
diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
index fc62962d67..e7cfefaea7 100644
--- a/scripts/docker/common.sh
+++ b/scripts/docker/common.sh
@@ -42,7 +42,7 @@ function common::get_airflow_version_specification() {
function common::override_pip_version_if_needed() {
if [[ -n ${AIRFLOW_VERSION} ]]; then
if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]];
then
- export AIRFLOW_PIP_VERSION="22.3"
+ export AIRFLOW_PIP_VERSION="22.3.1"
fi
fi
}
diff --git a/setup.cfg b/setup.cfg
index f6ba7e946e..d798c970e0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -90,6 +90,7 @@ install_requires =
# Every time we update FAB version here, please make sure that you review
the classes and models in
# `airflow/www/fab_security` with their upstream counterparts. In
particular, make sure any breaking changes,
# for example any new methods, are accounted for.
+ # NOTE! When you change the value here, you also have to update
flask-appbuilder[oauth] in setup.py
flask-appbuilder==4.1.4
flask-caching>=1.5.0
flask-login>=0.6.2
diff --git a/setup.py b/setup.py
index bb8e0bdda7..f40be59dc9 100644
--- a/setup.py
+++ b/setup.py
@@ -276,7 +276,9 @@ doc = [
"sphinxcontrib-spelling>=7.3",
]
flask_appbuilder_oauth = [
- "flask-appbuilder[oauth]",
+ "authlib>=1.0.0",
+ # The version here should be upgraded at the same time as flask-appbuilder
in setup.cfg
+ "flask-appbuilder[oauth]==4.1.4",
]
kerberos = [
"pykerberos>=1.1.13",
@@ -365,7 +367,7 @@ devel_only = [
"click>=8.0",
"coverage",
"filelock",
- "flake8>=3.6.0",
+ "flake8>=3.9.0",
"flake8-colors",
"flake8-implicit-str-concat",
"flaky",
