This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 5f504e9a17 Upgrade botocore/aiobotocore minimum requirements (#33649)
5f504e9a17 is described below
commit 5f504e9a17353259e70bd3ed54f8edd2e465882c
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Aug 23 14:04:16 2023 +0200
Upgrade botocore/aiobotocore minimum requirements (#33649)
Botocore has a very peculiar process of releasing new version
every day, which means that it gives `pip` hard time to figure
what will be the non-conflicting set of packages when we have
too low of a minium version set as requirement.
Since we had > 1.24 that means that `pip` had to consider
more than 340 versions for botocore, but also for related
mypy packages and also a number of aiobotocore packages when
resolving eager-upgrade.
We limit all the relevant packages to 1.28 as minimum version
now, and we should continue doing that regularly in the future.
---
Dockerfile.ci | 9 ++++-----
airflow/providers/amazon/provider.yaml | 20 ++++++++++++++------
generated/provider_dependencies.json | 11 ++++++-----
scripts/in_container/_in_container_utils.sh | 10 ++++++++++
4 files changed, 34 insertions(+), 16 deletions(-)
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 96468ceb36..0761021304 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -1393,11 +1393,10 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
RUN echo "Airflow version: ${AIRFLOW_VERSION}"
# Those are additional constraints that are needed for some extras but we do
not want to
-# force them on the main Airflow package.
-# Recent aiobotocore (2.6.0+) requires this specific botocore version, while
-# boto3 1.28.17 is the last version supporting that, so we pin them to help pip
-# avoid extra work. Revisit this when a new aiobotocore version is released.
-ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="botocore==1.31.17 boto3<=1.28.17"
+# force them on the main Airflow package. Currently we need no extra limits as
PIP 23.1+ has much better
+# dependency resolution and we do not need to limit the versions of the
dependencies
+# aiobotocore is limited temporarily until it stops backtracking pip
+ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=""
ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
ARG VERSION_SUFFIX_FOR_PYPI=""
diff --git a/airflow/providers/amazon/provider.yaml
b/airflow/providers/amazon/provider.yaml
index 6aa3282510..2959764e00 100644
--- a/airflow/providers/amazon/provider.yaml
+++ b/airflow/providers/amazon/provider.yaml
@@ -67,7 +67,19 @@ dependencies:
- apache-airflow>=2.4.0
- apache-airflow-providers-common-sql>=1.3.1
- apache-airflow-providers-http
- - boto3>=1.24.0
+ # We should update minimum version of boto3 and here regularly to avoid
`pip` backtracking with the number
+ # of candidates to consider. We should also make sure that all the below
related packages have also the
+ # same minimum version specified. Boto3 1.28.0 has been released on July 6
2023. We should also make sure we
+ # set it to the version that `aiobotocore` supports (see `aiobotocore`
optional dependency at the end
+ # of this file). Currently we set aiobotocore as minimum 2.5.3 - as this is
was the first version
+ # that supported boto3 1.28. NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3
MINOR VERSIONS
+ - boto3>=1.28.0
+ - mypy-boto3-rds>=1.28.0
+ - mypy-boto3-redshift-data>=1.28.0
+ - mypy-boto3-s3>=1.28.0
+ - mypy-boto3-appflow>=1.28.0
+ # NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS
+ - botocore>=1.31.0
- asgiref
# watchtower 3 has been released end Jan and introduced breaking change
across the board that might
# change logging behaviour:
@@ -77,11 +89,7 @@ dependencies:
- jsonpath_ng>=1.5.3
- redshift_connector>=2.0.888
- sqlalchemy_redshift>=0.8.6
- - mypy-boto3-rds>=1.24.0
- - mypy-boto3-redshift-data>=1.24.0
- - mypy-boto3-appflow>=1.28.16
- asgiref
- - mypy-boto3-s3>=1.24.0
integrations:
- integration-name: Amazon Athena
@@ -699,7 +707,7 @@ additional-extras:
# boto3 have native async support and we move away from aio aiobotocore
- name: aiobotocore
dependencies:
- - aiobotocore[boto3]>=2.2.0
+ - aiobotocore[boto3]>=2.5.3
- name: cncf.kubernetes
dependencies:
- apache-airflow-providers-cncf-kubernetes>=7.2.0
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index 0d5a7408ea..23de3da647 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -26,12 +26,13 @@
"apache-airflow>=2.4.0",
"asgiref",
"asgiref",
- "boto3>=1.24.0",
+ "boto3>=1.28.0",
+ "botocore>=1.31.0",
"jsonpath_ng>=1.5.3",
- "mypy-boto3-appflow>=1.28.16",
- "mypy-boto3-rds>=1.24.0",
- "mypy-boto3-redshift-data>=1.24.0",
- "mypy-boto3-s3>=1.24.0",
+ "mypy-boto3-appflow>=1.28.0",
+ "mypy-boto3-rds>=1.28.0",
+ "mypy-boto3-redshift-data>=1.28.0",
+ "mypy-boto3-s3>=1.28.0",
"redshift_connector>=2.0.888",
"sqlalchemy_redshift>=0.8.6",
"watchtower~=2.0.1"
diff --git a/scripts/in_container/_in_container_utils.sh
b/scripts/in_container/_in_container_utils.sh
index 16663f4738..288248b4ec 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -297,6 +297,12 @@ function
install_all_providers_from_pypi_with_eager_upgrade() {
local res
for provider_package in ${ALL_PROVIDERS_PACKAGES}
do
+ # Until we release "yandex" provider with protobuf support we need to
remove it from the list of providers
+ # to install, because it is impossible to find common requirements for
already released yandex provider
+ # and current airflow
+ if [[ ${provider_package} == "apache-airflow-providers-yandex" ]]; then
+ continue
+ fi
echo -n "Checking if ${provider_package} is available in PyPI: "
res=$(curl --head -s -o /dev/null -w "%{http_code}"
"https://pypi.org/project/${provider_package}/";)
if [[ ${res} == "200" ]]; then
@@ -306,7 +312,11 @@ function
install_all_providers_from_pypi_with_eager_upgrade() {
echo "${COLOR_YELLOW}Skipped${COLOR_RESET}"
fi
done
+
+
echo "Installing provider packages: ${packages_to_install[*]}"
+
+
# we add eager requirements to make sure to take into account limitations
that will allow us to
# install all providers. We install only those packages that are available
in PyPI - we might
# Have some new providers in the works and they might not yet be simply
available in PyPI
