This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e87562219e4 Bump min versions of crucial providers (#50076)
e87562219e4 is described below
commit e87562219e41856417e077166f5acce34d4d083d
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu May 1 17:05:17 2025 +0200
Bump min versions of crucial providers (#50076)
There are a few min-versions of providers that should make its way
into apache-airflow pyproject.toml - because they contain crucial
Airflow 3 fixes and users should not be installng those
providers in lower versions.
This PR adds such limits and makes it clearer when updates happen
what happens and where to override them.
---
.pre-commit-config.yaml | 2 +-
dev/breeze/src/airflow_breeze/utils/packages.py | 29 +++++++++++++---------
pyproject.toml | 12 ++++-----
.../ci/pre_commit/update_airflow_pyproject_toml.py | 21 +++++++++-------
4 files changed, 36 insertions(+), 28 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c85cc059e77..8e5c8cfdab0 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -889,7 +889,7 @@ repos:
name: Update Airflow's meta-package pyproject.toml
language: python
entry: ./scripts/ci/pre_commit/update_airflow_pyproject_toml.py
- files: ^pyproject\.toml$
+ files:
^.*/pyproject\.toml$|^scripts/ci/pre_commit/update_airflow_pyproject_toml\.py$
pass_filenames: false
require_serial: true
additional_dependencies: ['rich>=12.4.4', 'tomli>=2.0.1',
'packaging>=23.2' ]
diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py
b/dev/breeze/src/airflow_breeze/utils/packages.py
index ea4564022c8..3eb6e1e43d0 100644
--- a/dev/breeze/src/airflow_breeze/utils/packages.py
+++ b/dev/breeze/src/airflow_breeze/utils/packages.py
@@ -1040,22 +1040,27 @@ def
update_version_suffix_in_non_provider_pyproject_toml(version_suffix: str, py
lines = pyproject_toml_path.read_text().splitlines()
updated_lines = []
for line in lines:
- if line.startswith("version = "):
+ base_line, comment = line.split(" #", 1) if " #" in line else (line,
"")
+ if comment:
+ comment = " #" + comment
+ if base_line.startswith("version = "):
get_console().print(f"[info]Updating version suffix to
{version_suffix} for {line}.")
- line = line.rstrip('"') + f'{version_suffix}"'
+ base_line = base_line.rstrip('"') + f'{version_suffix}"'
# do not modify references for .post prefixes
if not version_suffix.startswith(".post"):
- if line.strip().startswith('"apache-airflow-') and ">=" in line:
+ if base_line.strip().startswith('"apache-airflow-') and ">=" in
base_line:
floored_version_suffix = floor_version_suffix(version_suffix)
- get_console().print(f"[info]Updating version suffix to
{floored_version_suffix} for {line}.")
- line = line.rstrip('",') + f'{floored_version_suffix}",'
- if line.strip().startswith('"apache-airflow-core') and "==" in
line:
- get_console().print(f"[info]Updating version suffix to
{version_suffix} for {line}.")
- line = line.rstrip('",') + f'{version_suffix}",'
- if line.strip().startswith('"apache-airflow-task-sdk') and "==" in
line:
- get_console().print(f"[info]Updating version suffix to
{version_suffix} for {line}.")
- line = line.rstrip('",') + f'{version_suffix}",'
- updated_lines.append(line)
+ get_console().print(
+ f"[info]Updating version suffix to
{floored_version_suffix} for {base_line}."
+ )
+ base_line = base_line.rstrip('",') +
f'{floored_version_suffix}",'
+ if base_line.strip().startswith('"apache-airflow-core') and "=="
in base_line:
+ get_console().print(f"[info]Updating version suffix to
{version_suffix} for {base_line}.")
+ base_line = base_line.rstrip('",') + f'{version_suffix}",'
+ if base_line.strip().startswith('"apache-airflow-task-sdk') and
"==" in base_line:
+ get_console().print(f"[info]Updating version suffix to
{version_suffix} for {base_line}.")
+ base_line = base_line.rstrip('",') + f'{version_suffix}",'
+ updated_lines.append(f"{base_line}{comment}")
new_content = "\n".join(updated_lines) + "\n"
get_console().print(f"[info]Writing updated content to
{pyproject_toml_path}.\n")
pyproject_toml_path.write_text(new_content)
diff --git a/pyproject.toml b/pyproject.toml
index 0542218d26f..79a32907a3e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -212,7 +212,7 @@ packages = []
"apache-airflow-providers-exasol>=4.6.1"
]
"fab" = [
- "apache-airflow-providers-fab>=2.0.0"
+ "apache-airflow-providers-fab>=2.0.2" # Set from MIN_VERSION_OVERRIDE in
update_airflow_pyproject_toml.py
]
"facebook" = [
"apache-airflow-providers-facebook>=3.7.0"
@@ -221,7 +221,7 @@ packages = []
"apache-airflow-providers-ftp>=3.12.0"
]
"git" = [
- "apache-airflow-providers-git>=0.0.1"
+ "apache-airflow-providers-git>=0.0.2" # Set from MIN_VERSION_OVERRIDE in
update_airflow_pyproject_toml.py
]
"github" = [
"apache-airflow-providers-github>=2.8.0"
@@ -281,7 +281,7 @@ packages = []
"apache-airflow-providers-openfaas>=3.7.0"
]
"openlineage" = [
- "apache-airflow-providers-openlineage>=2.1.3"
+ "apache-airflow-providers-openlineage>=2.1.3" # Set from
MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
]
"opensearch" = [
"apache-airflow-providers-opensearch>=1.5.0"
@@ -422,10 +422,10 @@ packages = []
"apache-airflow-providers-edge3>=1.0.0",
"apache-airflow-providers-elasticsearch>=5.5.2",
"apache-airflow-providers-exasol>=4.6.1",
- "apache-airflow-providers-fab>=2.0.0",
+ "apache-airflow-providers-fab>=2.0.2", # Set from MIN_VERSION_OVERRIDE in
update_airflow_pyproject_toml.py
"apache-airflow-providers-facebook>=3.7.0",
"apache-airflow-providers-ftp>=3.12.0",
- "apache-airflow-providers-git>=0.0.1",
+ "apache-airflow-providers-git>=0.0.2", # Set from MIN_VERSION_OVERRIDE in
update_airflow_pyproject_toml.py
"apache-airflow-providers-github>=2.8.0",
"apache-airflow-providers-google>=10.24.0",
"apache-airflow-providers-grpc>=3.7.0",
@@ -445,7 +445,7 @@ packages = []
"apache-airflow-providers-odbc>=4.8.0",
"apache-airflow-providers-openai>=1.5.0",
"apache-airflow-providers-openfaas>=3.7.0",
- "apache-airflow-providers-openlineage>=2.1.3",
+ "apache-airflow-providers-openlineage>=2.1.3", # Set from
MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
"apache-airflow-providers-opensearch>=1.5.0",
"apache-airflow-providers-opsgenie>=5.8.0",
"apache-airflow-providers-oracle>=3.12.0",
diff --git a/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
b/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
index e4c56ea7d8c..224ecdaac87 100755
--- a/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
+++ b/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
@@ -56,9 +56,9 @@ CUT_OFF_TIMEDELTA = timedelta(days=6 * 30)
# minimum versions for compatibility with Airflow 3
MIN_VERSION_OVERRIDE: dict[str, Version] = {
"amazon": parse_version("2.1.3"),
- "fab": parse_version("2.0.0"),
+ "fab": parse_version("2.0.2"),
"openlineage": parse_version("2.1.3"),
- "git": parse_version("0.0.1"),
+ "git": parse_version("0.0.2"),
"common.messaging": parse_version("1.0.0"),
}
@@ -88,7 +88,7 @@ console.print("[bright_blue]Updating min-provider versions in
apache-airflow\n")
all_providers_metadata = json.loads(PROVIDER_METADATA_FILE_PATH.read_text())
-def find_min_provider_version(provider_id: str) -> Version | None:
+def find_min_provider_version(provider_id: str) -> tuple[Version | None, str]:
metadata = all_providers_metadata.get(provider_id)
# We should periodically update the starting date to avoid pip install
resolution issues
# TODO: when min Python version is 3.11 change back the code to
fromisoformat
@@ -101,7 +101,7 @@ def find_min_provider_version(provider_id: str) -> Version
| None:
min_version_override = MIN_VERSION_OVERRIDE.get(provider_id)
if not metadata:
if not min_version_override:
- return None
+ return None, ""
last_version_newer_than_cutoff = min_version_override
else:
versions: list[Version] = sorted([parse_version(version) for version
in metadata], reverse=True)
@@ -117,14 +117,17 @@ def find_min_provider_version(provider_id: str) ->
Version | None:
f"[bright_blue]Provider id {provider_id} min version found:[/] "
f"{last_version_newer_than_cutoff} (date {date_released}"
)
+ override_comment = ""
if last_version_newer_than_cutoff:
if min_version_override and min_version_override >
last_version_newer_than_cutoff:
console.print(
f"[yellow]Overriding provider id {provider_id} min version:[/]
{min_version_override} "
- f"overridden from hard-coded versions."
+ f"set from hard-coded versions.\n\n"
+ f"[yellow]Modify MIN_VERSION_OVERRIDE in {__file__} to set
different version![/]\n"
)
last_version_newer_than_cutoff = min_version_override
- return last_version_newer_than_cutoff
+ override_comment = f" # Set from MIN_VERSION_OVERRIDE in
{Path(__file__).name}"
+ return last_version_newer_than_cutoff, override_comment
PROVIDER_MIN_VERSIONS: dict[str, str | None] = {}
@@ -141,11 +144,11 @@ if __name__ == "__main__":
all_provider_lines = []
for provider_id in all_providers:
distribution_name = provider_distribution_name(provider_id)
- min_provider_version = find_min_provider_version(provider_id)
+ min_provider_version, comment = find_min_provider_version(provider_id)
if min_provider_version:
- all_provider_lines.append(f'
"{distribution_name}>={min_provider_version}",\n')
+ all_provider_lines.append(f'
"{distribution_name}>={min_provider_version}",{comment}\n')
all_optional_dependencies.append(
- f'"{provider_id}" = [\n
"{distribution_name}>={min_provider_version}"\n]\n'
+ f'"{provider_id}" = [\n
"{distribution_name}>={min_provider_version}"{comment}\n]\n'
)
else:
all_optional_dependencies.append(f'"{provider_id}" = [\n
"{distribution_name}"\n]\n')
