This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-0-test by this push:
new 211b51004bf [v3-0-test] Bump min versions of crucial providers
(#50076) (#50083)
211b51004bf is described below
commit 211b51004bf1c10fc30e213e8596298559defe7f
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu May 1 17:14:44 2025 +0200
[v3-0-test] Bump min versions of crucial providers (#50076) (#50083)
There are a few min-versions of providers that should make its way
into apache-airflow pyproject.toml - because they contain crucial
Airflow 3 fixes and users should not be installng those
providers in lower versions.
This PR adds such limits and makes it clearer when updates happen
what happens and where to override them.
(cherry picked from commit e87562219e41856417e077166f5acce34d4d083d)
Co-authored-by: Jarek Potiuk <[email protected]>
---
.pre-commit-config.yaml | 2 +-
dev/breeze/src/airflow_breeze/utils/packages.py | 29 +++++++++++++---------
pyproject.toml | 12 ++++-----
.../ci/pre_commit/update_airflow_pyproject_toml.py | 21 +++++++++-------
4 files changed, 36 insertions(+), 28 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 9624a9cc11f..1fc3e81f2ad 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -886,7 +886,7 @@ repos:
name: Update Airflow's meta-package pyproject.toml
language: python
entry: ./scripts/ci/pre_commit/update_airflow_pyproject_toml.py
- files: ^pyproject\.toml$
+ files:
^.*/pyproject\.toml$|^scripts/ci/pre_commit/update_airflow_pyproject_toml\.py$
pass_filenames: false
require_serial: true
additional_dependencies: ['rich>=12.4.4', 'tomli>=2.0.1',
'packaging>=23.2' ]
diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py
b/dev/breeze/src/airflow_breeze/utils/packages.py
index be84201e464..f8c92047668 100644
--- a/dev/breeze/src/airflow_breeze/utils/packages.py
+++ b/dev/breeze/src/airflow_breeze/utils/packages.py
@@ -1049,22 +1049,27 @@ def
update_version_suffix_in_non_provider_pyproject_toml(version_suffix: str, py
lines = pyproject_toml_path.read_text().splitlines()
updated_lines = []
for line in lines:
- if line.startswith("version = "):
+ base_line, comment = line.split(" #", 1) if " #" in line else (line,
"")
+ if comment:
+ comment = " #" + comment
+ if base_line.startswith("version = "):
get_console().print(f"[info]Updating version suffix to
{version_suffix} for {line}.")
- line = line.rstrip('"') + f'{version_suffix}"'
+ base_line = base_line.rstrip('"') + f'{version_suffix}"'
# do not modify references for .post prefixes
if not version_suffix.startswith(".post"):
- if line.strip().startswith('"apache-airflow-') and ">=" in line:
+ if base_line.strip().startswith('"apache-airflow-') and ">=" in
base_line:
floored_version_suffix = floor_version_suffix(version_suffix)
- get_console().print(f"[info]Updating version suffix to
{floored_version_suffix} for {line}.")
- line = line.rstrip('",') + f'{floored_version_suffix}",'
- if line.strip().startswith('"apache-airflow-core') and "==" in
line:
- get_console().print(f"[info]Updating version suffix to
{version_suffix} for {line}.")
- line = line.rstrip('",') + f'{version_suffix}",'
- if line.strip().startswith('"apache-airflow-task-sdk') and "==" in
line:
- get_console().print(f"[info]Updating version suffix to
{version_suffix} for {line}.")
- line = line.rstrip('",') + f'{version_suffix}",'
- updated_lines.append(line)
+ get_console().print(
+ f"[info]Updating version suffix to
{floored_version_suffix} for {base_line}."
+ )
+ base_line = base_line.rstrip('",') +
f'{floored_version_suffix}",'
+ if base_line.strip().startswith('"apache-airflow-core') and "=="
in base_line:
+ get_console().print(f"[info]Updating version suffix to
{version_suffix} for {base_line}.")
+ base_line = base_line.rstrip('",') + f'{version_suffix}",'
+ if base_line.strip().startswith('"apache-airflow-task-sdk') and
"==" in base_line:
+ get_console().print(f"[info]Updating version suffix to
{version_suffix} for {base_line}.")
+ base_line = base_line.rstrip('",') + f'{version_suffix}",'
+ updated_lines.append(f"{base_line}{comment}")
new_content = "\n".join(updated_lines) + "\n"
get_console().print(f"[info]Writing updated content to
{pyproject_toml_path}.\n")
pyproject_toml_path.write_text(new_content)
diff --git a/pyproject.toml b/pyproject.toml
index 4e1f580941f..c6c56fbeb00 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -209,7 +209,7 @@ packages = []
"apache-airflow-providers-exasol>=4.6.1"
]
"fab" = [
- "apache-airflow-providers-fab>=2.0.0"
+ "apache-airflow-providers-fab>=2.0.2" # Set from MIN_VERSION_OVERRIDE in
update_airflow_pyproject_toml.py
]
"facebook" = [
"apache-airflow-providers-facebook>=3.7.0"
@@ -218,7 +218,7 @@ packages = []
"apache-airflow-providers-ftp>=3.12.0"
]
"git" = [
- "apache-airflow-providers-git>=0.0.1"
+ "apache-airflow-providers-git>=0.0.2" # Set from MIN_VERSION_OVERRIDE in
update_airflow_pyproject_toml.py
]
"github" = [
"apache-airflow-providers-github>=2.8.0"
@@ -278,7 +278,7 @@ packages = []
"apache-airflow-providers-openfaas>=3.7.0"
]
"openlineage" = [
- "apache-airflow-providers-openlineage>=2.1.3"
+ "apache-airflow-providers-openlineage>=2.1.3" # Set from
MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
]
"opensearch" = [
"apache-airflow-providers-opensearch>=1.5.0"
@@ -418,10 +418,10 @@ packages = []
"apache-airflow-providers-edge3>=1.0.0",
"apache-airflow-providers-elasticsearch>=5.5.2",
"apache-airflow-providers-exasol>=4.6.1",
- "apache-airflow-providers-fab>=2.0.0",
+ "apache-airflow-providers-fab>=2.0.2", # Set from MIN_VERSION_OVERRIDE in
update_airflow_pyproject_toml.py
"apache-airflow-providers-facebook>=3.7.0",
"apache-airflow-providers-ftp>=3.12.0",
- "apache-airflow-providers-git>=0.0.1",
+ "apache-airflow-providers-git>=0.0.2", # Set from MIN_VERSION_OVERRIDE in
update_airflow_pyproject_toml.py
"apache-airflow-providers-github>=2.8.0",
"apache-airflow-providers-google>=10.24.0",
"apache-airflow-providers-grpc>=3.7.0",
@@ -441,7 +441,7 @@ packages = []
"apache-airflow-providers-odbc>=4.8.0",
"apache-airflow-providers-openai>=1.5.0",
"apache-airflow-providers-openfaas>=3.7.0",
- "apache-airflow-providers-openlineage>=2.1.3",
+ "apache-airflow-providers-openlineage>=2.1.3", # Set from
MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
"apache-airflow-providers-opensearch>=1.5.0",
"apache-airflow-providers-opsgenie>=5.8.0",
"apache-airflow-providers-oracle>=3.12.0",
diff --git a/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
b/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
index b3b6fe90c0f..64828883b94 100755
--- a/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
+++ b/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
@@ -57,9 +57,9 @@ CUT_OFF_TIMEDELTA = timedelta(days=6 * 30)
# minimum versions for compatibility with Airflow 3
MIN_VERSION_OVERRIDE: dict[str, Version] = {
"amazon": parse_version("2.1.3"),
- "fab": parse_version("2.0.0"),
+ "fab": parse_version("2.0.2"),
"openlineage": parse_version("2.1.3"),
- "git": parse_version("0.0.1"),
+ "git": parse_version("0.0.2"),
"common.messaging": parse_version("1.0.0"),
}
@@ -89,7 +89,7 @@ console.print("[bright_blue]Updating min-provider versions in
apache-airflow\n")
all_providers_metadata = json.loads(PROVIDER_METADATA_FILE_PATH.read_text())
-def find_min_provider_version(provider_id: str) -> Version | None:
+def find_min_provider_version(provider_id: str) -> tuple[Version | None, str]:
metadata = all_providers_metadata.get(provider_id)
# We should periodically update the starting date to avoid pip install
resolution issues
# TODO: when min Python version is 3.11 change back the code to
fromisoformat
@@ -102,7 +102,7 @@ def find_min_provider_version(provider_id: str) -> Version
| None:
min_version_override = MIN_VERSION_OVERRIDE.get(provider_id)
if not metadata:
if not min_version_override:
- return None
+ return None, ""
last_version_newer_than_cutoff = min_version_override
else:
versions: list[Version] = sorted([parse_version(version) for version
in metadata], reverse=True)
@@ -118,14 +118,17 @@ def find_min_provider_version(provider_id: str) ->
Version | None:
f"[bright_blue]Provider id {provider_id} min version found:[/] "
f"{last_version_newer_than_cutoff} (date {date_released}"
)
+ override_comment = ""
if last_version_newer_than_cutoff:
if min_version_override and min_version_override >
last_version_newer_than_cutoff:
console.print(
f"[yellow]Overriding provider id {provider_id} min version:[/]
{min_version_override} "
- f"overridden from hard-coded versions."
+ f"set from hard-coded versions.\n\n"
+ f"[yellow]Modify MIN_VERSION_OVERRIDE in {__file__} to set
different version![/]\n"
)
last_version_newer_than_cutoff = min_version_override
- return last_version_newer_than_cutoff
+ override_comment = f" # Set from MIN_VERSION_OVERRIDE in
{Path(__file__).name}"
+ return last_version_newer_than_cutoff, override_comment
PROVIDER_MIN_VERSIONS: dict[str, str | None] = {}
@@ -142,11 +145,11 @@ if __name__ == "__main__":
all_provider_lines = []
for provider_id in all_providers:
distribution_name = provider_distribution_name(provider_id)
- min_provider_version = find_min_provider_version(provider_id)
+ min_provider_version, comment = find_min_provider_version(provider_id)
if min_provider_version:
- all_provider_lines.append(f'
"{distribution_name}>={min_provider_version}",\n')
+ all_provider_lines.append(f'
"{distribution_name}>={min_provider_version}",{comment}\n')
all_optional_dependencies.append(
- f'"{provider_id}" = [\n
"{distribution_name}>={min_provider_version}"\n]\n'
+ f'"{provider_id}" = [\n
"{distribution_name}>={min_provider_version}"{comment}\n]\n'
)
else:
all_optional_dependencies.append(f'"{provider_id}" = [\n
"{distribution_name}"\n]\n')
