This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f9b72d8ee44 Pin providers in constraints to the versions published in
PyPI (#71324)
f9b72d8ee44 is described below
commit f9b72d8ee442d5f03b1b554e10668c97f0cfda1e
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Aug 9 10:16:15 2026 +0800
Pin providers in constraints to the versions published in PyPI (#71324)
* Pin providers in constraints to the versions published in PyPI
Constraints cut for a release candidate had to land on the wave being voted
on, whose providers exist in PyPI only as rc versions. Asking uv for a
pre-release strategy left it free to answer with any version satisfying the
lower bounds, so the pins were neither the candidate nor the last release,
and
what a candidate shipped depended on how the resolution happened to go.
Retrieving the versions from PyPI and naming them leaves nothing to resolve:
the constraints pin what is actually published. A pre-release is only ever
considered when the run allows it, and even then it has to sort above every
final release, so a provider without a candidate in the wave keeps its
release
and a released constraints file can never carry an rc pin.
A candidate is exempt from the downgrade check - it sorts below the releases
the constraints branch already carries, so the comparison says nothing
there.
Re-running the workflow for the same candidate now replaces that candidate's
branch and tag instead of pushing onto them, so the two always describe the
run
that produced them and a wave can be re-cut.
* fixup! Pin providers in constraints to the versions published in PyPI
* fixup! Pin providers in constraints to the versions published in PyPI
* fixup! Pin providers in constraints to the versions published in PyPI
* fixup! Pin providers in constraints to the versions published in PyPI
* fixup! Pin providers in constraints to the versions published in PyPI
---
.github/workflows/release-constraints.yml | 34 +++-
dev/README_RELEASE_AIRFLOW.md | 23 ++-
.../airflow_breeze/utils/release_constraints.py | 11 +-
dev/registry/pyproject.toml | 2 +-
providers/akeyless/docs/index.rst | 8 +-
providers/akeyless/pyproject.toml | 2 +-
providers/apache/spark/pyproject.toml | 2 +-
providers/common/ai/pyproject.toml | 2 +-
providers/samba/docs/index.rst | 8 +-
providers/samba/pyproject.toml | 2 +-
scripts/in_container/run_generate_constraints.py | 171 ++++++++++++++-------
.../in_container/test_run_generate_constraints.py | 164 +++++++++++++++++---
12 files changed, 325 insertions(+), 104 deletions(-)
diff --git a/.github/workflows/release-constraints.yml
b/.github/workflows/release-constraints.yml
index 94506af6f01..d815251380f 100644
--- a/.github/workflows/release-constraints.yml
+++ b/.github/workflows/release-constraints.yml
@@ -20,10 +20,11 @@
#
# The stage is derived from the version, so the two cannot be mismatched by
hand:
#
-# * a candidate (`3.1.3rc1`) resolves with pre-releases allowed - the
providers of the wave
-# being voted on exist on PyPI only as rc versions - and lands on a branch
of its own, so a
-# candidate never moves the branch every other build reads;
-# * a final (`3.1.3`) resolves without them, against the providers now
published as finals, and
+# * a candidate (`3.3.1rc1`) pins the providers at the versions PyPI holds -
the wave being
+# voted on exists there only as rc versions - and lands on a branch of its
own, so a candidate
+# never moves the branch every other build reads. Re-running it for the
same candidate
+# replaces that branch and its tag, so the two always describe the run
that produced them;
+# * a final (`3.3.1`) pins the providers at their released versions,
ignoring any candidate, and
# commits onto `constraints-X-Y` itself, which is what makes the released
constraints the
# baseline everything downstream reads.
#
@@ -35,11 +36,11 @@ on: # yamllint disable-line rule:truthy
workflow_dispatch:
inputs:
version:
- description: "Version the constraints belong to, e.g. 3.1.3rc1 or
3.1.3"
+ description: "Version the constraints belong to, e.g. 3.3.1rc1 or
3.3.1"
required: true
type: string
ref:
- description: "Ref the constraints are resolved from, e.g. v3-1-stable
or the release tag"
+ description: "Ref the constraints are resolved from, e.g. v3-3-stable
or the release tag"
required: true
type: string
permissions:
@@ -203,6 +204,7 @@ jobs:
VERSION: ${{ inputs.version }}
CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.constraints-branch }}
TARGET_BRANCH: ${{ needs.build-info.outputs.target-branch }}
+ ALLOW_PRE_RELEASES: ${{ needs.build-info.outputs.allow-pre-releases }}
steps:
- name: "Cleanup repo"
shell: bash
@@ -226,6 +228,26 @@ jobs:
with:
pattern: constraints-*
path: ./files
+ # A candidate's branch and tag belong to that candidate alone, so
re-running for the same
+ # rc replaces them rather than adding to them: the branch would
otherwise already hold the
+ # previous run's constraints (making the push a non-fast-forward) and
the tag already exist.
+ # A final never gets this - it commits onto the shared constraints-X-Y
branch, whose history
+ # every other build reads.
+ - name: "Delete the previous ${{ needs.build-info.outputs.target-branch
}} branch and tag"
+ if: needs.build-info.outputs.allow-pre-releases == 'true'
+ working-directory: "constraints"
+ shell: bash
+ run: |
+ if git ls-remote --exit-code origin "refs/heads/${TARGET_BRANCH}" >
/dev/null; then
+ echo "Deleting the existing '${TARGET_BRANCH}' branch."
+ git push origin --delete "refs/heads/${TARGET_BRANCH}"
+ fi
+ if git ls-remote --exit-code origin
"refs/tags/constraints-${VERSION}" > /dev/null; then
+ echo "Deleting the existing 'constraints-${VERSION}' tag."
+ git push origin --delete "refs/tags/constraints-${VERSION}"
+ fi
+ git tag --delete "constraints-${VERSION}" > /dev/null 2>&1 || true
+ git branch --delete --force "${TARGET_BRANCH}" > /dev/null 2>&1 ||
true
- name: "Switch to ${{ needs.build-info.outputs.target-branch }}"
working-directory: "constraints"
shell: bash
diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md
index 0841475566c..6f140f3126e 100644
--- a/dev/README_RELEASE_AIRFLOW.md
+++ b/dev/README_RELEASE_AIRFLOW.md
@@ -1406,15 +1406,22 @@ breeze workflow-run release-constraints --version
${VERSION} --ref v3-1-stable
The workflow derives the stage from `--version` alone, so there is no separate
switch that could
disagree with the version:
-| `--version` | Pre-releases | Lands on | Tagged |
+| `--version` | Provider pins | Lands on | Tagged |
|---|---|---|---|
-| `3.1.3rc1` | providers only | `constraints-3.1.3rc1`, branched off
`constraints-3-1` | `constraints-3.1.3rc1` |
-| `3.1.3` | none | `constraints-3-1` (commit) | `constraints-3.1.3` |
-
-"Providers only" is literal: the resolution names every
`apache-airflow-providers-*` with a
-pre-release lower bound and runs under uv's `explicit` strategy, which permits
a pre-release solely
-for a package marked that way. Nothing else in the dependency graph can
resolve to one — which is
-why `--pre` is not used, since it would apply to the whole resolution.
+| `3.3.1rc1` | newest in PyPI, `rcN` included | `constraints-3.3.1rc1`,
branched off `constraints-3-3` | `constraints-3.3.1rc1` |
+| `3.3.1` | newest final in PyPI | `constraints-3-3` (commit) |
`constraints-3.3.1` |
+
+The providers are pinned rather than resolved: every
`apache-airflow-providers-*` is named with
+`==` at the newest version PyPI can install, so the constraints record what is
actually published
+instead of whatever the resolver settles on. A candidate takes the wave's
`rcN` versions along with
+it - a pre-release only wins by sorting above every final release, so a
provider without a
+candidate in the wave keeps its release. A final ignores candidates entirely,
which is what makes
+it impossible for a released constraints file to carry an `rc` pin. Nothing
else in the dependency
+graph can resolve to a pre-release, since no other requirement mentions one.
+
+Re-running the workflow for the same candidate deletes and re-creates that
candidate's branch and
+tag, so redoing a candidate's constraints just works. A final is never treated
this way - it commits
+onto the shared `constraints-X-Y` branch, whose history everything downstream
reads.
4. Make sure to update Airflow version in ``v3-*-test`` branch after
cherry-picking to X.Y.1 in
diff --git a/dev/breeze/src/airflow_breeze/utils/release_constraints.py
b/dev/breeze/src/airflow_breeze/utils/release_constraints.py
index 466ab689120..103f45e750b 100644
--- a/dev/breeze/src/airflow_breeze/utils/release_constraints.py
+++ b/dev/breeze/src/airflow_breeze/utils/release_constraints.py
@@ -39,11 +39,12 @@ APACHE_AIRFLOW_REPO = "apache/airflow"
def publish_constraints(*, version: str, ref: str, workflow_branch: str =
"main") -> None:
"""Resolve, publish and tag the constraints belonging to ``version``.
- ``version`` alone decides what happens: a candidate (``3.1.3rc1``)
resolves with pre-releases
- allowed - the providers of the wave being voted on are on PyPI only as rc
versions - and lands
- on a branch of its own, leaving the branch every other build reads where
it was. A final
- (``3.1.3``) resolves without them and commits onto ``constraints-X-Y``,
which is what makes the
- released constraints the baseline everything downstream reads.
+ ``version`` alone decides what happens: a candidate (``3.3.1rc1``) pins
the providers at the
+ versions PyPI holds - the wave being voted on is there only as rc versions
- and lands on a
+ branch of its own, leaving the branch every other build reads where it
was. Re-running it for
+ the same candidate replaces that branch and its tag, so a wave can be
re-cut. A final
+ (``3.3.1``) resolves against the published releases and commits onto
``constraints-X-Y``, which
+ is what makes the released constraints the baseline everything downstream
reads.
"""
stage = "candidate" if "rc" in version else "final"
if not confirm_action(
diff --git a/dev/registry/pyproject.toml b/dev/registry/pyproject.toml
index 2ceef36c66b..6f21421596b 100644
--- a/dev/registry/pyproject.toml
+++ b/dev/registry/pyproject.toml
@@ -45,7 +45,7 @@ dependencies = [
]
[dependency-groups]
-dev = ["pytest"]
+dev = ["pytest>=9.1.1"]
[tool.hatch.build.targets.wheel]
packages = ["registry_tools"]
diff --git a/providers/akeyless/docs/index.rst
b/providers/akeyless/docs/index.rst
index ec6bbd5f176..8e18b2cefc9 100644
--- a/providers/akeyless/docs/index.rst
+++ b/providers/akeyless/docs/index.rst
@@ -117,11 +117,11 @@ Install them when installing from PyPI. For example:
pip install apache-airflow-providers-akeyless[cloud_id]
-============ =====================
+============ ============================
Extra Dependencies
-============ =====================
-``cloud_id`` ``akeyless_cloud_id``
-============ =====================
+============ ============================
+``cloud_id`` ``akeyless-cloud-id>=0.3.0``
+============ ============================
Downloading official packages
-----------------------------
diff --git a/providers/akeyless/pyproject.toml
b/providers/akeyless/pyproject.toml
index f9e70fdf01a..b4b79d90f28 100644
--- a/providers/akeyless/pyproject.toml
+++ b/providers/akeyless/pyproject.toml
@@ -68,7 +68,7 @@ dependencies = [
# Any change in the dependencies is preserved when the file is regenerated
[project.optional-dependencies]
"cloud_id" = [
- "akeyless_cloud_id",
+ "akeyless-cloud-id>=0.3.0",
]
[dependency-groups]
diff --git a/providers/apache/spark/pyproject.toml
b/providers/apache/spark/pyproject.toml
index b7209a2f35c..37e6f6dd898 100644
--- a/providers/apache/spark/pyproject.toml
+++ b/providers/apache/spark/pyproject.toml
@@ -89,7 +89,7 @@ dev = [
"apache-airflow-providers-common-compat",
# Additional devel dependencies (do not remove this line and add extra
development dependencies)
"apache-airflow-providers-openlineage",
- "pyspark"
+ "pyspark>=4.0.0"
]
# To build docs:
diff --git a/providers/common/ai/pyproject.toml
b/providers/common/ai/pyproject.toml
index e97151ecadb..07a65f18dd4 100644
--- a/providers/common/ai/pyproject.toml
+++ b/providers/common/ai/pyproject.toml
@@ -137,7 +137,7 @@ dev = [
"apache-airflow-providers-standard",
# Additional devel dependencies (do not remove this line and add extra
development dependencies)
"sqlglot>=30.0.0",
- "pydantic-ai-slim[mcp]",
+ "pydantic-ai-slim[mcp]>=2.0.0",
"pydantic-ai-skills>=1.2.0",
"apache-airflow-providers-common-sql[datafusion]",
"langchain>=1.0.0",
diff --git a/providers/samba/docs/index.rst b/providers/samba/docs/index.rst
index 35c48f21651..1241d03f31f 100644
--- a/providers/samba/docs/index.rst
+++ b/providers/samba/docs/index.rst
@@ -134,12 +134,12 @@ Install them when installing from PyPI. For example:
pip install apache-airflow-providers-samba[google]
-============ ==========================================
+============ =================================================
Extra Dependencies
-============ ==========================================
+============ =================================================
``google`` ``apache-airflow-providers-google``
-``kerberos`` ``krb5``, ``smbprotocol[kerberos]>=1.5.0``
-============ ==========================================
+``kerberos`` ``krb5>=0.8.0``, ``smbprotocol[kerberos]>=1.5.0``
+============ =================================================
Downloading official packages
-----------------------------
diff --git a/providers/samba/pyproject.toml b/providers/samba/pyproject.toml
index d5105dd606d..36dc9b2b6d9 100644
--- a/providers/samba/pyproject.toml
+++ b/providers/samba/pyproject.toml
@@ -71,7 +71,7 @@ dependencies = [
"apache-airflow-providers-google"
]
"kerberos" = [
- "krb5",
+ "krb5>=0.8.0",
"smbprotocol[kerberos]>=1.5.0",
]
diff --git a/scripts/in_container/run_generate_constraints.py
b/scripts/in_container/run_generate_constraints.py
index c1cca95e2c2..6bd75777253 100755
--- a/scripts/in_container/run_generate_constraints.py
+++ b/scripts/in_container/run_generate_constraints.py
@@ -21,15 +21,19 @@ import ast
import json
import os
import sys
+from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from datetime import datetime
from functools import cached_property
+from itertools import repeat
from pathlib import Path
from typing import TextIO
import requests
from click import Choice
from in_container_utils import AIRFLOW_DIST_PATH, AIRFLOW_ROOT_PATH, click,
console, run_command
+from packaging.specifiers import InvalidSpecifier, SpecifierSet
+from packaging.version import InvalidVersion, Version
try:
import tomllib
@@ -39,6 +43,8 @@ except ImportError:
DEFAULT_BRANCH = os.environ.get("DEFAULT_BRANCH", "main")
PYTHON_VERSION = os.environ.get("PYTHON_MAJOR_MINOR_VERSION", "3.10")
GENERATED_PROVIDER_DEPENDENCIES_FILE = AIRFLOW_ROOT_PATH / "generated" /
"provider_dependencies.json"
+PYPI_JSON_API_URL = "https://pypi.org/pypi/{distribution}/json";
+PYPI_LOOKUP_PARALLELISM = 8
def _read_version_from_pyproject(pyproject_path: Path) -> str:
@@ -380,11 +386,19 @@ def check_providers_not_downgraded(config_params:
ConfigParams) -> None:
Fail generation if any released provider is downgraded compared to the
latest constraints.
Released provider versions only ever move forward on PyPI, so a lower
version in the freshly
- generated constraints signals a resolution problem (a broken dependency
forcing an old provider
- back in) rather than an intended change. We stop here so it is caught
instead of being published.
- """
- from packaging.version import InvalidVersion, Version
+ generated constraints means the version we had pinned is no longer
installable - yanked, or
+ left without a file for the Python being resolved. We stop here so it is
caught instead of
+ being published.
+ A release candidate is exempt: it pins the wave being voted on, which
sorts below the released
+ versions a constraints branch may already carry, so the comparison says
nothing there.
+ """
+ if config_params.allow_pre_releases:
+ console.print(
+ "[yellow]Pre-releases are allowed - skipping the provider
downgrade check, a candidate "
+ "sorts below the releases the constraints branch carries."
+ )
+ return
if not config_params.latest_constraints_file.exists():
console.print("[yellow]No previous constraints file downloaded -
skipping provider downgrade check.")
return
@@ -408,10 +422,10 @@ def check_providers_not_downgraded(config_params:
ConfigParams) -> None:
for provider, latest_version, current_version in sorted(downgraded):
console.print(f"[red] * {provider}: {latest_version} ->
{current_version}")
console.print(
- "[yellow]Released providers should never be downgraded. This
usually means a broken "
- "dependency version forced an older provider back in during
resolution. Investigate the "
- "diff above and, if needed, add an exclusion in the "
- f"`additional_constraints_for_highest_resolution` list in [/]
{__file__}"
+ "[yellow]Released providers should never be downgraded. The
providers are pinned at the "
+ "newest version PyPI serves for this Python, so a lower one means
the version we had is "
+ "gone - yanked, or no longer offering a file this Python can
install. Investigate the "
+ "diff above before publishing.[/]"
)
write_provider_downgrade_slack_message(config_params, downgraded)
sys.exit(1)
@@ -459,16 +473,82 @@ def get_all_active_provider_distributions(python_version:
str | None = None) ->
]
-def build_provider_pre_release_requirements(python_version: str) -> list[str]:
- """Requirements that let only the providers resolve to a pre-release.
-
- uv considers a pre-release for a package only when some requirement for it
mentions one, so a
- pre-release lower bound on each provider confines the allowance to them.
`--pre` would apply to
- the whole resolution and could put a pre-release of any third-party
dependency into the
- constraints a release ships.
+def is_file_installable(pypi_file: dict, target_python: Version) -> bool:
+ """Whether a file PyPI lists for a release can be installed on
``target_python``."""
+ if pypi_file.get("yanked"):
+ return False
+ requires_python = pypi_file.get("requires_python")
+ if not requires_python:
+ return True
+ try:
+ return target_python in SpecifierSet(requires_python)
+ except InvalidSpecifier:
+ return True
+
+
+def find_newest_version_in_pypi(
+ distribution: str, python_version: str, allow_pre_releases: bool
+) -> str | None:
+ """Return the newest version of ``distribution`` PyPI can install for
``python_version``.
+
+ A pre-release is only ever considered when ``allow_pre_releases`` is set,
and even then it wins
+ only by sorting above every final release - so a provider with a candidate
in the wave being
+ voted on resolves to that candidate while one without keeps its last
release. Versions PyPI can
+ no longer serve for the Python being resolved - fully yanked, or excluded
by ``requires_python``
+ - are passed over, because pinning one leaves the resolution nothing to
install. ``None``
+ (nothing installable at all, e.g. a provider whose first release is still
in this wave) leaves
+ the distribution unpinned rather than pinned to a version that cannot be
had.
"""
+ response =
requests.get(PYPI_JSON_API_URL.format(distribution=distribution), timeout=60)
+ if response.status_code == 404:
+ console.print(f"[yellow]{distribution} is not published in PyPI -
leaving it unpinned.")
+ return None
+ response.raise_for_status()
+ target_python = Version(python_version)
+ newest_version: Version | None = None
+ for version, files in response.json().get("releases", {}).items():
+ try:
+ parsed_version = Version(version)
+ except InvalidVersion:
+ continue
+ if parsed_version.is_prerelease and not allow_pre_releases:
+ continue
+ if not any(is_file_installable(pypi_file, target_python) for pypi_file
in files):
+ continue
+ if newest_version is None or parsed_version > newest_version:
+ newest_version = parsed_version
+ if newest_version is None:
+ console.print(f"[yellow]{distribution} has no installable version in
PyPI - leaving it unpinned.")
+ return None
+ return str(newest_version)
+
+
+def build_pinned_provider_requirements(python_version: str,
allow_pre_releases: bool) -> list[str]:
+ """Exact pins for every active provider, at the newest version PyPI holds
for it.
+
+ Naming the versions retrieved from PyPI leaves the providers nothing to
resolve: the constraints
+ pin what is actually published rather than whatever the resolver settles
on. It is what makes a
+ release candidate land on the wave being voted on, whose providers are
published only as rc
+ versions - handing uv a pre-release strategy instead left it free to
answer with any version
+ satisfying the lower bounds. It also keeps pre-releases confined to the
providers: no
+ third-party dependency can answer with one, because no requirement here
mentions a pre-release
+ of anything else.
+ """
+ distributions = get_all_active_provider_distributions(python_version)
+ console.print(f"[bright_blue]Retrieving the newest PyPI version of
{len(distributions)} providers.")
+ with ThreadPoolExecutor(max_workers=PYPI_LOOKUP_PARALLELISM) as executor:
+ newest_versions = list(
+ executor.map(
+ find_newest_version_in_pypi,
+ distributions,
+ repeat(python_version),
+ repeat(allow_pre_releases),
+ )
+ )
return [
- f"{distribution}>=0.0.0rc0" for distribution in
get_all_active_provider_distributions(python_version)
+ f"{distribution}=={version}"
+ for distribution, version in zip(distributions, newest_versions)
+ if version is not None
]
@@ -532,17 +612,13 @@ def generate_constraints_pypi_providers(config_params:
ConfigParams) -> None:
# that the resolver will not downgrade the provider.
# * opentelemetry-exporter-prometheus>=0.47b0 — this package only ever
publishes beta versions
# (airflow-core requires ``>=0.47b0`` and released constraints already
pin a beta, e.g.
- # ``==0.65b0``). For a release candidate the resolution runs with
``--prerelease explicit``,
- # which permits a pre-release only for a package some requirement marks
as such and drops the
- # if-necessary fallback; without this entry the package cannot resolve
and generation fails
- # with "No solution found". Keeping it here (rather than the provider
pre-release list) marks
- # it as an always-allowed pre-release across every resolution, matching
how it already ships.
+ # ``==0.65b0``). Marking it as a pre-release here rather than relying on
uv's if-necessary
+ # fallback keeps the choice deliberate and identical across every
resolution.
# * opentelemetry-semantic-conventions>=0.48b0 — same story: a beta-only
package, pulled in as a
- # hard dependency of opentelemetry-sdk (via opentelemetry-exporter-otlp
and shared/observability),
- # so the ``--prerelease explicit`` resolution needs the same explicit
mark. The floor only marks
- # it as a pre-release and must stay at the version paired with our
``opentelemetry-*>=1.27.0``
- # floor — opentelemetry-sdk exact-pins the semantic conventions version
it ships with, so a
- # higher floor here would conflict with any sdk older than that pairing.
+ # hard dependency of opentelemetry-sdk (via opentelemetry-exporter-otlp
and shared/observability).
+ # The floor only marks it as a pre-release and must stay at the version
paired with our
+ # ``opentelemetry-*>=1.27.0`` floor — opentelemetry-sdk exact-pins the
semantic conventions
+ # version it ships with, so a higher floor here would conflict with any
sdk older than that pairing.
#
# These two are the only pre-releases in the constraints we tag, and
removing the need for the
# exception is tracked at https://github.com/apache/airflow/issues/71176
@@ -554,27 +630,16 @@ def generate_constraints_pypi_providers(config_params:
ConfigParams) -> None:
"opentelemetry-semantic-conventions>=0.48b0",
]
- # Constraints cut for a release candidate have to pin the candidates
themselves - the providers
- # for that wave exist on PyPI only as rcN versions, and uv will not
resolve to a pre-release
- # unless asked. The final release regenerates these without it, so a
released constraints file
- # can never carry an rc pin.
- #
- # Scoped to the providers rather than passing `--pre`, which applies to
the whole resolution
- # and would let any dependency answer with a pre-release - putting, say, a
beta of a
- # third-party library into the constraints a release ships. uv has no
per-package pre-release
- # flag, so the scoping is expressed the way it does support: `explicit`
permits a pre-release
- # only for a package some requirement marks as such, and the rc lower
bound below is that mark.
- # `explicit` rather than the default `if-necessary-or-explicit` also drops
the fallback that
- # would otherwise let an unmarked package resolve to a pre-release when no
final satisfies.
- pre_release_requirements: list[str] = []
- pre_release_strategy: list[str] = []
- if config_params.allow_pre_releases:
- pre_release_requirements =
build_provider_pre_release_requirements(config_params.python)
- pre_release_strategy = ["--prerelease", "explicit"]
- console.print(
- f"[bright_blue]Allowing pre-releases for
{len(pre_release_requirements)} provider "
- "distributions - no other package can resolve to one."
- )
+ # Every run pins the providers at the versions PyPI holds when it is
generated. Only a run for
+ # a release candidate considers the rc versions of the wave being voted
on; a final regenerates
+ # these against the published releases, so a released constraints file can
never carry an rc pin.
+ pinned_provider_requirements = build_pinned_provider_requirements(
+ config_params.python, config_params.allow_pre_releases
+ )
+ console.print(
+ f"[bright_blue]Pinning {len(pinned_provider_requirements)} provider
distributions to the "
+ f"{'newest' if config_params.allow_pre_releases else 'newest final'}
versions in PyPI."
+ )
result = run_command(
cmd=[
@@ -589,8 +654,7 @@ def generate_constraints_pypi_providers(config_params:
ConfigParams) -> None:
f"apache-airflow-task-sdk=={AIRFLOW_TASK_SDK_VERSION}",
"./airflow-ctl",
*additional_constraints_for_highest_resolution,
- *pre_release_requirements,
- *pre_release_strategy,
+ *pinned_provider_requirements,
"--reinstall", # We need to pull the provider distributions from
PyPI or dist, not the local ones
"--resolution",
"highest",
@@ -603,7 +667,9 @@ def generate_constraints_pypi_providers(config_params:
ConfigParams) -> None:
if result.returncode != 0:
console.print(
"[red]Failed to install airflow with PyPI providers with highest
resolution.[/]\n"
- "[yellow]Please check the output above for details. One of they
ways how to resolve it, in "
+ "[yellow]Please check the output above for details. The providers
are pinned at the newest "
+ "versions PyPI serves, so two of them requiring incompatible
dependencies fails here rather "
+ "than quietly settling on an older provider. One of they ways how
to resolve it, in "
"case it is caused by a specific broken dependency version, is to
exclude it above in the "
f"`additional_constraints_for_highest_resolution` list in [/]
{__file__}"
)
@@ -693,8 +759,9 @@ ALLOWED_CONSTRAINTS_MODES = ["constraints",
"constraints-source-providers", "con
"--allow-pre-releases",
is_flag=True,
default=False,
- help="Allow pre-release versions of Airflow and providers to be pinned.
Used when constraints "
- "are generated for a release candidate, whose providers are only on PyPI
as rc versions.",
+ help="Let the provider pins use pre-release versions when those are newer
than any final "
+ "release. Used when constraints are generated for a release candidate,
whose providers are "
+ "only on PyPI as rc versions.",
envvar="ALLOW_PRE_RELEASES",
)
def generate_constraints(
diff --git a/scripts/tests/in_container/test_run_generate_constraints.py
b/scripts/tests/in_container/test_run_generate_constraints.py
index 88af9e84102..99c7f4c5ee9 100644
--- a/scripts/tests/in_container/test_run_generate_constraints.py
+++ b/scripts/tests/in_container/test_run_generate_constraints.py
@@ -24,15 +24,38 @@ import pytest
import run_generate_constraints as m
-def _config(latest: Path, current: Path, constraints_dir: Path | None = None)
-> SimpleNamespace:
+def _config(
+ latest: Path,
+ current: Path,
+ constraints_dir: Path | None = None,
+ allow_pre_releases: bool = False,
+) -> SimpleNamespace:
return SimpleNamespace(
latest_constraints_file=latest,
current_constraints_file=current,
constraints_dir=constraints_dir if constraints_dir is not None else
latest.parent,
python="3.10",
+ allow_pre_releases=allow_pre_releases,
)
+def _file(*, yanked: bool = False, requires_python: str | None = ">=3.10") ->
dict:
+ return {"yanked": yanked, "requires_python": requires_python}
+
+
+class _FakeResponse:
+ def __init__(self, payload: dict, status_code: int = 200):
+ self._payload = payload
+ self.status_code = status_code
+
+ def json(self) -> dict:
+ return self._payload
+
+ def raise_for_status(self) -> None:
+ if self.status_code >= 400:
+ raise AssertionError(f"Unexpected status {self.status_code}")
+
+
class TestReadProviderVersionsFromConstraints:
def test_extracts_only_providers_and_strips_markers(self, tmp_path):
constraints = tmp_path / "constraints.txt"
@@ -82,6 +105,16 @@ class TestCheckProvidersNotDowngraded:
payload = json.loads((tmp_path /
"provider-downgrade-slack-message.json").read_text())
assert payload["channel"] == "some-other-channel"
+ def test_skips_the_check_for_a_release_candidate(self, tmp_path):
+ latest = tmp_path / "latest.txt"
+ current = tmp_path / "current.txt"
+ latest.write_text("apache-airflow-providers-amazon==8.0.0\n")
+ current.write_text("apache-airflow-providers-amazon==7.5.0\n")
+
+ m.check_providers_not_downgraded(_config(latest, current,
allow_pre_releases=True))
+
+ assert not (tmp_path /
"provider-downgrade-slack-message.json").exists()
+
def test_passes_when_no_provider_is_downgraded(self, tmp_path):
latest = tmp_path / "latest.txt"
current = tmp_path / "current.txt"
@@ -111,40 +144,131 @@ class TestCheckProvidersNotDowngraded:
m.check_providers_not_downgraded(_config(latest, current))
-class TestBuildProviderPreReleaseRequirements:
- """Pre-releases are allowed for the providers only, never for the whole
resolution."""
+class TestFindNewestVersionInPypi:
+ @pytest.mark.parametrize(
+ "releases, allow_pre_releases, expected",
+ [
+ pytest.param(
+ {"1.0.0": [_file()], "1.1.0": [_file()]},
+ False,
+ "1.1.0",
+ id="newest-final-release",
+ ),
+ pytest.param(
+ {"1.1.0": [_file()], "1.2.0rc1": [_file()]},
+ True,
+ "1.2.0rc1",
+ id="candidate-newer-than-the-release-wins-when-allowed",
+ ),
+ pytest.param(
+ {"1.1.0": [_file()], "1.2.0rc1": [_file()]},
+ False,
+ "1.1.0",
+ id="candidate-is-ignored-when-not-allowed",
+ ),
+ pytest.param(
+ {"1.1.0rc1": [_file()], "1.1.0": [_file()]},
+ True,
+ "1.1.0",
+ id="superseded-candidate-loses-to-its-release",
+ ),
+ pytest.param(
+ {"1.0.0": [_file()], "1.1.0": [_file(yanked=True)]},
+ False,
+ "1.0.0",
+ id="yanked-version-is-passed-over",
+ ),
+ pytest.param(
+ {"1.0.0": [_file()], "1.1.0": []},
+ False,
+ "1.0.0",
+ id="version-with-no-files-is-passed-over",
+ ),
+ pytest.param(
+ {"1.0.0": [_file()], "1.1.0":
[_file(requires_python=">=3.12")]},
+ False,
+ "1.0.0",
+ id="version-excluded-by-requires-python-is-passed-over",
+ ),
+ pytest.param(
+ {"1.0.0": [_file()], "not-a-version": [_file()]},
+ False,
+ "1.0.0",
+ id="unparseable-version-is-passed-over",
+ ),
+ pytest.param(
+ {"1.0.0": [_file(requires_python=None)]},
+ False,
+ "1.0.0",
+ id="file-without-requires-python-is-installable",
+ ),
+ pytest.param(
+ {"1.0.0": [_file(requires_python="not-a-specifier")]},
+ False,
+ "1.0.0",
+ id="file-with-broken-requires-python-is-not-excluded",
+ ),
+ pytest.param({"1.0.0": [_file(yanked=True)]}, False, None,
id="nothing-installable-is-unpinned"),
+ pytest.param({"1.0.0rc1": [_file()]}, False, None,
id="only-a-candidate-is-unpinned"),
+ ],
+ )
+ def test_picks_the_newest_installable_version(self, monkeypatch, releases,
allow_pre_releases, expected):
+ monkeypatch.setattr(m.requests, "get", lambda url, timeout:
_FakeResponse({"releases": releases}))
+
+ newest =
m.find_newest_version_in_pypi("apache-airflow-providers-amazon", "3.10",
allow_pre_releases)
+
+ assert newest == expected
+
+ def test_unpublished_distribution_is_unpinned(self, monkeypatch):
+ monkeypatch.setattr(m.requests, "get", lambda url, timeout:
_FakeResponse({}, status_code=404))
+
+ assert
m.find_newest_version_in_pypi("apache-airflow-providers-brand-new", "3.10",
True) is None
+
- def
test_every_requirement_names_a_provider_and_permits_a_pre_release(self,
monkeypatch):
+class TestBuildPinnedProviderRequirements:
+ """Providers are pinned at what PyPI holds rather than left for the
resolution to pick."""
+
+ def test_every_provider_is_pinned_to_its_newest_pypi_version(self,
monkeypatch):
monkeypatch.setattr(
m,
"get_all_active_provider_distributions",
lambda python_version=None: [
"apache-airflow-providers-amazon",
"apache-airflow-providers-cncf-kubernetes",
+ "apache-airflow-providers-brand-new",
],
)
+ newest = {
+ "apache-airflow-providers-amazon": "9.0.0rc1",
+ "apache-airflow-providers-cncf-kubernetes": "10.20.0",
+ "apache-airflow-providers-brand-new": None,
+ }
+ monkeypatch.setattr(
+ m, "find_newest_version_in_pypi", lambda dist, python_version,
allow_pre_releases: newest[dist]
+ )
- requirements = m.build_provider_pre_release_requirements("3.10")
-
- assert requirements == [
- "apache-airflow-providers-amazon>=0.0.0rc0",
- "apache-airflow-providers-cncf-kubernetes>=0.0.0rc0",
+ assert m.build_pinned_provider_requirements("3.10", True) == [
+ "apache-airflow-providers-amazon==9.0.0rc1",
+ "apache-airflow-providers-cncf-kubernetes==10.20.0",
]
- # The rc lower bound is what marks the package as explicit to uv;
without it the
- # requirement would not permit a pre-release at all.
- assert all(requirement.endswith(">=0.0.0rc0") for requirement in
requirements)
- assert all(requirement.startswith("apache-airflow-providers-") for
requirement in requirements)
- def test_the_python_version_is_passed_through(self, monkeypatch):
- """Providers excluded on a Python version must not be named for it."""
- seen: list[str | None] = []
+ def
test_the_python_version_and_the_pre_release_choice_reach_the_lookup(self,
monkeypatch):
+ """Providers excluded on a Python version must not be named, nor
pinned to a version it cannot install."""
+ seen_for_distributions: list[str | None] = []
+ seen_for_lookup: list[tuple[str, bool]] = []
def fake_distributions(python_version=None):
- seen.append(python_version)
- return []
+ seen_for_distributions.append(python_version)
+ return ["apache-airflow-providers-amazon"]
+
+ def fake_lookup(distribution, python_version, allow_pre_releases):
+ seen_for_lookup.append((python_version, allow_pre_releases))
+ return "9.0.0"
monkeypatch.setattr(m, "get_all_active_provider_distributions",
fake_distributions)
+ monkeypatch.setattr(m, "find_newest_version_in_pypi", fake_lookup)
- m.build_provider_pre_release_requirements("3.14")
+ m.build_pinned_provider_requirements("3.14", False)
- assert seen == ["3.14"]
+ assert seen_for_distributions == ["3.14"]
+ assert seen_for_lookup == [("3.14", False)]
