This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-4-test by this push:
new 471367f59ac [v3-4-test] Stop generating constraints for PRs that do
not use them (#76685) (#76849)
471367f59ac is described below
commit 471367f59ac8757567c2b7032711fc208e64b986
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Nov 10 21:54:45 2026 +0100
[v3-4-test] Stop generating constraints for PRs that do not use them
(#76685) (#76849)
Generate constraints ran for every PR that builds the CI image, about 149
jobs a day, though for 46% of those runs nothing reads its output. The PROD
image, its only reader on an ordinary PR, reads just the source-providers
file. The PyPI step asks PyPI for 108 providers and resolves 725 packages
against what it serves, which a change that touches no dependency cannot
affect.
That step turned PyPI outages into red PRs: of the 51 jobs that failed from
2026-10-16 to 2026-11-05, 40 were network errors and 38 were on PRs that
touched no dependency files. The only two failures no other job caught were
in PRs that changed dependency files, which still run it. Canary runs still
cover the state of PyPI.
Claude-Session: https://claude.ai/code/session_01BVx6g6v9hW7wi1tV4bqvrp
(cherry picked from commit e1150d774f8ac3462771138a0e03932d6af83735)
Generated-by: Claude Opus 5
Co-authored-by: Shahar Epstein <[email protected]>
---
.github/workflows/ci-amd.yml | 6 +-
.github/workflows/ci-arm.yml | 6 +-
dev/breeze/doc/ci/04_selective_checks.md | 8 ++
dev/breeze/doc/ci/05_workflows.md | 9 ++-
.../src/airflow_breeze/utils/selective_checks.py | 18 +++++
dev/breeze/tests/test_selective_checks.py | 93 ++++++++++++++++++++++
6 files changed, 135 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml
index 64879e8598a..10cd2a85960 100644
--- a/.github/workflows/ci-amd.yml
+++ b/.github/workflows/ci-amd.yml
@@ -297,6 +297,8 @@ jobs:
excluded-providers-as-string: ${{
steps.selective-checks.outputs.excluded-providers-as-string }}
force-pip: ${{ steps.selective-checks.outputs.force-pip }}
full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed
}}
+ generate-constraints: ${{
steps.selective-checks.outputs.generate-constraints }}
+ generate-pypi-constraints: ${{
steps.selective-checks.outputs.generate-pypi-constraints }}
has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
helm-test-packages: ${{
steps.selective-checks.outputs.helm-test-packages }}
helm-test-kubernetes-versions: ${{
steps.selective-checks.outputs.helm-test-kubernetes-versions }}
@@ -564,13 +566,13 @@ jobs:
name: "Generate constraints"
needs: [build-info, build-ci-images]
uses: ./.github/workflows/generate-constraints.yml
- if: needs.build-info.outputs.ci-image-build == 'true'
+ if: needs.build-info.outputs.generate-constraints == 'true'
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
- generate-pypi-constraints: "true"
+ generate-pypi-constraints: ${{
needs.build-info.outputs.generate-pypi-constraints }}
# generate no providers constraints only in canary builds - they take
quite some time to generate
# they are not needed for regular builds, they are only needed to update
constraints in canaries
generate-no-providers-constraints: ${{
needs.build-info.outputs.canary-run }}
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
index e35600faa6a..0c877c17767 100644
--- a/.github/workflows/ci-arm.yml
+++ b/.github/workflows/ci-arm.yml
@@ -173,6 +173,8 @@ jobs:
excluded-providers-as-string: ${{
steps.selective-checks.outputs.excluded-providers-as-string }}
force-pip: ${{ steps.selective-checks.outputs.force-pip }}
full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed
}}
+ generate-constraints: ${{
steps.selective-checks.outputs.generate-constraints }}
+ generate-pypi-constraints: ${{
steps.selective-checks.outputs.generate-pypi-constraints }}
has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
helm-test-packages: ${{
steps.selective-checks.outputs.helm-test-packages }}
helm-test-kubernetes-versions: ${{
steps.selective-checks.outputs.helm-test-kubernetes-versions }}
@@ -440,13 +442,13 @@ jobs:
name: "Generate constraints"
needs: [build-info, build-ci-images]
uses: ./.github/workflows/generate-constraints.yml
- if: needs.build-info.outputs.ci-image-build == 'true'
+ if: needs.build-info.outputs.generate-constraints == 'true'
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
- generate-pypi-constraints: "true"
+ generate-pypi-constraints: ${{
needs.build-info.outputs.generate-pypi-constraints }}
# generate no providers constraints only in canary builds - they take
quite some time to generate
# they are not needed for regular builds, they are only needed to update
constraints in canaries
generate-no-providers-constraints: ${{
needs.build-info.outputs.canary-run }}
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index 4c14a33e999..28222b79c76 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -510,6 +510,12 @@ together using `pytest-xdist` (pytest-xdist distributes
the tests among parallel
* If no Source files are changed - no tests are run and no further rules below
are checked.
* `CI Image building` is enabled if either test are run, docs are build.
* `PROD Image building` is enabled when kubernetes tests are run.
+* `Generate constraints` runs when the PROD image is built, because that build
reads the
+ `constraints-source-providers` file, or when `generate-pypi-constraints` is
on. The PyPI constraints and
+ the distributions they need are generated only when the CI image is built
and `pyproject.toml` or `uv.lock`
+ changed, full tests are needed (which covers the job's own workflow and
script), the run is a canary, or
+ the `upgrade to newer dependencies` label is set. Any other change leaves
nothing for this resolution to
+ check except the state of PyPI, which the canary runs cover.
* In case of `Providers` test in regular PRs, additional check is done in
order to determine which
providers are affected and the actual selection is made based on that:
* if directly provider code is changed (either in the provider, test or
system tests) then this provider
@@ -709,6 +715,8 @@ GitHub Actions to pass the list of parameters to a command
to execute
| excluded-providers-as-string | List of providers
that should be excluded from the build as space-separated string
| amazon google | * |
| force-pip | Whether pip should
be forced in the image build instead of uv ("true"/"false")
| false | |
| full-tests-needed | Whether this build
runs complete set of tests or only subset (for faster PR builds) \[1\]
| false | |
+| generate-constraints | Whether the
Generate constraints job runs
| false | |
+| generate-pypi-constraints | Whether that job
also resolves the constraints against PyPI
| false | |
| generated-dependencies-changed | Whether generated
dependencies have changed ("true"/"false")
| false | |
| has-migrations | Whether the PR has
migrations ("true"/"false")
| false | |
| hatch-build-changed | When hatch
build.py changed in the PR.
| false | |
diff --git a/dev/breeze/doc/ci/05_workflows.md
b/dev/breeze/doc/ci/05_workflows.md
index 8ea6810bb20..698513a035e 100644
--- a/dev/breeze/doc/ci/05_workflows.md
+++ b/dev/breeze/doc/ci/05_workflows.md
@@ -448,7 +448,7 @@ Here's what each workflow group does and when it runs:
| **Basic Tests** | Runs quick validation: Breeze, UI,
static checks | Yes | Yes | Yes |
| **CI Image Build** | Builds Docker images for testing
| Yes | Yes | Yes |
| **Additional CI Image Checks** | Validates CI images meet quality
standards | Yes | Yes | Yes |
-| **Generate Constraints** | Creates dependency version lock files
| Yes | Yes | Yes |
+| **Generate Constraints** | Creates dependency version lock files
| Yes (6) | Yes | Yes (6) |
| **CI Image Checks** | Runs static analysis and builds docs
(AMD only) | Yes | Yes | Yes |
| **Provider Tests** | Tests integration with external
services | Yes | Yes | Yes (1) |
| **Unit Tests (Multiple Groups)** | Runs core functionality tests with
different databases | Yes | Yes | Yes (1) |
@@ -541,6 +541,13 @@ The summary runs only when the run upgrades to newer
dependencies: on scheduled
dispatched runs, when a `pyproject.toml` file changes, or on a PR with the
`upgrade to newer dependencies` label. It needs the CI image, so it never runs
without one.
+**`(6)` Generate Constraints**
+
+The job runs only when the PROD image is built or when the constraints are
resolved against PyPI. It
+resolves against PyPI when dependencies change, when full tests are needed, on
canary runs, and on a PR
+with the `upgrade to newer dependencies` label. A run that only builds the
PROD image generates just the
+`constraints-source-providers` file that image reads.
+
## Runners
Two kinds of GitHub-hosted runner are used, and which one a job gets depends
on what
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index f8c3524e801..d04fabb16e9 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -2168,6 +2168,24 @@ class SelectiveChecks:
return True
return False
+ @cached_property
+ def generate_pypi_constraints(self) -> bool:
+ # Without a dependency or tooling change this resolution only reports
on the state of PyPI,
+ # which the canary runs cover. Their dependency report and
`update-constraints` read its output.
+ if not self.ci_image_build:
+ return False
+ return (
+ self.upgrade_to_newer_dependencies
+ or self.is_uv_lock_update
+ or self.full_tests_needed
+ or self._is_canary_run()
+ )
+
+ @cached_property
+ def generate_constraints(self) -> bool:
+ # The PROD image build reads the source-providers constraints the job
always generates.
+ return self.generate_pypi_constraints or self.prod_image_build
+
@cached_property
def docs_list_as_string(self) -> str | None:
_ALL_DOCS_LIST = ""
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 8ec11a13d1c..025c3a3c8a8 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -3703,6 +3703,97 @@ def test_upgrade_to_newer_dependencies(
assert_outputs_are_printed(expected_outputs, str(stderr))
[email protected](
+ ("files", "pr_labels", "github_event", "expected_constraints",
"expected_pypi"),
+ [
+ pytest.param(
+
("providers/edge3/src/airflow/providers/edge3/models/edge_job.py",),
+ (),
+ GithubEvents.PULL_REQUEST,
+ "false",
+ "false",
+ id="provider-source-needs-neither",
+ ),
+ pytest.param(
+ ("airflow-core/docs/index.rst",),
+ (),
+ GithubEvents.PULL_REQUEST,
+ "false",
+ "false",
+ id="docs-build-the-ci-image-only",
+ ),
+ pytest.param(
+
("providers/edge3/src/airflow/providers/edge3/models/edge_job.py",),
+ ("area:e2e-tests",),
+ GithubEvents.PULL_REQUEST,
+ "true",
+ "false",
+ id="prod-image-needs-source-constraints-only",
+ ),
+ pytest.param(
+ ("providers/edge3/pyproject.toml",),
+ (),
+ GithubEvents.PULL_REQUEST,
+ "true",
+ "true",
+ id="provider-pyproject-resolves-against-pypi",
+ ),
+ pytest.param(
+ ("uv.lock", "airflow-core/src/airflow/models/dag.py"),
+ (),
+ GithubEvents.PULL_REQUEST,
+ "true",
+ "true",
+ id="uv-lock-resolves-against-pypi",
+ ),
+ pytest.param(
+ ("scripts/in_container/run_generate_constraints.py",),
+ (),
+ GithubEvents.PULL_REQUEST,
+ "true",
+ "true",
+ id="constraints-script-resolves-against-pypi",
+ ),
+ pytest.param(
+
("providers/edge3/src/airflow/providers/edge3/models/edge_job.py",),
+ ("upgrade to newer dependencies",),
+ GithubEvents.PULL_REQUEST,
+ "true",
+ "true",
+ id="upgrade-label-resolves-against-pypi",
+ ),
+ pytest.param(
+
("providers/edge3/src/airflow/providers/edge3/models/edge_job.py",),
+ ("canary",),
+ GithubEvents.PULL_REQUEST,
+ "true",
+ "true",
+ id="canary-label-resolves-against-pypi",
+ ),
+ pytest.param(
+
("providers/edge3/src/airflow/providers/edge3/models/edge_job.py",),
+ (),
+ GithubEvents.SCHEDULE,
+ "true",
+ "true",
+ id="schedule-resolves-against-pypi",
+ ),
+ ],
+)
+def test_generate_constraints(files, pr_labels, github_event,
expected_constraints, expected_pypi):
+ checks = SelectiveChecks(
+ files=files,
+ commit_ref="HEAD",
+ github_event=github_event,
+ default_branch="main",
+ pr_labels=pr_labels,
+ )
+ assert_outputs_are_printed(
+ {"generate-constraints": expected_constraints,
"generate-pypi-constraints": expected_pypi},
+ str(checks),
+ )
+
+
@pytest.mark.parametrize(
("files", "expected_outputs"),
[
@@ -5949,6 +6040,8 @@ def
test_component_branch_upgrades_dependencies_only_with_the_ci_image(context,
checks = _component_branch_checks(context, ("airflow-ctl/pyproject.toml",
"chart/pyproject.toml"))
assert checks.upgrade_to_newer_dependencies is expected
+ assert checks.generate_constraints is expected
+ assert checks.generate_pypi_constraints is expected
@pytest.mark.parametrize(
