This is an automated email from the ASF dual-hosted git repository. vatsrahul1001 pushed a commit to branch constraints-build-not-ready-providers-from-source in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 544945269566244979d17bc060939bf7db3c08c0 Author: Rahul Vats <[email protected]> AuthorDate: Wed Aug 5 14:19:05 2026 +0530 Build not-ready providers from source when generating PyPI constraints The release constraints resolution installs apache-airflow[all], which depends on every non-suspended provider. not-ready providers (currently ibm.mq and common.dataquality) are never published to PyPI, so the PyPI-only release path cannot satisfy them and fails with "No solution found". Install them from local source so the resolution stays solvable; pip freeze renders a path install as a file:// line that is already dropped from the constraints file, so a not-ready provider is never pinned in released constraints. --- scripts/in_container/run_generate_constraints.py | 27 +++++++++++++++++ .../in_container/test_run_generate_constraints.py | 34 ++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/scripts/in_container/run_generate_constraints.py b/scripts/in_container/run_generate_constraints.py index ba23a0042b0..2247c22cbb7 100755 --- a/scripts/in_container/run_generate_constraints.py +++ b/scripts/in_container/run_generate_constraints.py @@ -459,6 +459,30 @@ def get_all_active_provider_distributions(python_version: str | None = None) -> ] +def get_not_ready_provider_source_paths(python_version: str | None = None) -> list[str]: + """ + Get local source paths of the ``not-ready`` providers that ``apache-airflow[all]`` requires. + + ``[all]`` depends on every non-suspended provider, but ``not-ready`` providers are never + published to PyPI, so a PyPI-only resolution cannot satisfy them and fails with "No solution + found". The release path (``pypi-providers-only``) resolves from PyPI and skips building + providers into ``dist`` first, so these have nowhere to resolve from; installing them from local + source keeps the resolution solvable. ``pip freeze`` renders a path install as a + ``@ file://...`` line, which ``freeze_distributions_to_file`` already drops, so a not-ready + provider never lands in the released constraints file. + """ + all_provider_dependencies = json.loads(GENERATED_PROVIDER_DEPENDENCIES_FILE.read_text()) + return [ + f"./providers/{provider.replace('.', '/')}" + for provider in all_provider_dependencies.keys() + if all_provider_dependencies[provider]["state"] == "not-ready" + and ( + python_version is None + or python_version not in all_provider_dependencies[provider]["excluded-python-versions"] + ) + ] + + def build_provider_pre_release_requirements(python_version: str) -> list[str]: """Requirements that let only the providers resolve to a pre-release. @@ -577,6 +601,9 @@ def generate_constraints_pypi_providers(config_params: ConfigParams) -> None: f"apache-airflow-core[all]=={AIRFLOW_CORE_VERSION}", f"apache-airflow-task-sdk=={AIRFLOW_TASK_SDK_VERSION}", "./airflow-ctl", + # not-ready providers are required by [all] but never published to PyPI - build them + # from local source so the resolution is solvable (they are dropped from the file below). + *get_not_ready_provider_source_paths(config_params.python), *additional_constraints_for_highest_resolution, *pre_release_requirements, *pre_release_strategy, diff --git a/scripts/tests/in_container/test_run_generate_constraints.py b/scripts/tests/in_container/test_run_generate_constraints.py index 88af9e84102..b6a2060ec5e 100644 --- a/scripts/tests/in_container/test_run_generate_constraints.py +++ b/scripts/tests/in_container/test_run_generate_constraints.py @@ -148,3 +148,37 @@ class TestBuildProviderPreReleaseRequirements: m.build_provider_pre_release_requirements("3.14") assert seen == ["3.14"] + + +class TestGetNotReadyProviderSourcePaths: + """Only ``not-ready`` providers are built from local source; they are never on PyPI.""" + + @pytest.fixture + def dependencies_file(self, tmp_path, monkeypatch): + path = tmp_path / "provider_dependencies.json" + path.write_text( + json.dumps( + { + "amazon": {"state": "ready", "excluded-python-versions": []}, + "ibm.mq": {"state": "not-ready", "excluded-python-versions": []}, + "common.dataquality": {"state": "not-ready", "excluded-python-versions": []}, + "apache.beam": {"state": "suspended", "excluded-python-versions": []}, + "legacy.thing": {"state": "not-ready", "excluded-python-versions": ["3.10"]}, + } + ) + ) + monkeypatch.setattr(m, "GENERATED_PROVIDER_DEPENDENCIES_FILE", path) + return path + + def test_returns_only_not_ready_providers_as_source_paths(self, dependencies_file): + assert m.get_not_ready_provider_source_paths("3.12") == [ + "./providers/ibm/mq", + "./providers/common/dataquality", + "./providers/legacy/thing", + ] + + def test_excludes_provider_not_supported_on_the_python_version(self, dependencies_file): + assert m.get_not_ready_provider_source_paths("3.10") == [ + "./providers/ibm/mq", + "./providers/common/dataquality", + ]
