This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch exclude-not-ready-providers-from-all in repository https://gitbox.apache.org/repos/asf/airflow.git
commit a0de5f4dc2bbc7bce4b257564af659d0f5a2abd8 Author: Jarek Potiuk <[email protected]> AuthorDate: Wed Aug 5 17:07:01 2026 +0800 Keep not-ready providers out of the all extra A not-ready provider has never been published, so naming it in an extra makes that extra impossible to satisfy - there is no version on PyPI to resolve to. It broke constraints generation for 3.3.1rc1, where apache-airflow[all] could not resolve because it required apache-airflow-providers-ibm-mq. --- pyproject.toml | 20 ---- scripts/ci/prek/common_prek_utils.py | 17 +++- scripts/ci/prek/update_airflow_pyproject_toml.py | 13 ++- uv.lock | 111 +---------------------- 4 files changed, 27 insertions(+), 134 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index be805a23ca0..eaaab506557 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -191,9 +191,6 @@ apache-airflow = "airflow.__main__:main" "common.compat" = [ "apache-airflow-providers-common-compat>=1.2.1" ] -"common.dataquality" = [ - "apache-airflow-providers-common-dataquality>=0.1.0" -] "common.io" = [ "apache-airflow-providers-common-io>=1.4.2" ] @@ -257,9 +254,6 @@ apache-airflow = "airflow.__main__:main" "http" = [ "apache-airflow-providers-http>=4.13.2" ] -"ibm.mq" = [ - "apache-airflow-providers-ibm-mq>=0.1.0; platform_machine !=\"aarch64\" and platform_machine !=\"arm64\"" -] "imap" = [ "apache-airflow-providers-imap>=3.8.0" ] @@ -444,7 +438,6 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-cohere>=1.4.0", "apache-airflow-providers-common-ai>=0.1.0", "apache-airflow-providers-common-compat>=1.2.1", - "apache-airflow-providers-common-dataquality>=0.1.0", "apache-airflow-providers-common-io>=1.4.2", "apache-airflow-providers-common-messaging>=2.0.0", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-common-sql>=1.18.0", @@ -466,7 +459,6 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-grpc>=3.7.0", "apache-airflow-providers-hashicorp>=4.0.0", "apache-airflow-providers-http>=4.13.2", - "apache-airflow-providers-ibm-mq>=0.1.0; platform_machine !=\"aarch64\" and platform_machine !=\"arm64\"", "apache-airflow-providers-imap>=3.8.0", "apache-airflow-providers-influxdb>=2.8.0", "apache-airflow-providers-informatica>=0.1.1", @@ -1154,8 +1146,6 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/common/ai/tests", "$MYPY_CONFIG_FILE_DIR/providers/common/compat/src", "$MYPY_CONFIG_FILE_DIR/providers/common/compat/tests", - "$MYPY_CONFIG_FILE_DIR/providers/common/dataquality/src", - "$MYPY_CONFIG_FILE_DIR/providers/common/dataquality/tests", "$MYPY_CONFIG_FILE_DIR/providers/common/io/src", "$MYPY_CONFIG_FILE_DIR/providers/common/io/tests", "$MYPY_CONFIG_FILE_DIR/providers/common/messaging/src", @@ -1198,8 +1188,6 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/hashicorp/tests", "$MYPY_CONFIG_FILE_DIR/providers/http/src", "$MYPY_CONFIG_FILE_DIR/providers/http/tests", - "$MYPY_CONFIG_FILE_DIR/providers/ibm/mq/src", - "$MYPY_CONFIG_FILE_DIR/providers/ibm/mq/tests", "$MYPY_CONFIG_FILE_DIR/providers/imap/src", "$MYPY_CONFIG_FILE_DIR/providers/imap/tests", "$MYPY_CONFIG_FILE_DIR/providers/influxdb/src", @@ -1468,7 +1456,6 @@ apache-airflow-providers-cncf-kubernetes = false apache-airflow-providers-cohere = false apache-airflow-providers-common-ai = false apache-airflow-providers-common-compat = false -apache-airflow-providers-common-dataquality = false apache-airflow-providers-common-io = false apache-airflow-providers-common-messaging = false apache-airflow-providers-common-sql = false @@ -1490,7 +1477,6 @@ apache-airflow-providers-google = false apache-airflow-providers-grpc = false apache-airflow-providers-hashicorp = false apache-airflow-providers-http = false -apache-airflow-providers-ibm-mq = false apache-airflow-providers-imap = false apache-airflow-providers-influxdb = false apache-airflow-providers-informatica = false @@ -1620,7 +1606,6 @@ apache-airflow-providers-cncf-kubernetes = false apache-airflow-providers-cohere = false apache-airflow-providers-common-ai = false apache-airflow-providers-common-compat = false -apache-airflow-providers-common-dataquality = false apache-airflow-providers-common-io = false apache-airflow-providers-common-messaging = false apache-airflow-providers-common-sql = false @@ -1642,7 +1627,6 @@ apache-airflow-providers-google = false apache-airflow-providers-grpc = false apache-airflow-providers-hashicorp = false apache-airflow-providers-http = false -apache-airflow-providers-ibm-mq = false apache-airflow-providers-imap = false apache-airflow-providers-influxdb = false apache-airflow-providers-informatica = false @@ -1783,7 +1767,6 @@ apache-airflow-providers-cncf-kubernetes = { workspace = true } apache-airflow-providers-cohere = { workspace = true } apache-airflow-providers-common-ai = { workspace = true } apache-airflow-providers-common-compat = { workspace = true } -apache-airflow-providers-common-dataquality = { workspace = true } apache-airflow-providers-common-io = { workspace = true } apache-airflow-providers-common-messaging = { workspace = true } apache-airflow-providers-common-sql = { workspace = true } @@ -1805,7 +1788,6 @@ apache-airflow-providers-google = { workspace = true } apache-airflow-providers-grpc = { workspace = true } apache-airflow-providers-hashicorp = { workspace = true } apache-airflow-providers-http = { workspace = true } -apache-airflow-providers-ibm-mq = { workspace = true } apache-airflow-providers-imap = { workspace = true } apache-airflow-providers-influxdb = { workspace = true } apache-airflow-providers-informatica = { workspace = true } @@ -1924,7 +1906,6 @@ members = [ "providers/cohere", "providers/common/ai", "providers/common/compat", - "providers/common/dataquality", "providers/common/io", "providers/common/messaging", "providers/common/sql", @@ -1946,7 +1927,6 @@ members = [ "providers/grpc", "providers/hashicorp", "providers/http", - "providers/ibm/mq", "providers/imap", "providers/influxdb", "providers/informatica", diff --git a/scripts/ci/prek/common_prek_utils.py b/scripts/ci/prek/common_prek_utils.py index e316eb154ae..559c2f0a047 100644 --- a/scripts/ci/prek/common_prek_utils.py +++ b/scripts/ci/prek/common_prek_utils.py @@ -591,19 +591,30 @@ def get_provider_base_dir_from_path(file_path: Path) -> Path | None: return None -def get_all_provider_ids(exclude_suspended_providers: bool = False) -> list[str]: +def get_all_provider_ids( + exclude_suspended_providers: bool = False, exclude_not_ready_providers: bool = False +) -> list[str]: """ Get all providers from the new provider structure + + :param exclude_suspended_providers: skip providers whose state is ``suspended`` + :param exclude_not_ready_providers: skip providers whose state is ``not-ready`` - those have + never been published, so anything describing what is installable must leave them out """ all_provider_ids = [] + excluded_states = set() + if exclude_suspended_providers: + excluded_states.add("suspended") + if exclude_not_ready_providers: + excluded_states.add("not-ready") for provider_file in AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml"): if provider_file.is_relative_to(AIRFLOW_PROVIDERS_ROOT_PATH / "src"): continue - if exclude_suspended_providers: + if excluded_states: import yaml provider_info = yaml.safe_load(provider_file.read_text()) - if provider_info.get("state") == "suspended": + if provider_info.get("state") in excluded_states: continue provider_id = get_provider_id_from_path(provider_file) if provider_id: diff --git a/scripts/ci/prek/update_airflow_pyproject_toml.py b/scripts/ci/prek/update_airflow_pyproject_toml.py index 2313790e0a5..39a82a96062 100755 --- a/scripts/ci/prek/update_airflow_pyproject_toml.py +++ b/scripts/ci/prek/update_airflow_pyproject_toml.py @@ -269,7 +269,11 @@ if __name__ == "__main__": all_optional_dependencies.append(f'"{optional}" = [\n "apache-airflow-core[{optional}]"\n]\n') optional_airflow_task_sdk_dependencies = get_optional_dependencies(AIRFLOW_TASK_SDK_PYPROJECT_TOML_FILE) all_optional_dependencies.append('"all-task-sdk" = [\n "apache-airflow-task-sdk[all]"\n]\n') - all_providers = sorted(get_all_provider_ids(exclude_suspended_providers=True)) + # A not-ready provider has never been released, so naming it in an extra makes that extra + # unsatisfiable - there is no version of it on PyPI to resolve to. + all_providers = sorted( + get_all_provider_ids(exclude_suspended_providers=True, exclude_not_ready_providers=True) + ) all_provider_lines = [] for provider_id in all_providers: distribution_name = provider_distribution_name(provider_id) @@ -288,10 +292,15 @@ if __name__ == "__main__": all_provider_lines.append(f' "{distribution_name}",\n') all_optional_dependencies.append('"all" = [\n') optional_apache_airflow_dependencies = get_optional_dependencies(AIRFLOW_PYPROJECT_TOML_FILE) + # Filtered against every provider id rather than against `all_providers`: a provider left out of + # `all_providers` still has an extra named after it, and testing only against the included ones + # would sweep that extra in here instead of dropping it - which is how a not-ready provider would + # come back into `all` through the side door. + every_provider_id = set(get_all_provider_ids()) all_local_extras = [ extra for extra in sorted(optional_apache_airflow_dependencies) - if extra not in all_providers and not extra.startswith("all") + if extra not in every_provider_id and not extra.startswith("all") ] all_optional_dependencies.append(f' "apache-airflow[{",".join(all_local_extras)}]",\n') all_optional_dependencies.append(' "apache-airflow-core[all]",\n') diff --git a/uv.lock b/uv.lock index 3b52d8d26c2..532992ded2b 100644 --- a/uv.lock +++ b/uv.lock @@ -33,7 +33,6 @@ apache-airflow-providers-openlineage = false apache-airflow-providers-sftp = false apache-airflow-e2e-tests = false apache-airflow-shared-logging = false -apache-airflow-providers-common-dataquality = false apache-airflow-providers-apache-drill = false apache-airflow-providers-pgvector = false apache-airflow-providers-imap = false @@ -64,9 +63,9 @@ apache-airflow-providers-apache-cassandra = false apache-airflow-providers-asana = false apache-airflow-providers-oracle = false apache-airflow-providers-mysql = false -apache-airflow-providers-teradata = false apache-airflow-providers-alibaba = false apache-airflow-providers-microsoft-mssql = false +apache-airflow-providers-teradata = false apache-airflow-providers-jdbc = false apache-airflow-helm-chart = false apache-airflow-providers-anthropic = false @@ -96,7 +95,6 @@ apache-airflow-providers-databricks = false apache-airflow-shared-state = false apache-airflow-docker-stack = false apache-airflow-providers-sqlite = false -apache-airflow-providers-ibm-mq = false apache-airflow-shared-module-loading = false apache-airflow-providers-yandex = false apache-airflow-shared-serialization = false @@ -205,7 +203,6 @@ members = [ "apache-airflow-providers-cohere", "apache-airflow-providers-common-ai", "apache-airflow-providers-common-compat", - "apache-airflow-providers-common-dataquality", "apache-airflow-providers-common-io", "apache-airflow-providers-common-messaging", "apache-airflow-providers-common-sql", @@ -227,7 +224,6 @@ members = [ "apache-airflow-providers-grpc", "apache-airflow-providers-hashicorp", "apache-airflow-providers-http", - "apache-airflow-providers-ibm-mq", "apache-airflow-providers-imap", "apache-airflow-providers-influxdb", "apache-airflow-providers-informatica", @@ -1032,7 +1028,6 @@ all = [ { name = "apache-airflow-providers-cohere" }, { name = "apache-airflow-providers-common-ai" }, { name = "apache-airflow-providers-common-compat" }, - { name = "apache-airflow-providers-common-dataquality" }, { name = "apache-airflow-providers-common-io" }, { name = "apache-airflow-providers-common-messaging" }, { name = "apache-airflow-providers-common-sql", extra = ["pandas", "polars"] }, @@ -1054,7 +1049,6 @@ all = [ { name = "apache-airflow-providers-grpc" }, { name = "apache-airflow-providers-hashicorp" }, { name = "apache-airflow-providers-http" }, - { name = "apache-airflow-providers-ibm-mq", marker = "platform_machine != 'aarch64' and platform_machine != 'arm64'" }, { name = "apache-airflow-providers-imap" }, { name = "apache-airflow-providers-influxdb" }, { name = "apache-airflow-providers-informatica" }, @@ -1217,9 +1211,6 @@ common-ai = [ common-compat = [ { name = "apache-airflow-providers-common-compat" }, ] -common-dataquality = [ - { name = "apache-airflow-providers-common-dataquality" }, -] common-io = [ { name = "apache-airflow-providers-common-io" }, ] @@ -1297,9 +1288,6 @@ hashicorp = [ http = [ { name = "apache-airflow-providers-http" }, ] -ibm-mq = [ - { name = "apache-airflow-providers-ibm-mq", marker = "platform_machine != 'aarch64' and platform_machine != 'arm64'" }, -] imap = [ { name = "apache-airflow-providers-imap" }, ] @@ -1632,8 +1620,6 @@ requires-dist = [ { name = "apache-airflow-providers-common-ai", marker = "extra == 'common-ai'", editable = "providers/common/ai" }, { name = "apache-airflow-providers-common-compat", marker = "extra == 'all'", editable = "providers/common/compat" }, { name = "apache-airflow-providers-common-compat", marker = "extra == 'common-compat'", editable = "providers/common/compat" }, - { name = "apache-airflow-providers-common-dataquality", marker = "extra == 'all'", editable = "providers/common/dataquality" }, - { name = "apache-airflow-providers-common-dataquality", marker = "extra == 'common-dataquality'", editable = "providers/common/dataquality" }, { name = "apache-airflow-providers-common-io", marker = "extra == 'all'", editable = "providers/common/io" }, { name = "apache-airflow-providers-common-io", marker = "extra == 'common-io'", editable = "providers/common/io" }, { name = "apache-airflow-providers-common-messaging", marker = "extra == 'all'", editable = "providers/common/messaging" }, @@ -1680,8 +1666,6 @@ requires-dist = [ { name = "apache-airflow-providers-hashicorp", marker = "extra == 'hashicorp'", editable = "providers/hashicorp" }, { name = "apache-airflow-providers-http", marker = "extra == 'all'", editable = "providers/http" }, { name = "apache-airflow-providers-http", marker = "extra == 'http'", editable = "providers/http" }, - { name = "apache-airflow-providers-ibm-mq", marker = "platform_machine != 'aarch64' and platform_machine != 'arm64' and extra == 'all'", editable = "providers/ibm/mq" }, - { name = "apache-airflow-providers-ibm-mq", marker = "platform_machine != 'aarch64' and platform_machine != 'arm64' and extra == 'ibm-mq'", editable = "providers/ibm/mq" }, { name = "apache-airflow-providers-imap", marker = "extra == 'all'", editable = "providers/imap" }, { name = "apache-airflow-providers-imap", marker = "extra == 'imap'", editable = "providers/imap" }, { name = "apache-airflow-providers-influxdb", marker = "extra == 'all'", editable = "providers/influxdb" }, @@ -1792,7 +1776,7 @@ requires-dist = [ { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.30.0" }, { name = "uv", marker = "extra == 'uv'", specifier = ">=0.11.29" }, ] -provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhou [...] +provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhou [...] [package.metadata.requires-dev] ci-image = [ @@ -4574,35 +4558,6 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] -[[package]] -name = "apache-airflow-providers-common-dataquality" -version = "0.1.0" -source = { editable = "providers/common/dataquality" } -dependencies = [ - { name = "apache-airflow" }, -] - -[package.dev-dependencies] -dev = [ - { name = "apache-airflow" }, - { name = "apache-airflow-devel-common" }, - { name = "apache-airflow-task-sdk" }, -] -docs = [ - { name = "apache-airflow-devel-common", extra = ["docs"] }, -] - -[package.metadata] -requires-dist = [{ name = "apache-airflow", editable = "." }] - -[package.metadata.requires-dev] -dev = [ - { name = "apache-airflow", editable = "." }, - { name = "apache-airflow-devel-common", editable = "devel-common" }, - { name = "apache-airflow-task-sdk", editable = "task-sdk" }, -] -docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] - [[package]] name = "apache-airflow-providers-common-io" version = "1.8.0" @@ -5947,57 +5902,6 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] -[[package]] -name = "apache-airflow-providers-ibm-mq" -version = "0.1.0" -source = { editable = "providers/ibm/mq" } -dependencies = [ - { name = "apache-airflow" }, - { name = "apache-airflow-providers-common-compat" }, - { name = "asgiref" }, -] - -[package.optional-dependencies] -common-messaging = [ - { name = "apache-airflow-providers-common-messaging" }, -] -ibmmq = [ - { name = "ibmmq", marker = "platform_machine != 'aarch64' and platform_machine != 'arm64'" }, -] - -[package.dev-dependencies] -dev = [ - { name = "apache-airflow" }, - { name = "apache-airflow-devel-common" }, - { name = "apache-airflow-providers-common-compat" }, - { name = "apache-airflow-providers-common-messaging" }, - { name = "apache-airflow-task-sdk" }, -] -docs = [ - { name = "apache-airflow-devel-common", extra = ["docs"] }, -] - -[package.metadata] -requires-dist = [ - { name = "apache-airflow", editable = "." }, - { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, - { name = "apache-airflow-providers-common-messaging", marker = "extra == 'common-messaging'", editable = "providers/common/messaging" }, - { name = "asgiref", marker = "python_full_version < '3.14'", specifier = ">=2.3.0" }, - { name = "asgiref", marker = "python_full_version >= '3.14'", specifier = ">=3.11.1" }, - { name = "ibmmq", marker = "platform_machine != 'aarch64' and platform_machine != 'arm64' and extra == 'ibmmq'", specifier = ">=2.0.6" }, -] -provides-extras = ["ibmmq", "common-messaging"] - -[package.metadata.requires-dev] -dev = [ - { name = "apache-airflow", editable = "." }, - { name = "apache-airflow-devel-common", editable = "devel-common" }, - { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, - { name = "apache-airflow-providers-common-messaging", editable = "providers/common/messaging" }, - { name = "apache-airflow-task-sdk", editable = "task-sdk" }, -] -docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] - [[package]] name = "apache-airflow-providers-imap" version = "3.12.1" @@ -14336,17 +14240,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/50/c94fc1cbe59c33b561e0d8e327172a4f451ff9f13b497c04149c6de0f76a/ibmcloudant-0.11.9-py3-none-any.whl";, hash = "sha256:1738e0dcf888a88fbe0c8ddd3640aa0fa2c17e19cd916415aada7af0a889d8b5", size = 120805, upload-time = "2026-07-07T09:37:02.045Z" }, ] -[[package]] -name = "ibmmq" -version = "2.1.0" -source = { registry = "https://pypi.org/simple"; } -sdist = { url = "https://files.pythonhosted.org/packages/6a/6b/e8e38a3e1b828c029e4b0aa39ecdd940319ffe725273a0f8be6efae88a7c/ibmmq-2.1.0.tar.gz";, hash = "sha256:63ac906290700b55e88cc7c32232579e4419aba209830b6009d6d89cbb089555", size = 436005, upload-time = "2026-06-17T11:48:41.238Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/e6/d17301d1171dea40b6ed642557174c0de78fea46568756106d143e98cc88/ibmmq-2.1.0-cp39-abi3-macosx_11_0_arm64.whl";, hash = "sha256:9dbb2b509ec1e86a8111480e35c5b346fa21bae7266c35634d76dd377da80fdb", size = 395482, upload-time = "2026-06-17T11:48:36.803Z" }, - { url = "https://files.pythonhosted.org/packages/d1/2e/f8b4f8088ca697d9d5978fba6e5d9a1adf83421346c51800b3700b179f52/ibmmq-2.1.0-cp39-abi3-manylinux_2_28_x86_64.whl";, hash = "sha256:81c65da995be055d4fe02eb65ab05b8c5465324d3c320d5def63bbea32c08087", size = 435602, upload-time = "2026-07-06T08:02:24.283Z" }, - { url = "https://files.pythonhosted.org/packages/20/09/e833b4e18115c94df8b3e6c293755a1cdb5bbd30b935ecc71b89e0206874/ibmmq-2.1.0-cp39-abi3-win_amd64.whl";, hash = "sha256:5d68c837f216721b2d6d7495cf91c717670a5cec383b28aa584616a9c48c1c30", size = 400532, upload-time = "2026-06-17T11:48:38.923Z" }, -] - [[package]] name = "icalendar" version = "7.2.0"
