This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new caf26246c1c Stop pydantic-ai providers that models cannot use from
failing the common.ai vendor check (#76312)
caf26246c1c is described below
commit caf26246c1c695e9dfecc71ae2c99c7801a3106f
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Nov 4 15:26:03 2026 +0100
Stop pydantic-ai providers that models cannot use from failing the
common.ai vendor check (#76312)
Each pydantic-ai release that adds a provider module infer_model cannot
route to, such as the generic OpenAI-compatible provider in 2.63.0, failed the
provider.yaml drift tests until someone listed it by hand, which blocked the
automated CI environment upgrade. Whether infer_model routes to a module can be
asked of the installed pydantic-ai itself, so only a new vendor a connection
can actually reach still needs a human.
Generated-by: Claude Opus 5
---
.../tests/unit/common/ai/test_provider_metadata.py | 101 ++++++++++++++-------
1 file changed, 68 insertions(+), 33 deletions(-)
diff --git a/providers/common/ai/tests/unit/common/ai/test_provider_metadata.py
b/providers/common/ai/tests/unit/common/ai/test_provider_metadata.py
index 16ae1309350..f630485f8c4 100644
--- a/providers/common/ai/tests/unit/common/ai/test_provider_metadata.py
+++ b/providers/common/ai/tests/unit/common/ai/test_provider_metadata.py
@@ -18,7 +18,7 @@
Drift tripwires between ``provider.yaml``'s declared ``external-services`` and
what
pydantic-ai actually installs.
-How ``LABELS``, ``NON_MODULE_SERVICES``, ``UNREACHABLE_MODULES``,
+How ``LABELS``, ``NON_MODULE_SERVICES``,
``DEPRECATED_UPSTREAM_MODULES`` and ``HOOK_INCOMPATIBLE_AUTH_MODULES`` below
were derived,
including the empirical checks that back each exclusion, is documented in the
comment directly
above each constant.
@@ -32,6 +32,7 @@ import pkgutil
import re
from importlib.metadata import version
from pathlib import Path
+from typing import Any, cast
import pydantic_ai.providers as pydantic_ai_providers
import pytest
@@ -105,19 +106,10 @@ LABELS = {
# module would go.
NON_MODULE_SERVICES: set[str] = set()
-# Modules `pkgutil` discovers under `pydantic_ai.providers` that `infer_model`
never
-# routes to -- `infer_model("voyageai:...")` /
`infer_model("sentence-transformers:...")`
-# raise `UserError: Unknown model`, confirmed by
`test_unreachable_modules_stay_unreachable`
-# below. Not listed in `LABELS`/`provider.yaml` because there is no
`common.ai` code path
-# (chat or embedding) that reaches them. `elevenlabs` (added in
pydantic-ai-slim 2.59.0) is
-# the same: its provider only backs ElevenLabs speech/realtime models, which
`infer_model`
-# does not route to (`UserError: Unknown model: elevenlabs:...`, checked on
2.59.0).
-# `openai_compatible` (added in pydantic-ai-slim 2.63.0) is a generic provider
for any
-# OpenAI-compatible endpoint that must be constructed with a service `name`;
`infer_model` has no
-# prefix for it (`UserError: Unknown model: openai-compatible:...`, checked on
2.63.0).
-UNREACHABLE_MODULES = {"voyageai", "sentence_transformers", "elevenlabs",
"openai_compatible"}
-
-# Modules that ARE reachable through `infer_model` (unlike
UNREACHABLE_MODULES) but are
+# `gateway` is reached as `gateway/<vendor>:<model>`; `gateway:<model>` is an
unknown model.
+INFER_MODEL_PROBES = {"gateway": "gateway/openai:placeholder"}
+
+# Modules that ARE reachable through `infer_model` but are
# excluded by decision because pydantic-ai itself marks the upstream backend
dead:
# `pydantic_ai.providers.github.GitHubProvider` is decorated `@deprecated`
with the message
# "GitHub Models was retired on 2026-07-30 ... this provider is deprecated and
will be
@@ -130,7 +122,7 @@ UNREACHABLE_MODULES = {"voyageai", "sentence_transformers",
"elevenlabs", "opena
# of silently staying excluded.
DEPRECATED_UPSTREAM_MODULES = {"github"}
-# Modules that ARE reachable through `infer_model` (unlike
UNREACHABLE_MODULES) but are excluded
+# Modules that ARE reachable through `infer_model` but are excluded
# by decision because `PydanticAIHook` cannot supply the credentials they
require from a connection.
# `PydanticAIHook._get_provider_kwargs()` -- the generic connection type's
only credential path
# (`hooks/pydantic_ai.py`) -- always sends `api_key` (from `conn.password`)
and/or `base_url`
@@ -239,14 +231,63 @@ def _find_discovered_provider_modules() -> set[str]:
}
+def _find_provider_names(module: str) -> set[str]:
+ """
+ The ``name`` of each ``*Provider`` class in ``module``, which
``infer_model`` matches as the
+ model prefix, read without building a provider; empty when the module
needs an optional SDK.
+ """
+ try:
+ mod = importlib.import_module(f"pydantic_ai.providers.{module}")
+ except ImportError:
+ return set()
+ names = set()
+ for cls in _find_provider_classes(mod):
+ name = inspect.getattr_static(cls, "name", None)
+ try:
+ value = name.fget(None) if isinstance(name, property) and
name.fget else name
+ except Exception:
+ continue
+ if isinstance(value, str):
+ names.add(value)
+ return names
+
+
+def _routes_to_a_model(model: str) -> bool:
+ try:
+ infer_model(model, provider_factory=lambda _p:
cast("pydantic_ai_providers.Provider[Any]", object()))
+ except UserError as exc:
+ return not str(exc).startswith("Unknown model")
+ except Exception:
+ # A missing optional SDK or the placeholder provider fails only after
routing.
+ return True
+ return True
+
+
+def _is_routed_by_infer_model(module: str) -> bool:
+ """
+ Whether ``infer_model`` routes a model string to ``module``.
+
+ Modules it never routes to (embedding-only ones such as ``voyageai``,
speech-only ones such
+ as ``elevenlabs``, or ``openai_compatible`` from pydantic-ai-slim 2.63.0,
which needs a
+ service name to be built) have no ``common.ai`` code path, so they are
neither labelled nor
+ declared. The prefix is tried both as the module name and as each
provider's own ``name``, so
+ a vendor whose prefix differs from its module name is not mistaken for an
unreachable one.
+ """
+ if module in INFER_MODEL_PROBES:
+ return _routes_to_a_model(INFER_MODEL_PROBES[module])
+ prefixes = {module.replace("_", "-")} | _find_provider_names(module)
+ return any(_routes_to_a_model(f"{prefix}:placeholder") for prefix in
prefixes)
+
+
def _find_reachable_provider_modules() -> set[str]:
"""The discovered modules `common.ai` actually reaches, i.e. minus the
documented exclusions."""
- return (
- _find_discovered_provider_modules()
- - UNREACHABLE_MODULES
+ return {
+ module
+ for module in _find_discovered_provider_modules()
- DEPRECATED_UPSTREAM_MODULES
- HOOK_INCOMPATIBLE_AUTH_MODULES
- )
+ if _is_routed_by_infer_model(module)
+ }
def _find_derived_vendor_labels() -> set[str]:
@@ -256,13 +297,18 @@ def _find_derived_vendor_labels() -> set[str]:
} | NON_MODULE_SERVICES
-def _provider_classes_in_module(mod) -> list[type]:
- """The ``*Provider`` classes defined in ``mod`` itself; fails if there are
none."""
- provider_classes = [
+def _find_provider_classes(mod) -> list[type]:
+ """The ``*Provider`` classes defined in ``mod`` itself."""
+ return [
obj
for obj in vars(mod).values()
if isinstance(obj, type) and obj.__module__ == mod.__name__ and
obj.__name__.endswith("Provider")
]
+
+
+def _provider_classes_in_module(mod) -> list[type]:
+ """The ``*Provider`` classes defined in ``mod`` itself; fails if there are
none."""
+ provider_classes = _find_provider_classes(mod)
assert provider_classes, f"no Provider class found in {mod.__name__}"
return provider_classes
@@ -332,17 +378,6 @@ class TestPydanticAIExternalServicesDrift:
stale = set(LABELS) - _find_reachable_provider_modules()
assert not stale, f"label(s) for module(s) pydantic-ai no longer has:
{sorted(stale)}"
- def test_unreachable_modules_stay_unreachable(self):
- """
- Guard the other direction: if a future pydantic-ai release makes one of
- ``UNREACHABLE_MODULES`` reachable through ``infer_model``, this must
fail so a human
- adds it to ``LABELS`` and ``provider.yaml`` instead of it silently
staying missing.
- """
- for module in UNREACHABLE_MODULES:
- prefix = module.replace("_", "-")
- with pytest.raises(UserError, match="Unknown model"):
- infer_model(f"{prefix}:placeholder", provider_factory=lambda
_p: object())
-
def test_deprecated_upstream_modules_stay_deprecated(self):
"""
Guard the other direction, for every member of
``DEPRECATED_UPSTREAM_MODULES`` (not just
