This is an automated email from the ASF dual-hosted git repository.
Lee-W pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 43384c7d8bc Show which LLM providers each Common AI connection type
reaches (#70497)
43384c7d8bc is described below
commit 43384c7d8bcf1f96b65a88067e002ffb70e6461e
Author: Wei Lee <[email protected]>
AuthorDate: Thu Aug 6 18:01:53 2026 +0800
Show which LLM providers each Common AI connection type reaches (#70497)
---
airflow-core/src/airflow/provider.yaml.schema.json | 8 ++++
.../23_provider_hook_migration_to_yaml.rst | 26 ++++++++++
dev/registry/extract_metadata.py | 6 ++-
dev/registry/extract_versions.py | 1 +
dev/registry/registry_contract_models.py | 1 +
dev/registry/tests/test_extract_metadata.py | 56 ++++++++++++++++++++++
dev/registry/tests/test_extract_versions.py | 40 ++++++++++++++++
.../tests/test_registry_contract_models.py | 19 ++++++++
providers/common/ai/provider.yaml | 28 +++++++++++
.../providers/common/ai/get_provider_info.py | 24 ++++++++++
registry/src/css/main.css | 34 +++++++++++++
registry/src/provider-version.njk | 24 ++++++++++
12 files changed, 266 insertions(+), 1 deletion(-)
diff --git a/airflow-core/src/airflow/provider.yaml.schema.json
b/airflow-core/src/airflow/provider.yaml.schema.json
index 0a700aa88d5..376eacdf93f 100644
--- a/airflow-core/src/airflow/provider.yaml.schema.json
+++ b/airflow-core/src/airflow/provider.yaml.schema.json
@@ -471,6 +471,14 @@
},
"additionalProperties": false
},
+ "external-services": {
+ "description": "Representative, non-exhaustive list of
upstream provider/service names this connection type is known to reach over the
network (e.g. 'Anthropic', 'AWS Bedrock', 'Ollama') -- examples, not an
exhaustive compatibility matrix. Hooks that resolve the destination from a
caller-supplied model identifier (as with pydantic-ai) can reach any service
the model id names, so the list can never be complete. Not to be confused with
the top-level 'integrations' k [...]
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "minItems": 1
+ },
"conn-fields": {
"description": "Custom connection fields stored in
Connection.extra JSON",
"type": "object",
diff --git a/contributing-docs/23_provider_hook_migration_to_yaml.rst
b/contributing-docs/23_provider_hook_migration_to_yaml.rst
index f751f9675a9..54706f6c4dc 100644
--- a/contributing-docs/23_provider_hook_migration_to_yaml.rst
+++ b/contributing-docs/23_provider_hook_migration_to_yaml.rst
@@ -60,6 +60,32 @@ Customizations for standard connection fields:
placeholders:
port: '5432'
+external-services
+~~~~~~~~~~~~~~~~~
+
+List of upstream provider or model-serving services that this connection type
+reaches over the network (e.g. ``OpenAI``, ``AWS Bedrock``, ``Ollama``). The
registry
+renders this list as a table on the provider's version page, so someone
browsing the
+registry can see at a glance which external services a given connection type
talks
+to. Because some hooks resolve the destination from a caller-supplied model
+identifier (for example ``PydanticAIHook``), this list is representative, not
+exhaustive -- treat it as example services, not a compatibility matrix.
+
+This is unrelated to the top-level ``integrations`` key in ``provider.yaml``,
which
+describes *framework* integrations (e.g. Kubernetes, Docker) and drives the
+provider's docs pages, logo, and tags. ``external-services`` only applies
inside a
+``connection-types`` entry and only lists services reached over the network —
it has
+no effect on docs generation, logos, or tags.
+
+.. code-block:: yaml
+
+ connection-types:
+ - hook-class-name:
airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIAzureHook
+ hook-name: "Pydantic AI (Azure OpenAI)"
+ connection-type: pydanticai-azure
+ external-services:
+ - Azure OpenAI
+
conn-fields
~~~~~~~~~~~
diff --git a/dev/registry/extract_metadata.py b/dev/registry/extract_metadata.py
index 1c557a3a8fa..77d34d8397d 100644
--- a/dev/registry/extract_metadata.py
+++ b/dev/registry/extract_metadata.py
@@ -373,7 +373,9 @@ class Provider:
}
)
categories: list[dict] = field(default_factory=list)
- connection_types: list[dict] = field(default_factory=list) # {conn_type,
hook_class, docs_url}
+ connection_types: list[dict] = field(
+ default_factory=list
+ ) # {conn_type, hook_class, docs_url, external_services}
requires_python: str = "" # e.g., ">=3.10"
dependencies: list[str] = field(default_factory=list) # from
pyproject.toml
optional_extras: dict[str, list[str]] = field(default_factory=dict) #
{extra_name: [deps]}
@@ -782,12 +784,14 @@ def main():
for conn in provider_yaml.get("connection-types", []):
conn_type = conn.get("connection-type", "")
hook_class = conn.get("hook-class-name", "")
+ external_services = conn.get("external-services", [])
if conn_type:
connection_types.append(
{
"conn_type": conn_type,
"hook_class": hook_class,
"docs_url": resolve_connection_docs_url(conn_type,
conn_url_map, base_docs_url),
+ "external_services": external_services,
}
)
diff --git a/dev/registry/extract_versions.py b/dev/registry/extract_versions.py
index c47d91a7043..064fd4a8f86 100644
--- a/dev/registry/extract_versions.py
+++ b/dev/registry/extract_versions.py
@@ -449,6 +449,7 @@ def extract_version_data(
"conn_type": conn_type,
"hook_class": ct.get("hook-class-name", ""),
"docs_url": resolve_connection_docs_url(conn_type,
conn_url_map, base_docs_url),
+ "external_services": ct.get("external-services", []),
}
)
diff --git a/dev/registry/registry_contract_models.py
b/dev/registry/registry_contract_models.py
index 7295cfb4036..475b14d5d0f 100644
--- a/dev/registry/registry_contract_models.py
+++ b/dev/registry/registry_contract_models.py
@@ -52,6 +52,7 @@ class ConnectionTypeContract(BaseModel):
conn_type: str
hook_class: str = ""
docs_url: str | None = None
+ external_services: list[str] = Field(default_factory=list)
class ProviderContract(BaseModel):
diff --git a/dev/registry/tests/test_extract_metadata.py
b/dev/registry/tests/test_extract_metadata.py
index e4e98fcdc69..faf17f6f9c8 100644
--- a/dev/registry/tests/test_extract_metadata.py
+++ b/dev/registry/tests/test_extract_metadata.py
@@ -20,6 +20,7 @@ from __future__ import annotations
import http.client
import json
+import sys
import textwrap
from pathlib import Path
from unittest.mock import MagicMock, patch
@@ -36,6 +37,7 @@ from extract_metadata import (
find_latest_released_version,
find_related_providers,
load_release_tags,
+ main,
module_path_to_file_path,
parse_pyproject_toml,
preserve_nonzero_downloads,
@@ -828,3 +830,57 @@ class TestVersionsListFiltering:
filtered = [v for v in raw_versions if f"providers-{provider_id}/{v}"
in release_tags]
# Order from raw_versions is preserved; only the phantom is dropped
assert filtered == ["9.26.0", "9.25.0", "9.24.0"]
+
+
+# ---------------------------------------------------------------------------
+# main() -- connection-types `external-services` propagation
+# ---------------------------------------------------------------------------
+class TestMainConnectionTypesExternalServices:
+ """The connection-types extraction loop lives inline in main() rather than
+ a standalone function, so this drives main() end-to-end (with network and
+ filesystem dependencies mocked/redirected) to prove `external-services`
+ from provider.yaml reaches the written providers.json, and round-trips
+ through the `ConnectionTypeContract` (extra="forbid") validation main()
+ already runs -- catching a key-name drift between provider.yaml, this
+ script, and registry_contract_models.py.
+ """
+
+ @patch("extract_metadata.fetch_provider_inventory", autospec=True,
return_value=None)
+ @patch("extract_metadata.fetch_pypi_data_parallel", autospec=True,
return_value={})
+ @patch("extract_metadata.load_release_tags", autospec=True,
return_value=set())
+ def test_external_services_propagates_to_providers_json(
+ self, _load_release_tags, _fetch_pypi_data_parallel,
_fetch_provider_inventory, tmp_path
+ ):
+ providers_dir = tmp_path / "providers"
+ provider_dir = providers_dir / "testprov"
+ provider_dir.mkdir(parents=True)
+ (provider_dir / "provider.yaml").write_text(
+ textwrap.dedent("""\
+ name: Test Provider
+ description: A test provider.
+ versions:
+ - 1.0.0
+ connection-types:
+ - connection-type: testconn
+ hook-class-name: airflow.providers.test.hooks.TestHook
+ external-services:
+ - openai
+ - anthropic
+ """)
+ )
+ output_dir = tmp_path / "output"
+ script_dir = tmp_path / "script"
+ output_dir.mkdir()
+ script_dir.mkdir()
+
+ with (
+ patch("extract_metadata.PROVIDERS_DIR", providers_dir),
+ patch("extract_metadata.OUTPUT_DIR", output_dir),
+ patch("extract_metadata.SCRIPT_DIR", script_dir),
+ patch.object(sys, "argv", ["extract_metadata.py"]),
+ ):
+ main()
+
+ written = json.loads((output_dir / "providers.json").read_text())
+ provider = next(p for p in written["providers"] if p["id"] ==
"testprov")
+ assert provider["connection_types"][0]["external_services"] ==
["openai", "anthropic"]
diff --git a/dev/registry/tests/test_extract_versions.py
b/dev/registry/tests/test_extract_versions.py
index a2c8ce33575..061424eb918 100644
--- a/dev/registry/tests/test_extract_versions.py
+++ b/dev/registry/tests/test_extract_versions.py
@@ -18,12 +18,16 @@
from __future__ import annotations
+import textwrap
+from unittest.mock import patch
+
import pytest
from extract_versions import (
AIRFLOW_ROOT,
PROVIDERS_JSON_CANDIDATES,
SCRIPT_DIR,
extract_modules_from_yaml,
+ extract_version_data,
)
from registry_tools.types import CLASS_LEVEL_SECTIONS,
DICT_SHAPED_CLASS_LEVEL_SECTIONS
@@ -170,3 +174,39 @@ class TestExtractModulesFromYamlDictShapedSections:
modules = _extract_class_level_modules(provider_yaml)
assert {m["type"] for m in modules} == {t for t, _, _ in
DICT_SHAPED_CLASS_LEVEL_SECTIONS.values()}
+
+
+class TestExtractVersionDataConnectionTypes:
+ """`external-services` on a connection-types entry must survive into the
+ per-version metadata.json (extract_versions.py:399) the same way it does
+ for the latest release in providers.json (extract_metadata.py). A
+ superseded release only has this file as its data source, so a dropped
+ or mis-keyed field here silently vanishes from just that version's page.
+ """
+
+ PROVIDER_YAML = textwrap.dedent("""\
+ name: Test Provider
+ connection-types:
+ - connection-type: testconn
+ hook-class-name: airflow.providers.test.hooks.TestHook
+ external-services:
+ - openai
+ - anthropic
+ """)
+
+ @patch("extract_versions.extract_modules_from_yaml", autospec=True,
return_value=[])
+ @patch("extract_versions.fetch_provider_inventory", autospec=True,
return_value=None)
+ @patch("extract_versions.git_show", autospec=True)
+ @patch("extract_versions.detect_layout", autospec=True, return_value="new")
+ @patch("extract_versions.git_tag_exists", autospec=True, return_value=True)
+ def test_external_services_propagates_to_version_metadata(
+ self, _tag_exists, _layout, mock_git_show, _inventory, _modules
+ ):
+ mock_git_show.side_effect = lambda tag, path: (
+ self.PROVIDER_YAML if path.endswith("provider.yaml") else None
+ )
+
+ result = extract_version_data("test", "1.0.0", "test")
+
+ assert result is not None
+ assert result["connection_types"][0]["external_services"] ==
["openai", "anthropic"]
diff --git a/dev/registry/tests/test_registry_contract_models.py
b/dev/registry/tests/test_registry_contract_models.py
index 120c682da78..2a98d848261 100644
--- a/dev/registry/tests/test_registry_contract_models.py
+++ b/dev/registry/tests/test_registry_contract_models.py
@@ -21,6 +21,7 @@ from __future__ import annotations
import pytest
from pydantic import ValidationError
from registry_contract_models import (
+ ConnectionTypeContract,
build_openapi_document,
validate_modules_catalog,
validate_provider_parameters,
@@ -100,6 +101,24 @@ def
test_module_contract_preserves_supports_durable_execution_true():
assert validated["modules"][0]["supports_durable_execution"] is True
+def test_connection_type_contract_defaults_external_services_to_empty_list():
+ """Legacy connection-types entries (provider.yaml without
`external-services`)
+ must still validate, with the field defaulting to an empty list."""
+ validated = ConnectionTypeContract.model_validate({"conn_type": "test",
"hook_class": "x.y.Hook"})
+ assert validated.external_services == []
+
+
+def test_connection_type_contract_round_trips_external_services():
+ validated = ConnectionTypeContract.model_validate(
+ {
+ "conn_type": "test",
+ "hook_class": "x.y.Hook",
+ "external_services": ["openai", "anthropic"],
+ }
+ )
+ assert validated.external_services == ["openai", "anthropic"]
+
+
def
test_validate_version_metadata_accepts_legacy_version_modules_without_ids():
payload = {
"provider_id": "test",
diff --git a/providers/common/ai/provider.yaml
b/providers/common/ai/provider.yaml
index 1feaa6ed5c6..b6133e2e2e4 100644
--- a/providers/common/ai/provider.yaml
+++ b/providers/common/ai/provider.yaml
@@ -147,6 +147,16 @@ connection-types:
- hook-class-name:
airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIHook
hook-name: "Pydantic AI"
connection-type: pydanticai
+ external-services:
+ - OpenAI
+ - Anthropic
+ - Google
+ - AWS Bedrock
+ - Groq
+ - Mistral AI
+ - DeepSeek
+ - Ollama
+ - vLLM
ui-field-behaviour:
hidden-fields:
- schema
@@ -167,6 +177,8 @@ connection-types:
- hook-class-name:
airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIAzureHook
hook-name: "Pydantic AI (Azure OpenAI)"
connection-type: pydanticai-azure
+ external-services:
+ - Azure OpenAI
ui-field-behaviour:
hidden-fields:
- schema
@@ -195,6 +207,8 @@ connection-types:
- hook-class-name:
airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIBedrockHook
hook-name: "Pydantic AI (AWS Bedrock)"
connection-type: pydanticai-bedrock
+ external-services:
+ - AWS Bedrock
ui-field-behaviour:
hidden-fields:
- schema
@@ -278,6 +292,8 @@ connection-types:
- hook-class-name:
airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIVertexHook
hook-name: "Pydantic AI (Google Vertex AI)"
connection-type: pydanticai-vertex
+ external-services:
+ - Google Vertex AI
ui-field-behaviour:
hidden-fields:
- schema
@@ -374,6 +390,14 @@ connection-types:
- hook-class-name: airflow.providers.common.ai.hooks.langchain.LangChainHook
hook-name: "LangChain"
connection-type: langchain
+ external-services:
+ - OpenAI
+ - Anthropic
+ - Groq
+ - Mistral AI
+ - DeepSeek
+ - Ollama
+ - vLLM
ui-field-behaviour:
hidden-fields:
- schema
@@ -407,6 +431,10 @@ connection-types:
- hook-class-name:
airflow.providers.common.ai.hooks.llamaindex.LlamaIndexHook
hook-name: "LlamaIndex"
connection-type: llamaindex
+ external-services:
+ - OpenAI
+ - Ollama
+ - vLLM
ui-field-behaviour:
hidden-fields:
- schema
diff --git
a/providers/common/ai/src/airflow/providers/common/ai/get_provider_info.py
b/providers/common/ai/src/airflow/providers/common/ai/get_provider_info.py
index 89d96737d94..96e87e5945d 100644
--- a/providers/common/ai/src/airflow/providers/common/ai/get_provider_info.py
+++ b/providers/common/ai/src/airflow/providers/common/ai/get_provider_info.py
@@ -120,6 +120,17 @@ def get_provider_info():
"hook-class-name":
"airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIHook",
"hook-name": "Pydantic AI",
"connection-type": "pydanticai",
+ "external-services": [
+ "OpenAI",
+ "Anthropic",
+ "Google",
+ "AWS Bedrock",
+ "Groq",
+ "Mistral AI",
+ "DeepSeek",
+ "Ollama",
+ "vLLM",
+ ],
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "login"],
"relabeling": {"password": "API Key"},
@@ -137,6 +148,7 @@ def get_provider_info():
"hook-class-name":
"airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIAzureHook",
"hook-name": "Pydantic AI (Azure OpenAI)",
"connection-type": "pydanticai-azure",
+ "external-services": ["Azure OpenAI"],
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "login"],
"relabeling": {"password": "API Key", "host": "Azure
Endpoint"},
@@ -159,6 +171,7 @@ def get_provider_info():
"hook-class-name":
"airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIBedrockHook",
"hook-name": "Pydantic AI (AWS Bedrock)",
"connection-type": "pydanticai-bedrock",
+ "external-services": ["AWS Bedrock"],
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "login", "host",
"password"],
"relabeling": {},
@@ -221,6 +234,7 @@ def get_provider_info():
"hook-class-name":
"airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIVertexHook",
"hook-name": "Pydantic AI (Google Vertex AI)",
"connection-type": "pydanticai-vertex",
+ "external-services": ["Google Vertex AI"],
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "login", "host",
"password"],
"relabeling": {},
@@ -295,6 +309,15 @@ def get_provider_info():
"hook-class-name":
"airflow.providers.common.ai.hooks.langchain.LangChainHook",
"hook-name": "LangChain",
"connection-type": "langchain",
+ "external-services": [
+ "OpenAI",
+ "Anthropic",
+ "Groq",
+ "Mistral AI",
+ "DeepSeek",
+ "Ollama",
+ "vLLM",
+ ],
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "login"],
"relabeling": {"password": "API Key"},
@@ -319,6 +342,7 @@ def get_provider_info():
"hook-class-name":
"airflow.providers.common.ai.hooks.llamaindex.LlamaIndexHook",
"hook-name": "LlamaIndex",
"connection-type": "llamaindex",
+ "external-services": ["OpenAI", "Ollama", "vLLM"],
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "login"],
"relabeling": {"password": "API Key"},
diff --git a/registry/src/css/main.css b/registry/src/css/main.css
index 3e0b077f429..ef006000248 100644
--- a/registry/src/css/main.css
+++ b/registry/src/css/main.css
@@ -3761,6 +3761,40 @@ main {
border-color: var(--color-green-400);
}
+/* Per-connection external services matrix */
+.provider-detail-page .connections-card .conn-external-services-table {
+ width: 100%;
+ margin-top: var(--space-4);
+ border-collapse: collapse;
+}
+
+.provider-detail-page .connections-card .conn-external-services-header th {
+ padding: var(--space-2) var(--space-3);
+ font-size: var(--text-sm);
+ font-weight: var(--font-semibold);
+ color: var(--text-secondary);
+ text-align: left;
+ background: var(--bg-tertiary);
+}
+
+.provider-detail-page .connections-card .conn-external-services-row {
+ border-top: 1px solid var(--border-primary);
+}
+
+.provider-detail-page .connections-card .conn-external-services-row td {
+ padding: var(--space-2) var(--space-3);
+ vertical-align: top;
+}
+
+.provider-detail-page .connections-card .conn-external-services-row
td:first-child {
+ white-space: nowrap;
+}
+
+.provider-detail-page .connections-card .conn-external-services-list {
+ color: var(--text-secondary);
+ font-size: var(--text-sm);
+}
+
/* Shared builder panel */
.provider-detail-page .connections-card .conn-builder-panel {
margin-top: var(--space-3);
diff --git a/registry/src/provider-version.njk
b/registry/src/provider-version.njk
index c76be316bec..ec56d1167f8 100644
--- a/registry/src/provider-version.njk
+++ b/registry/src/provider-version.njk
@@ -248,6 +248,30 @@ eleventyComputed:
</button>
{% endfor %}
</div>
+ {% set hasExternalServices = false %}
+ {% for conn in conns %}
+ {% if conn.external_services and conn.external_services.length > 0
%}{% set hasExternalServices = true %}{% endif %}
+ {% endfor %}
+ {% if hasExternalServices %}
+ <table class="conn-external-services-table">
+ <thead>
+ <tr class="conn-external-services-header">
+ <th>Connection type</th>
+ <th>External services (examples)</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for conn in conns %}
+ {% if conn.external_services and conn.external_services.length > 0 %}
+ <tr class="conn-external-services-row">
+ <td><code>{{ conn.conn_type or conn.connection_type }}</code></td>
+ <td class="conn-external-services-list">{{ conn.external_services
| join(", ") }}</td>
+ </tr>
+ {% endif %}
+ {% endfor %}
+ </tbody>
+ </table>
+ {% endif %}
<div class="conn-builder-panel" id="conn-builder-panel" hidden>
<div class="conn-builder-header">
<span class="conn-builder-title" id="conn-builder-title"></span>
