This is an automated email from the ASF dual-hosted git repository.
Lee-W pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b2a195a4a2a Add more provider module categories to the provider page
(#70190)
b2a195a4a2a is described below
commit b2a195a4a2a8419802b51c0bf599c4f72e0136ba
Author: Wei Lee <[email protected]>
AuthorDate: Thu Aug 6 16:22:02 2026 +0800
Add more provider module categories to the provider page (#70190)
---
dev/registry/extract_parameters.py | 101 ++++++++-----
dev/registry/extract_versions.py | 73 +++++++--
dev/registry/registry_tools/types.py | 69 +++++++++
dev/registry/tests/test_extract_parameters.py | 99 +++++++++++++
dev/registry/tests/test_extract_versions.py | 121 +++++++++++++++
dev/registry/tests/test_types.py | 26 ++++
registry/.eleventy.js | 22 +++
registry/src/_data/types.json | 30 ++++
registry/src/css/main.css | 205 ++++++++++++++++++++++++++
registry/src/css/tokens.css | 6 +
registry/src/js/provider-detail.js | 47 +++++-
registry/src/js/search.js | 8 +-
registry/src/provider-version.njk | 31 +++-
13 files changed, 785 insertions(+), 53 deletions(-)
diff --git a/dev/registry/extract_parameters.py
b/dev/registry/extract_parameters.py
index e32b06bae7a..7a44c4614ca 100644
--- a/dev/registry/extract_parameters.py
+++ b/dev/registry/extract_parameters.py
@@ -54,7 +54,13 @@ from pathlib import Path
import yaml
from extract_metadata import fetch_provider_inventory, read_inventory
from registry_contract_models import validate_modules_catalog,
validate_provider_parameters
-from registry_tools.types import BASE_CLASS_IMPORTS, CLASS_LEVEL_SECTIONS,
MODULE_LEVEL_SECTIONS
+from registry_tools.types import (
+ BASE_CLASS_IMPORTS,
+ CLASS_LEVEL_CATEGORY_OVERRIDES,
+ CLASS_LEVEL_SECTIONS,
+ DICT_SHAPED_CLASS_LEVEL_SECTIONS,
+ MODULE_LEVEL_SECTIONS,
+)
AIRFLOW_ROOT = Path(__file__).parent.parent.parent
SCRIPT_DIR = Path(__file__).parent
@@ -354,9 +360,9 @@ def _should_skip_class(name: str) -> bool:
return False
-def _get_first_docstring_line(cls: type) -> str | None:
+def _get_first_docstring_line(obj: object) -> str | None:
"""Return the first non-empty line of a class docstring, or None."""
- doc = getattr(cls, "__doc__", None)
+ doc = getattr(obj, "__doc__", None)
if not doc:
return None
for line in doc.strip().splitlines():
@@ -426,6 +432,27 @@ def is_durable_capable(cls: type, resumable_mixin: type |
None) -> bool:
return "execute_resumable" in source
+def _resolve_dotted_path(class_path: str) -> tuple[str, str, object] | None:
+ """Split a dotted ``module.name`` path and import ``name`` from that
module.
+
+ Returns ``None`` (without logging) if ``class_path`` has no dot, or
``None``
+ (after logging a warning) if the module import fails. Otherwise returns
+ ``(module_path, name, obj)``, where ``obj`` may be ``None`` if the module
+ has no such attribute — callers decide how to treat a missing attribute.
+ """
+ parts = class_path.rsplit(".", 1)
+ if len(parts) != 2:
+ return None
+ module_path, name = parts
+ try:
+ mod = importlib.import_module(module_path)
+ obj = getattr(mod, name, None)
+ except Exception:
+ log.warning("Could not import %s", class_path)
+ return None
+ return module_path, name, obj
+
+
def discover_classes_from_provider(
provider_yaml_path: Path,
base_classes: dict[str, type],
@@ -583,29 +610,14 @@ def discover_classes_from_provider(
for class_path in provider_yaml.get(section_name, []):
if not class_path or not isinstance(class_path, str):
continue
- parts = class_path.rsplit(".", 1)
- if len(parts) != 2:
- continue
- module_path, class_name = parts
- try:
- mod = importlib.import_module(module_path)
- candidate = getattr(mod, class_name, None)
- except Exception:
- log.warning("Could not import %s", class_path)
+ if (resolved := _resolve_dotted_path(class_path)) is None:
continue
+ module_path, class_name, candidate = resolved
if candidate is None or not inspect.isclass(candidate):
log.warning("%s is not a class", class_path)
continue
cls = candidate
- # Use section name as category for class-level entries
- category_map = {
- "notifications": "notifications",
- "secrets-backends": "secrets",
- "logging": "logging",
- "executors": "executors",
- }
-
discovered.append(
make_entry(
cls,
@@ -613,29 +625,52 @@ def discover_classes_from_provider(
module_type,
class_path,
module_path,
- category=category_map.get(section_name, section_name),
+ category=CLASS_LEVEL_CATEGORY_OVERRIDES.get(section_name,
section_name),
+ )
+ )
+
+ # --- Dict-shaped class-level sections (plugins, dialects; each entry is a
+ # dict carrying an integration-name field plus a class-path field, see
types.py) ---
+ for section_name, (
+ module_type,
+ class_field,
+ integration_field,
+ ) in DICT_SHAPED_CLASS_LEVEL_SECTIONS.items():
+ for entry in provider_yaml.get(section_name, []):
+ if not isinstance(entry, dict):
+ continue
+ if not (class_path := entry.get(class_field, "")):
+ continue
+ if (resolved := _resolve_dotted_path(class_path)) is None:
+ continue
+ module_path, class_name, candidate = resolved
+ if candidate is None or not inspect.isclass(candidate):
+ log.warning("%s is not a class", class_path)
+ continue
+
+ discovered.append(
+ make_entry(
+ candidate,
+ class_name,
+ module_type,
+ class_path,
+ module_path,
+ integration=entry.get(integration_field, ""),
+ category=CLASS_LEVEL_CATEGORY_OVERRIDES.get(section_name,
section_name),
)
)
# --- Task decorators (class-name key in each entry) ---
for decorator in provider_yaml.get("task-decorators", []):
- class_path = decorator.get("class-name", "")
- decorator_name = decorator.get("name", "")
- if not class_path:
- continue
- parts = class_path.rsplit(".", 1)
- if len(parts) != 2:
+ if not (class_path := decorator.get("class-name", "")):
continue
- module_path, func_name = parts
- try:
- mod = importlib.import_module(module_path)
- obj = getattr(mod, func_name, None)
- except Exception:
- log.warning("Could not import %s", class_path)
+ if (resolved := _resolve_dotted_path(class_path)) is None:
continue
+ module_path, func_name, obj = resolved
if obj is None:
continue
+ decorator_name = decorator.get("name", "")
display_name = f"@task.{decorator_name}" if decorator_name else
func_name
docstring = _get_first_docstring_line(obj) if hasattr(obj, "__doc__")
else None
short_desc = docstring or f"Task decorator for {decorator_name or
func_name}"
diff --git a/dev/registry/extract_versions.py b/dev/registry/extract_versions.py
index cc5f094c79f..c47d91a7043 100644
--- a/dev/registry/extract_versions.py
+++ b/dev/registry/extract_versions.py
@@ -59,7 +59,13 @@ except ImportError:
sys.exit(1)
from extract_metadata import fetch_provider_inventory, read_connection_urls,
resolve_connection_docs_url
-from registry_tools.types import MODULE_LEVEL_SECTIONS, TYPE_SUFFIXES
+from registry_tools.types import (
+ CLASS_LEVEL_CATEGORY_OVERRIDES,
+ CLASS_LEVEL_SECTIONS,
+ DICT_SHAPED_CLASS_LEVEL_SECTIONS,
+ MODULE_LEVEL_SECTIONS,
+ TYPE_SUFFIXES,
+)
SCRIPT_DIR = Path(__file__).parent
AIRFLOW_ROOT = Path(__file__).parent.parent.parent
@@ -76,6 +82,23 @@ PROVIDERS_JSON_CANDIDATES = [
REGISTRY_DIR / "src" / "_data" / "providers.json",
]
+# Description suffix for class-level (FQCN) sections, keyed by module type id.
+# Kept local to this file (unlike category, which is shared via
+# CLASS_LEVEL_CATEGORY_OVERRIDES) because extract_parameters.py has a real
+# docstring to use instead and doesn't need a description suffix at all.
+FQCN_DESC_SUFFIXES: dict[str, str] = {
+ "notifier": "notifier",
+ "secret": "secrets backend",
+ "logging": "log handler",
+ "executor": "executor",
+ "extra_link": "extra link",
+ "queue": "queue",
+ "auth_manager": "auth manager",
+ "db_manager": "db manager",
+ "plugin": "plugin",
+ "dialect": "dialect",
+}
+
def build_provider_id_to_path_map() -> dict[str, str]:
"""Scan providers/ for provider.yaml to build provider_id ->
directory_path mapping."""
@@ -300,15 +323,13 @@ def extract_modules_from_yaml(
if mp:
process_module(mp, "transfer", source, get_category(source))
- # Class-level sections (full class paths, no source file parsing needed)
- FQCN_SECTIONS: dict[str, tuple[str, str, str]] = {
- # yaml_key: (module_type, category, description_suffix)
- "notifications": ("notifier", "notifications", "notifier"),
- "secrets-backends": ("secret", "secrets", "secrets backend"),
- "logging": ("logging", "logging", "log handler"),
- "executors": ("executor", "executors", "executor"),
- }
- for yaml_key, (mod_type, category, desc_suffix) in FQCN_SECTIONS.items():
+ # Class-level sections (full class paths, no source file parsing needed).
+ # Iterating CLASS_LEVEL_SECTIONS keeps this in sync with the single
+ # source of truth in types.py; a section added there without a matching
+ # FQCN_DESC_SUFFIXES entry raises KeyError instead of silently skipping.
+ for yaml_key, mod_type in CLASS_LEVEL_SECTIONS.items():
+ category = CLASS_LEVEL_CATEGORY_OVERRIDES.get(yaml_key, yaml_key)
+ desc_suffix = FQCN_DESC_SUFFIXES[mod_type]
for class_path in provider_yaml.get(yaml_key, []):
if not class_path:
continue
@@ -329,6 +350,38 @@ def extract_modules_from_yaml(
}
)
+ # Dict-shaped class-level sections (each entry is a dict carrying the
+ # class path under a section-specific field name, see types.py).
+ for yaml_key, (
+ mod_type,
+ class_path_field,
+ _integration_field,
+ ) in DICT_SHAPED_CLASS_LEVEL_SECTIONS.items():
+ category = CLASS_LEVEL_CATEGORY_OVERRIDES.get(yaml_key, yaml_key)
+ desc_suffix = FQCN_DESC_SUFFIXES[mod_type]
+ for entry in provider_yaml.get(yaml_key, []):
+ if not isinstance(entry, dict):
+ continue
+ class_path = entry.get(class_path_field, "")
+ if not class_path:
+ continue
+ parts = class_path.rsplit(".", 1)
+ if len(parts) != 2:
+ continue
+ mod_path, class_name = parts
+ api_ref = mod_path.replace(".", "/")
+ modules.append(
+ {
+ "name": class_name,
+ "type": mod_type,
+ "import_path": class_path,
+ "short_description": f"{class_name} {desc_suffix}",
+ "docs_url":
f"{base_docs_url}/_api/{api_ref}/index.html#{class_path}",
+ "source_url": f"{base_source_url}/{api_ref}.py",
+ "category": category,
+ }
+ )
+
return modules
diff --git a/dev/registry/registry_tools/types.py
b/dev/registry/registry_tools/types.py
index cd91b70ed65..8f88ffe6e97 100644
--- a/dev/registry/registry_tools/types.py
+++ b/dev/registry/registry_tools/types.py
@@ -116,6 +116,48 @@ MODULE_TYPES: dict[str, dict] = {
"label": "Retry Policies",
"icon": "R",
},
+ "extra_link": {
+ "yaml_key": "extra-links",
+ "level": "flat",
+ "suffixes": [],
+ "label": "Extra Links",
+ "icon": "I",
+ },
+ "queue": {
+ "yaml_key": "queues",
+ "level": "flat",
+ "suffixes": [],
+ "label": "Message Queues",
+ "icon": "Q",
+ },
+ "plugin": {
+ "yaml_key": "plugins",
+ "level": "flat",
+ "suffixes": [],
+ "label": "Plugins",
+ "icon": "P",
+ },
+ "auth_manager": {
+ "yaml_key": "auth-managers",
+ "level": "flat",
+ "suffixes": [],
+ "label": "Auth Managers",
+ "icon": "A",
+ },
+ "db_manager": {
+ "yaml_key": "db-managers",
+ "level": "flat",
+ "suffixes": [],
+ "label": "DB Managers",
+ "icon": "M",
+ },
+ "dialect": {
+ "yaml_key": "dialects",
+ "level": "flat",
+ "suffixes": [],
+ "label": "Dialects",
+ "icon": "D",
+ },
}
# Runtime base class imports for issubclass checks (extract_parameters.py).
@@ -147,11 +189,38 @@ TYPE_SUFFIXES: dict[str, list[str]] = {type_id:
info["suffixes"] for type_id, in
# Class-level sections used by extract_parameters.py (subset of flat that
# list full class paths rather than simple entries).
+#
+# "plugins" and "dialects" are also flat/class-path sections, but each entry
is a
+# dict (plugin-class / dialect-class-name) rather than a bare string, so they
are
+# handled by the generic dict-shaped loop driven by
DICT_SHAPED_CLASS_LEVEL_SECTIONS
+# below instead of this table.
CLASS_LEVEL_SECTIONS: dict[str, str] = {
"notifications": "notifier",
"secrets-backends": "secret",
"logging": "logging",
"executors": "executor",
+ "extra-links": "extra_link",
+ "queues": "queue",
+ "auth-managers": "auth_manager",
+ "db-managers": "db_manager",
+}
+
+# Maps yaml section key -> (type id, class-path field name, integration-name
+# field name) for dict-shaped class-level sections (each yaml entry is a dict,
+# not a bare class-path string). Shared by extract_versions.py and
+# extract_parameters.py so both walk these sections with a single generic loop
+# instead of one per section, and so the field names are defined exactly once.
+DICT_SHAPED_CLASS_LEVEL_SECTIONS: dict[str, tuple[str, str, str]] = {
+ "plugins": ("plugin", "plugin-class", "name"),
+ "dialects": ("dialect", "dialect-class-name", "dialect-type"),
+}
+
+# Maps yaml section key -> category string for class-level (FQCN) sections.
+# Only lists yaml keys whose category differs from the key itself. Callers
+# fall back to the yaml key via .get(section_name, section_name) for every
+# other key, including ones like "notifications" where key == category.
+CLASS_LEVEL_CATEGORY_OVERRIDES: dict[str, str] = {
+ "secrets-backends": "secrets",
}
# All type ids, ordered consistently.
diff --git a/dev/registry/tests/test_extract_parameters.py
b/dev/registry/tests/test_extract_parameters.py
index 2cdb0953229..3ebdf2004a7 100644
--- a/dev/registry/tests/test_extract_parameters.py
+++ b/dev/registry/tests/test_extract_parameters.py
@@ -30,6 +30,7 @@ from extract_parameters import (
Module,
_get_source_line,
_parse_requested_providers,
+ _resolve_dotted_path,
_should_skip_class,
compare_with_ast,
discover_classes_from_provider,
@@ -370,6 +371,18 @@ class FakeSecretBackend:
__module__ = "airflow.providers.amazon.aws.secrets.secrets_manager"
+class FakePlugin:
+ """S3 plugin."""
+
+ __module__ = "airflow.providers.amazon.aws.plugins.s3"
+
+
+class FakeDialect:
+ """Redshift dialect."""
+
+ __module__ = "airflow.providers.amazon.aws.dialects.redshift"
+
+
def _make_module(name: str, members: dict) -> types.ModuleType:
"""Create a fake module with given members."""
mod = types.ModuleType(name)
@@ -411,6 +424,20 @@ FAKE_PROVIDER_YAML = {
"executors": [
"airflow.providers.amazon.aws.executors.ecs.FakeExecutor",
],
+ "plugins": [
+ {
+ "name": "AmazonS3Plugin",
+ "plugin-class":
"airflow.providers.amazon.aws.plugins.s3.FakePlugin",
+ },
+ "not-a-dict-entry",
+ ],
+ "dialects": [
+ {
+ "dialect-type": "redshift",
+ "dialect-class-name":
"airflow.providers.amazon.aws.dialects.redshift.FakeDialect",
+ },
+ "not-a-dict-entry",
+ ],
"task-decorators": [],
}
@@ -473,6 +500,14 @@ class TestDiscoverClassesFromProvider:
"airflow.providers.amazon.aws.executors.ecs",
{"FakeExecutor": FakeExecutor},
),
+ "airflow.providers.amazon.aws.plugins.s3": _make_module(
+ "airflow.providers.amazon.aws.plugins.s3",
+ {"FakePlugin": FakePlugin},
+ ),
+ "airflow.providers.amazon.aws.dialects.redshift": _make_module(
+ "airflow.providers.amazon.aws.dialects.redshift",
+ {"FakeDialect": FakeDialect},
+ ),
}
if module_name in modules:
return modules[module_name]
@@ -513,6 +548,42 @@ class TestDiscoverClassesFromProvider:
assert len(hooks) == 1
assert hooks[0]["name"] == "FakeHook"
+ def test_discovers_plugin(self, provider_yaml_path, base_classes):
+ with (
+ patch("extract_parameters.PROVIDERS_DIR",
provider_yaml_path.parent.parent),
+ patch("extract_parameters.importlib.import_module",
side_effect=self._mock_import),
+ ):
+ result = discover_classes_from_provider(provider_yaml_path,
base_classes)
+
+ plugins = [r for r in result if r["type"] == "plugin"]
+ assert len(plugins) == 1
+ assert plugins[0]["name"] == "FakePlugin"
+ assert plugins[0]["category"] == "plugins"
+
+ def test_discovers_dialect(self, provider_yaml_path, base_classes):
+ with (
+ patch("extract_parameters.PROVIDERS_DIR",
provider_yaml_path.parent.parent),
+ patch("extract_parameters.importlib.import_module",
side_effect=self._mock_import),
+ ):
+ result = discover_classes_from_provider(provider_yaml_path,
base_classes)
+
+ dialects = [r for r in result if r["type"] == "dialect"]
+ assert len(dialects) == 1
+ assert dialects[0]["name"] == "FakeDialect"
+ assert dialects[0]["category"] == "dialects"
+
+ def test_skips_non_dict_plugin_and_dialect_entries(self,
provider_yaml_path, base_classes):
+ """FAKE_PROVIDER_YAML's plugins/dialects each carry a bare-string entry
+ alongside the valid dict entry; it must be skipped, not raise."""
+ with (
+ patch("extract_parameters.PROVIDERS_DIR",
provider_yaml_path.parent.parent),
+ patch("extract_parameters.importlib.import_module",
side_effect=self._mock_import),
+ ):
+ result = discover_classes_from_provider(provider_yaml_path,
base_classes)
+
+ assert len([r for r in result if r["type"] == "plugin"]) == 1
+ assert len([r for r in result if r["type"] == "dialect"]) == 1
+
def test_filters_reexported_classes(self, provider_yaml_path,
base_classes):
"""Classes where cls.__module__ != the module being scanned should be
excluded."""
with (
@@ -1086,3 +1157,31 @@ class TestParseRequestedProviders:
def test_duplicate_providers_collapsed(self):
assert _parse_requested_providers("amazon amazon google") ==
{"amazon", "google"}
+
+
+class TestResolveDottedPath:
+ @pytest.mark.parametrize(
+ "class_path",
+ (
+ "no_dot_here",
+ "this.module.does.not.exist.at.all.Foo",
+ ),
+ )
+ def test__resolve_dotted_path_returns_none(self, class_path: str):
+ assert _resolve_dotted_path(class_path) is None
+
+ @pytest.mark.parametrize(
+ ("class_path", "expected"),
+ (
+ (
+ "extract_parameters.ThisAttrDoesNotExist",
+ ("extract_parameters", "ThisAttrDoesNotExist", None),
+ ),
+ (
+ "extract_parameters.get_category",
+ ("extract_parameters", "get_category", get_category),
+ ),
+ ),
+ )
+ def test__resolve_dotted_path(self, class_path: str, expected: tuple[str,
str, object]):
+ assert _resolve_dotted_path(class_path) == expected
diff --git a/dev/registry/tests/test_extract_versions.py
b/dev/registry/tests/test_extract_versions.py
index 45a2e26a2ea..a2c8ce33575 100644
--- a/dev/registry/tests/test_extract_versions.py
+++ b/dev/registry/tests/test_extract_versions.py
@@ -18,11 +18,14 @@
from __future__ import annotations
+import pytest
from extract_versions import (
AIRFLOW_ROOT,
PROVIDERS_JSON_CANDIDATES,
SCRIPT_DIR,
+ extract_modules_from_yaml,
)
+from registry_tools.types import CLASS_LEVEL_SECTIONS,
DICT_SHAPED_CLASS_LEVEL_SECTIONS
class TestProvidersJsonCandidates:
@@ -49,3 +52,121 @@ class TestProvidersJsonCandidates:
# should be caught and fixed at the source. Match siblings (extract_
# parameters.py, extract_metadata.py) which use exactly these two.
assert len(PROVIDERS_JSON_CANDIDATES) == 2
+
+
+# Expected (category, description_suffix) per class-level section, entered by
+# hand from the PR #70190 spec rather than derived from production code, so a
+# regression in extract_versions.py's mapping logic is actually caught.
+EXPECTED_CLASS_LEVEL_CATEGORIES = {
+ "notifications": "notifications",
+ "secrets-backends": "secrets",
+ "logging": "logging",
+ "executors": "executors",
+ "extra-links": "extra-links",
+ "queues": "queues",
+ "auth-managers": "auth-managers",
+ "db-managers": "db-managers",
+}
+
+EXPECTED_CLASS_LEVEL_DESC_SUFFIXES = {
+ "notifier": "notifier",
+ "secret": "secrets backend",
+ "logging": "log handler",
+ "executor": "executor",
+ "extra_link": "extra link",
+ "queue": "queue",
+ "auth_manager": "auth manager",
+ "db_manager": "db manager",
+}
+
+
+def _extract_class_level_modules(provider_yaml: dict) -> list[dict]:
+ return extract_modules_from_yaml(
+ provider_yaml,
+ tag="providers-test/1.0.0",
+ layout="new",
+ dir_path="test",
+ provider_id="test",
+ version="1.0.0",
+ )
+
+
+class TestExtractModulesFromYamlClassLevelSections:
+ @pytest.mark.parametrize(("yaml_key", "mod_type"),
list(CLASS_LEVEL_SECTIONS.items()))
+ def test_class_level_section_produces_module(self, yaml_key, mod_type):
+ class_path = f"airflow.providers.test.{yaml_key.replace('-',
'_')}.example.ExampleClass"
+ provider_yaml = {yaml_key: [class_path]}
+
+ modules = _extract_class_level_modules(provider_yaml)
+
+ assert len(modules) == 1
+ module = modules[0]
+ assert module["type"] == mod_type
+ assert module["category"] == EXPECTED_CLASS_LEVEL_CATEGORIES[yaml_key]
+ expected_desc_suffix = EXPECTED_CLASS_LEVEL_DESC_SUFFIXES[mod_type]
+ assert module["short_description"] == f"ExampleClass
{expected_desc_suffix}"
+
+ def test_all_class_level_sections_produce_covered_types(self):
+ """Type coverage must track CLASS_LEVEL_SECTIONS; a key added there
+ without matching handling in extract_versions.py fails here."""
+ provider_yaml = {
+ yaml_key: [f"airflow.providers.test.{yaml_key.replace('-',
'_')}.example.ExampleClass"]
+ for yaml_key in CLASS_LEVEL_SECTIONS
+ }
+
+ modules = _extract_class_level_modules(provider_yaml)
+
+ assert {m["type"] for m in modules} ==
set(CLASS_LEVEL_SECTIONS.values())
+
+
+# plugins/dialects aren't in CLASS_LEVEL_CATEGORY_OVERRIDES, so category ==
+# yaml_key itself (same fallback as EXPECTED_CLASS_LEVEL_CATEGORIES above).
+EXPECTED_DICT_SHAPED_CATEGORIES = {
+ "plugins": "plugins",
+ "dialects": "dialects",
+}
+
+EXPECTED_DICT_SHAPED_DESC_SUFFIXES = {
+ "plugin": "plugin",
+ "dialect": "dialect",
+}
+
+
+class TestExtractModulesFromYamlDictShapedSections:
+ @pytest.mark.parametrize(
+ ("yaml_key", "type_and_field"),
+ list(DICT_SHAPED_CLASS_LEVEL_SECTIONS.items()),
+ )
+ def test_dict_shaped_section_produces_module(self, yaml_key,
type_and_field):
+ mod_type, class_path_field, _integration_field = type_and_field
+ class_path = f"airflow.providers.test.{yaml_key.replace('-',
'_')}.example.ExampleClass"
+ provider_yaml = {yaml_key: [{class_path_field: class_path}]}
+
+ modules = _extract_class_level_modules(provider_yaml)
+
+ assert len(modules) == 1
+ module = modules[0]
+ assert module["type"] == mod_type
+ assert module["category"] == EXPECTED_DICT_SHAPED_CATEGORIES[yaml_key]
+ expected_desc_suffix = EXPECTED_DICT_SHAPED_DESC_SUFFIXES[mod_type]
+ assert module["short_description"] == f"ExampleClass
{expected_desc_suffix}"
+
+ def test_all_dict_shaped_sections_produce_covered_types(self):
+ """Type coverage must track DICT_SHAPED_CLASS_LEVEL_SECTIONS; a key
+ added there without matching handling in extract_versions.py fails
here."""
+ provider_yaml = {
+ yaml_key: [
+ {
+ class_path_field:
f"airflow.providers.test.{yaml_key.replace('-', '_')}.example.ExampleClass"
+ }
+ ]
+ for yaml_key, (
+ _,
+ class_path_field,
+ _integration_field,
+ ) in DICT_SHAPED_CLASS_LEVEL_SECTIONS.items()
+ }
+
+ modules = _extract_class_level_modules(provider_yaml)
+
+ assert {m["type"] for m in modules} == {t for t, _, _ in
DICT_SHAPED_CLASS_LEVEL_SECTIONS.values()}
diff --git a/dev/registry/tests/test_types.py b/dev/registry/tests/test_types.py
index d5d2dfbf218..85bb578745d 100644
--- a/dev/registry/tests/test_types.py
+++ b/dev/registry/tests/test_types.py
@@ -22,7 +22,9 @@ import pytest
from registry_tools.types import (
ALL_TYPE_IDS,
BASE_CLASS_IMPORTS,
+ CLASS_LEVEL_CATEGORY_OVERRIDES,
CLASS_LEVEL_SECTIONS,
+ DICT_SHAPED_CLASS_LEVEL_SECTIONS,
FLAT_LEVEL_SECTIONS,
MODULE_LEVEL_SECTIONS,
MODULE_TYPES,
@@ -93,6 +95,30 @@ class TestDerivedLookups:
assert yaml_key in FLAT_LEVEL_SECTIONS
assert FLAT_LEVEL_SECTIONS[yaml_key] == type_id
+ def
test_class_level_category_overrides_are_subset_of_class_level_sections(self):
+ assert set(CLASS_LEVEL_CATEGORY_OVERRIDES.keys()) <=
set(CLASS_LEVEL_SECTIONS.keys())
+
+ def test_dict_shaped_class_level_sections_are_subset_of_flat(self):
+ for yaml_key, (type_id, _field_name, _integration_field) in
DICT_SHAPED_CLASS_LEVEL_SECTIONS.items():
+ assert yaml_key in FLAT_LEVEL_SECTIONS
+ assert FLAT_LEVEL_SECTIONS[yaml_key] == type_id
+
+ def test_every_flat_section_is_consumed_by_a_class_level_table(self):
+ # "transfers" and "task-decorators" are deliberately exempt: they are
+ # consumed by their own dedicated extraction paths rather than by
+ # CLASS_LEVEL_SECTIONS or DICT_SHAPED_CLASS_LEVEL_SECTIONS.
+ dedicated_path_exemptions = {"transfers", "task-decorators"}
+ consumed = (
+ set(CLASS_LEVEL_SECTIONS) | set(DICT_SHAPED_CLASS_LEVEL_SECTIONS)
| dedicated_path_exemptions
+ )
+ assert set(FLAT_LEVEL_SECTIONS) == consumed, (
+ f"set(FLAT_LEVEL_SECTIONS) {set(FLAT_LEVEL_SECTIONS)} != consumed
{consumed}. "
+ "Every flat-level section must be consumed by CLASS_LEVEL_SECTIONS
or "
+ "DICT_SHAPED_CLASS_LEVEL_SECTIONS, otherwise both extractors
silently skip it. "
+ "Add the new section to whichever table matches its yaml shape,
or, if it is "
+ "handled by a dedicated path, add it to dedicated_path_exemptions
above."
+ )
+
class TestBaseClassImports:
def test_all_entries_are_tuples(self):
diff --git a/registry/.eleventy.js b/registry/.eleventy.js
index bdd5ce7b20c..6a61ca7d792 100644
--- a/registry/.eleventy.js
+++ b/registry/.eleventy.js
@@ -99,6 +99,28 @@ module.exports = function(eleventyConfig) {
return JSON.stringify(obj);
});
+ // Attaches each type's module count and drops zero-count types, sorted by
+ // count descending -- used to render the provider-page module tabs with the
+ // busiest categories first.
+ eleventyConfig.addFilter("sortTypesByCount", (types, moduleCounts) => {
+ if (!Array.isArray(types)) return [];
+ const counts = moduleCounts || {};
+ return types
+ .filter((t) => (counts[t.id] || 0) > 0)
+ .map((t) => Object.assign({}, t, { count: counts[t.id] || 0 }))
+ .sort((a, b) => b.count - a.count);
+ });
+
+ // Looks up the curated single-letter icon for a type id from types.json,
instead of
+ // deriving it from the id's first character at each render site (which
reintroduces
+ // the letter collisions types.json's `icon` field was curated to avoid).
+ eleventyConfig.addFilter("typeIcon", (types, typeId) => {
+ if (!Array.isArray(types) || !typeId) return "";
+ const match = types.find((t) => t.id === typeId);
+ if (match && match.icon) return match.icon;
+ return typeId.charAt(0).toUpperCase();
+ });
+
eleventyConfig.addShortcode("year", () => `${new Date().getFullYear()}`);
return {
diff --git a/registry/src/_data/types.json b/registry/src/_data/types.json
index 150f986b2e4..cb4d36f0166 100644
--- a/registry/src/_data/types.json
+++ b/registry/src/_data/types.json
@@ -63,5 +63,35 @@
"id": "retry_policy",
"label": "Retry Policies",
"icon": "R"
+ },
+ {
+ "id": "extra_link",
+ "label": "Extra Links",
+ "icon": "I"
+ },
+ {
+ "id": "queue",
+ "label": "Message Queues",
+ "icon": "Q"
+ },
+ {
+ "id": "plugin",
+ "label": "Plugins",
+ "icon": "P"
+ },
+ {
+ "id": "auth_manager",
+ "label": "Auth Managers",
+ "icon": "A"
+ },
+ {
+ "id": "db_manager",
+ "label": "DB Managers",
+ "icon": "M"
+ },
+ {
+ "id": "dialect",
+ "label": "Dialects",
+ "icon": "D"
}
]
diff --git a/registry/src/css/main.css b/registry/src/css/main.css
index 8a262428b55..3e0b077f429 100644
--- a/registry/src/css/main.css
+++ b/registry/src/css/main.css
@@ -898,6 +898,14 @@ footer a:hover {
.provider-card .modules .logging { background: var(--color-logging); }
.provider-card .modules .bundle { background: var(--color-bundle); }
.provider-card .modules .decorator { background: var(--color-decorator); }
+.provider-card .modules .toolset { background: var(--color-toolset); }
+.provider-card .modules .retry_policy { background: var(--color-retry_policy);
}
+.provider-card .modules .extra_link { background: var(--color-extra_link); }
+.provider-card .modules .queue { background: var(--color-queue); }
+.provider-card .modules .plugin { background: var(--color-plugin); }
+.provider-card .modules .auth_manager { background: var(--color-auth_manager);
}
+.provider-card .modules .db_manager { background: var(--color-db_manager); }
+.provider-card .modules .dialect { background: var(--color-dialect); }
.provider-card .meta,
#featured-providers .meta {
@@ -2158,6 +2166,45 @@ main {
color: var(--color-decorator);
}
+.type-icon.toolset {
+ background: rgb(from var(--color-toolset) r g b / 0.2);
+ color: var(--color-toolset);
+}
+
+.type-icon.retry_policy {
+ background: rgb(from var(--color-retry_policy) r g b / 0.2);
+ color: var(--color-retry_policy);
+}
+
+.type-icon.extra_link {
+ background: rgb(from var(--color-extra_link) r g b / 0.2);
+ color: var(--color-extra_link);
+}
+
+.type-icon.queue {
+ background: rgb(from var(--color-queue) r g b / 0.2);
+ color: var(--color-queue);
+}
+
+.type-icon.plugin {
+ background: rgb(from var(--color-plugin) r g b / 0.2);
+ color: var(--color-plugin);
+}
+
+.type-icon.auth_manager {
+ background: rgb(from var(--color-auth_manager) r g b / 0.2);
+ color: var(--color-auth_manager);
+}
+
+.type-icon.db_manager {
+ background: rgb(from var(--color-db_manager) r g b / 0.2);
+ color: var(--color-db_manager);
+}
+
+.type-icon.dialect {
+ background: rgb(from var(--color-dialect) r g b / 0.2);
+ color: var(--color-dialect);
+}
.type-count {
font-size: var(--text-2xl);
@@ -2246,6 +2293,37 @@ main {
--meter-color: var(--color-decorator);
}
+.share-bar.toolset {
+ --meter-color: var(--color-toolset);
+}
+
+.share-bar.retry_policy {
+ --meter-color: var(--color-retry_policy);
+}
+
+.share-bar.extra_link {
+ --meter-color: var(--color-extra_link);
+}
+
+.share-bar.queue {
+ --meter-color: var(--color-queue);
+}
+
+.share-bar.plugin {
+ --meter-color: var(--color-plugin);
+}
+
+.share-bar.auth_manager {
+ --meter-color: var(--color-auth_manager);
+}
+
+.share-bar.db_manager {
+ --meter-color: var(--color-db_manager);
+}
+
+.share-bar.dialect {
+ --meter-color: var(--color-dialect);
+}
.share-label {
font-size: var(--text-xs);
@@ -3067,6 +3145,55 @@ main {
.tab-icon.decorator { background: rgb(from var(--color-decorator) r g b /
0.2); color: var(--color-decorator); }
.tab-icon.toolset { background: rgb(from var(--color-toolset) r g b / 0.2);
color: var(--color-toolset); }
.tab-icon.retry_policy { background: rgb(from var(--color-retry_policy) r g b
/ 0.2); color: var(--color-retry_policy); }
+.tab-icon.extra_link { background: rgb(from var(--color-extra_link) r g b /
0.2); color: var(--color-extra_link); }
+.tab-icon.queue { background: rgb(from var(--color-queue) r g b / 0.2); color:
var(--color-queue); }
+.tab-icon.plugin { background: rgb(from var(--color-plugin) r g b / 0.2);
color: var(--color-plugin); }
+.tab-icon.auth_manager { background: rgb(from var(--color-auth_manager) r g b
/ 0.2); color: var(--color-auth_manager); }
+.tab-icon.db_manager { background: rgb(from var(--color-db_manager) r g b /
0.2); color: var(--color-db_manager); }
+.tab-icon.dialect { background: rgb(from var(--color-dialect) r g b / 0.2);
color: var(--color-dialect); }
+
+/* Module Tabs "More" overflow menu */
+.module-tab-more {
+ position: relative;
+}
+
+.module-tab-more-btn {
+ display: inline-flex;
+}
+
+.module-tab-more-btn svg {
+ width: 1rem;
+ height: 1rem;
+ transition: transform var(--transition-base);
+}
+
+.module-tab-more-btn[aria-expanded="true"] svg {
+ transform: rotate(180deg);
+}
+
+.module-tab-more-menu {
+ position: absolute;
+ top: calc(100% + var(--space-2));
+ left: 0;
+ z-index: 20;
+ display: flex;
+ flex-direction: column;
+ gap: var(--space-1);
+ min-width: 12rem;
+ padding: var(--space-2);
+ background: var(--bg-secondary);
+ border: 1px solid var(--border-primary);
+ border-radius: var(--radius-lg);
+ box-shadow: var(--shadow-lg);
+}
+
+.module-tab-more-menu[hidden] {
+ display: none;
+}
+
+.module-tab-more-menu .module-tab {
+ justify-content: flex-start;
+}
/* Modules Layout (sidebar + content) */
.modules-layout {
@@ -3242,6 +3369,84 @@ main {
border: 1px solid rgb(from var(--color-bundle) r g b / 0.3);
}
+.provider-detail-page .modules .module .icon.notifier {
+ background: rgb(from var(--color-notifier) r g b / 0.2);
+ color: var(--color-notifier);
+ border: 1px solid rgb(from var(--color-notifier) r g b / 0.3);
+}
+
+.provider-detail-page .modules .module .icon.secret {
+ background: rgb(from var(--color-secret) r g b / 0.2);
+ color: var(--color-secret);
+ border: 1px solid rgb(from var(--color-secret) r g b / 0.3);
+}
+
+.provider-detail-page .modules .module .icon.logging {
+ background: rgb(from var(--color-logging) r g b / 0.2);
+ color: var(--color-logging);
+ border: 1px solid rgb(from var(--color-logging) r g b / 0.3);
+}
+
+.provider-detail-page .modules .module .icon.executor {
+ background: rgb(from var(--color-executor) r g b / 0.2);
+ color: var(--color-executor);
+ border: 1px solid rgb(from var(--color-executor) r g b / 0.3);
+}
+
+.provider-detail-page .modules .module .icon.decorator {
+ background: rgb(from var(--color-decorator) r g b / 0.2);
+ color: var(--color-decorator);
+ border: 1px solid rgb(from var(--color-decorator) r g b / 0.3);
+}
+
+.provider-detail-page .modules .module .icon.toolset {
+ background: rgb(from var(--color-toolset) r g b / 0.2);
+ color: var(--color-toolset);
+ border: 1px solid rgb(from var(--color-toolset) r g b / 0.3);
+}
+
+.provider-detail-page .modules .module .icon.retry_policy {
+ background: rgb(from var(--color-retry_policy) r g b / 0.2);
+ color: var(--color-retry_policy);
+ border: 1px solid rgb(from var(--color-retry_policy) r g b / 0.3);
+}
+
+.provider-detail-page .modules .module .icon.extra_link {
+ background: rgb(from var(--color-extra_link) r g b / 0.2);
+ color: var(--color-extra_link);
+ border: 1px solid rgb(from var(--color-extra_link) r g b / 0.3);
+}
+
+.provider-detail-page .modules .module .icon.queue {
+ background: rgb(from var(--color-queue) r g b / 0.2);
+ color: var(--color-queue);
+ border: 1px solid rgb(from var(--color-queue) r g b / 0.3);
+}
+
+.provider-detail-page .modules .module .icon.plugin {
+ background: rgb(from var(--color-plugin) r g b / 0.2);
+ color: var(--color-plugin);
+ border: 1px solid rgb(from var(--color-plugin) r g b / 0.3);
+}
+
+.provider-detail-page .modules .module .icon.auth_manager {
+ background: rgb(from var(--color-auth_manager) r g b / 0.2);
+ color: var(--color-auth_manager);
+ border: 1px solid rgb(from var(--color-auth_manager) r g b / 0.3);
+}
+
+.provider-detail-page .modules .module .icon.db_manager {
+ background: rgb(from var(--color-db_manager) r g b / 0.2);
+ color: var(--color-db_manager);
+ border: 1px solid rgb(from var(--color-db_manager) r g b / 0.3);
+}
+
+.provider-detail-page .modules .module .icon.dialect {
+ background: rgb(from var(--color-dialect) r g b / 0.2);
+ color: var(--color-dialect);
+ border: 1px solid rgb(from var(--color-dialect) r g b / 0.3);
+}
+
.provider-detail-page .modules .module .content {
flex: 1;
min-width: 0;
diff --git a/registry/src/css/tokens.css b/registry/src/css/tokens.css
index 74a6c039e00..5d8b27897f8 100644
--- a/registry/src/css/tokens.css
+++ b/registry/src/css/tokens.css
@@ -67,6 +67,12 @@
--color-decorator: #d946ef;
--color-toolset: #84cc16;
--color-retry_policy: #10b981;
+ --color-extra_link: #06b6d4;
+ --color-queue: #eab308;
+ --color-plugin: #8b5cf6;
+ --color-auth_manager: #ef4444;
+ --color-db_manager: #64748b;
+ --color-dialect: #f472b6;
/* Additional Colors */
--color-green-400: #4ade80;
diff --git a/registry/src/js/provider-detail.js
b/registry/src/js/provider-detail.js
index df6a81c02e1..2ecb7a52f99 100644
--- a/registry/src/js/provider-detail.js
+++ b/registry/src/js/provider-detail.js
@@ -25,7 +25,9 @@
var extraDepsList = document.getElementById('extra-deps-list');
var extrasDataEl = document.getElementById('extras-data');
var moduleSearch = document.getElementById('module-search');
- var moduleTabs = document.querySelectorAll('.module-tab');
+ // [data-type] excludes the "More" toggle button, which is a .module-tab
+ // for styling purposes only and has no data-type of its own.
+ var moduleTabs = document.querySelectorAll('.module-tab[data-type]');
var categoryBtns = document.querySelectorAll('.category-btn');
var moduleItems = document.querySelectorAll('.modules .module');
var copyImportBtns = document.querySelectorAll('.copy-import');
@@ -152,9 +154,52 @@
tab.classList.add('active');
currentType = tab.dataset.type || 'all';
filterModules();
+ if (moduleTabMoreBtn && moduleTabMoreMenu) {
+ moduleTabMoreBtn.classList.toggle('active',
moduleTabMoreMenu.contains(tab));
+ }
});
});
+ // "More" overflow menu for module tabs that don't fit in the visible row.
+ var moduleTabMoreBtn = document.getElementById('module-tab-more-btn');
+ var moduleTabMoreMenu = document.getElementById('module-tab-more-menu');
+
+ if (moduleTabMoreBtn && moduleTabMoreMenu) {
+ function closeMoreMenu() {
+ var focusWasInMenu = moduleTabMoreMenu.contains(document.activeElement);
+ moduleTabMoreBtn.setAttribute('aria-expanded', 'false');
+ moduleTabMoreMenu.hidden = true;
+ if (focusWasInMenu) { moduleTabMoreBtn.focus(); }
+ }
+
+ moduleTabMoreBtn.addEventListener('click', function(e) {
+ e.stopPropagation();
+ var isOpen = moduleTabMoreBtn.getAttribute('aria-expanded') === 'true';
+ if (isOpen) {
+ closeMoreMenu();
+ } else {
+ moduleTabMoreBtn.setAttribute('aria-expanded', 'true');
+ moduleTabMoreMenu.hidden = false;
+ }
+ });
+
+ document.addEventListener('click', function(e) {
+ if (!moduleTabMoreMenu.contains(e.target) && e.target !==
moduleTabMoreBtn) {
+ closeMoreMenu();
+ }
+ });
+
+ document.addEventListener('keydown', function(e) {
+ if (e.key === 'Escape') closeMoreMenu();
+ });
+
+ // Selecting a tab inside the menu also closes it (filtering itself is
+ // handled by the shared moduleTabs click listener above).
+ moduleTabMoreMenu.querySelectorAll('.module-tab').forEach(function(tab) {
+ tab.addEventListener('click', closeMoreMenu);
+ });
+ }
+
categoryBtns.forEach(function(btn) {
btn.addEventListener('click', function() {
categoryBtns.forEach(function(b) { b.classList.remove('active'); });
diff --git a/registry/src/js/search.js b/registry/src/js/search.js
index d7b13d523d6..1f984183418 100644
--- a/registry/src/js/search.js
+++ b/registry/src/js/search.js
@@ -24,17 +24,19 @@
let currentResults = [];
let searchId = 0;
- // Type labels loaded from types.json (injected via base.njk)
+ // Type labels/icons loaded from types.json (injected via base.njk)
const typeLabels = {};
+ const typeIcons = {};
try {
const typesEl = document.getElementById('types-data');
if (typesEl) {
for (const t of JSON.parse(typesEl.textContent)) {
typeLabels[t.id] = t.label;
+ typeIcons[t.id] = t.icon;
}
}
} catch (_) {
- // Fallback: empty object — badges will show raw type name
+ // Fallback: empty objects — badges show raw type name, icons fall back to
first letter
}
function escapeHtml(str) {
@@ -105,7 +107,7 @@
const providerName = result.meta.providerName || '';
const description = result.meta.description || result.excerpt;
const moduleType = result.meta.moduleType || '';
- const icon = type === 'provider' ? 'P' : (moduleType ?
moduleType[0].toUpperCase() : 'M');
+ const icon = type === 'provider' ? 'P' : (moduleType ?
(typeIcons[moduleType] || moduleType[0].toUpperCase()) : 'M');
const resultType = type === 'provider' ? 'provider' : moduleType;
return `
diff --git a/registry/src/provider-version.njk
b/registry/src/provider-version.njk
index 49356c0796f..c76be316bec 100644
--- a/registry/src/provider-version.njk
+++ b/registry/src/provider-version.njk
@@ -276,16 +276,35 @@ eleventyComputed:
</section>
{% if totalModules > 0 %}
- {# Module Type Tabs #}
+ {# Module Type Tabs: sorted by module count (busiest first), overflow into a
"More" menu #}
+ {% set moduleTabsVisibleCount = 8 %}
+ {% set sortedTypes = types | sortTypesByCount(moduleCounts) %}
+ {% set visibleTypes = sortedTypes | slice(0, moduleTabsVisibleCount) %}
+ {% set overflowTypes = sortedTypes | slice(moduleTabsVisibleCount,
sortedTypes.length) %}
<nav class="module-tabs">
<button class="module-tab active" data-type="all">All ({{ totalModules
}})</button>
- {% for t in types %}
- {% if moduleCounts[t.id] > 0 %}
+ {% for t in visibleTypes %}
<button class="module-tab" data-type="{{ t.id }}">
- <span class="tab-icon {{ t.id }}">{{ t.icon }}</span> {{ t.label }} ({{
moduleCounts[t.id] }})
+ <span class="tab-icon {{ t.id }}">{{ t.icon }}</span> {{ t.label }} ({{
t.count }})
</button>
- {% endif %}
{% endfor %}
+ {% if overflowTypes.length > 0 %}
+ <div class="module-tab-more">
+ <button class="module-tab module-tab-more-btn" type="button"
id="module-tab-more-btn" aria-haspopup="true" aria-expanded="false"
aria-controls="module-tab-more-menu">
+ More
+ <svg fill="none" stroke="currentColor" viewBox="0 0 24 24"
aria-hidden="true">
+ <path stroke-linecap="round" stroke-linejoin="round"
stroke-width="2" d="M19 9l-7 7-7-7" />
+ </svg>
+ </button>
+ <div class="module-tab-more-menu" id="module-tab-more-menu" hidden>
+ {% for t in overflowTypes %}
+ <button class="module-tab" data-type="{{ t.id }}">
+ <span class="tab-icon {{ t.id }}">{{ t.icon }}</span> {{ t.label }}
({{ t.count }})
+ </button>
+ {% endfor %}
+ </div>
+ </div>
+ {% endif %}
</nav>
{# Main content with sidebar #}
@@ -352,7 +371,7 @@ eleventyComputed:
{% for module in providerModules %}
<div id="{{ module.id or module.name }}" class="module card"
data-name="{{ module.name | lower }}" data-type="{{ module.type }}"
data-category="{{ module.category }}">
<div class="icon {{ module.type }}">
- <span>{{ module.type[0] | upper }}</span>
+ <span>{{ types | typeIcon(module.type) }}</span>
</div>
<div class="content">
<h3>
