This is an automated email from the ASF dual-hosted git repository.
kaxil pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 16813192320 Show provider URI schemes in the Provider Registry (#74284)
16813192320 is described below
commit 16813192320069263718645a62a4fb48f64c13dc
Author: Kaxil Naik <[email protected]>
AuthorDate: Tue Oct 6 12:52:54 2026 +0100
Show provider URI schemes in the Provider Registry (#74284)
The registry ignored the filesystems, asset-uris and remote-logging
sections of provider.yaml, so a provider page never said which URI
schemes the provider handles (for example, that microsoft-azure serves
sharepoint:// and onedrive:// paths through ObjectStoragePath).
Collect one entry per scheme from those three sections, expose it as
uri_schemes in providers.json and per-version metadata, and render a
URI Schemes table on the provider page.
A scheme-by-feature matrix showed a dash wherever a provider registers
no handler, which reads as missing support even when the feature does
not apply (redshift:// is never an object store). List the schemes
under each feature instead, so only what is registered is shown.
---
dev/registry/extract_metadata.py | 15 ++++
dev/registry/extract_versions.py | 6 ++
dev/registry/registry_contract_models.py | 34 ++++++++
dev/registry/registry_tools/uri_schemes.py | 98 +++++++++++++++++++++
dev/registry/tests/test_extract_metadata.py | 118 +++++++++++++++++--------
dev/registry/tests/test_extract_versions.py | 56 ++++++++++++
dev/registry/tests/test_uri_schemes.py | 128 ++++++++++++++++++++++++++++
registry/.eleventy.js | 37 ++++++++
registry/src/css/main.css | 77 +++++++++++++++++
registry/src/provider-version.njk | 28 ++++++
10 files changed, 563 insertions(+), 34 deletions(-)
diff --git a/dev/registry/extract_metadata.py b/dev/registry/extract_metadata.py
index 77d34d8397d..35ea9bf3628 100644
--- a/dev/registry/extract_metadata.py
+++ b/dev/registry/extract_metadata.py
@@ -51,6 +51,7 @@ else: # pragma: no cover -- Python 3.10 fallback
import yaml
from registry_contract_models import validate_providers_catalog
+from registry_tools.uri_schemes import ReadModuleSource, collect_uri_schemes
# External endpoints used by metadata extraction.
PYPISTATS_RECENT_URL =
"https://pypistats.org/api/packages/{package_name}/recent";
@@ -376,6 +377,7 @@ class Provider:
connection_types: list[dict] = field(
default_factory=list
) # {conn_type, hook_class, docs_url, external_services}
+ uri_schemes: list[dict] = field(default_factory=list) # {scheme,
filesystem?, asset?, remote_logging?}
requires_python: str = "" # e.g., ">=3.10"
dependencies: list[str] = field(default_factory=list) # from
pyproject.toml
optional_extras: dict[str, list[str]] = field(default_factory=dict) #
{extra_name: [deps]}
@@ -453,6 +455,18 @@ def module_path_to_file_path(module_path: str,
provider_path: Path) -> Path:
return file_path.with_suffix(".py")
+def module_source_reader(provider_path: Path) -> ReadModuleSource:
+ """Return a reader for module sources in the provider's working-tree
``src/`` directory."""
+
+ def read(module_path: str) -> str | None:
+ try:
+ return module_path_to_file_path(module_path,
provider_path).read_text()
+ except FileNotFoundError:
+ return None
+
+ return read
+
+
def determine_airflow_versions(dependencies: list[str]) -> list[str]:
"""Determine minimum Airflow version from pyproject.toml dependencies."""
for dep in dependencies:
@@ -823,6 +837,7 @@ def main():
pypi_downloads=pypi_downloads,
categories=[asdict(c) for c in categories],
connection_types=connection_types,
+ uri_schemes=collect_uri_schemes(provider_yaml,
module_source_reader(provider_path)),
requires_python=pyproject_data["requires_python"],
dependencies=pyproject_data["dependencies"],
optional_extras=pyproject_data.get("optional_extras", {}),
diff --git a/dev/registry/extract_versions.py b/dev/registry/extract_versions.py
index 064fd4a8f86..d831725f1c3 100644
--- a/dev/registry/extract_versions.py
+++ b/dev/registry/extract_versions.py
@@ -51,6 +51,7 @@ if sys.version_info >= (3, 11):
else: # pragma: no cover -- Python 3.10 fallback
import tomli as tomllib
from registry_contract_models import validate_provider_version_metadata
+from registry_tools.uri_schemes import collect_uri_schemes
try:
import yaml
@@ -453,6 +454,10 @@ def extract_version_data(
}
)
+ uri_schemes = collect_uri_schemes(
+ provider_yaml, lambda module_path: git_show(tag,
get_source_file_path(layout, dir_path, module_path))
+ )
+
# Extract modules from source files
modules = extract_modules_from_yaml(provider_yaml, tag, layout, dir_path,
provider_id, version)
module_counts = count_modules(modules)
@@ -466,6 +471,7 @@ def extract_version_data(
"dependencies": pyproject_data["dependencies"],
"optional_extras": pyproject_data["optional_extras"],
"connection_types": connection_types,
+ "uri_schemes": uri_schemes,
"module_counts": module_counts,
"modules": modules,
}
diff --git a/dev/registry/registry_contract_models.py
b/dev/registry/registry_contract_models.py
index 116b336d990..31f4001dc49 100644
--- a/dev/registry/registry_contract_models.py
+++ b/dev/registry/registry_contract_models.py
@@ -55,6 +55,38 @@ class ConnectionTypeContract(BaseModel):
external_services: list[str] = Field(default_factory=list)
+class AssetUriContract(BaseModel):
+ """Dotted paths an asset-uris entry registers for a scheme."""
+
+ model_config = ConfigDict(extra="forbid")
+
+ handler: str | None = Field(
+ default=None,
+ description="URI normalizer. null registers Airflow's no-op
normalizer; the scheme is still recognized.",
+ )
+ factory: str | None = Field(default=None, description="Asset factory. null
means none is registered.")
+ to_openlineage_converter: str | None = Field(
+ default=None, description="OpenLineage converter. null means none is
registered."
+ )
+
+
+class UriSchemeContract(BaseModel):
+ """What a provider registers for one URI scheme."""
+
+ model_config = ConfigDict(extra="forbid")
+
+ scheme: str
+ filesystem: str | None = Field(
+ default=None, description="Module exposing get_fs for
ObjectStoragePath paths with this scheme."
+ )
+ asset: AssetUriContract | None = Field(
+ default=None, description="Present when the provider registers Asset
URI handling for this scheme."
+ )
+ remote_logging: str | None = Field(
+ default=None, description="RemoteLogIO class used when
remote_base_log_folder has this scheme."
+ )
+
+
class ProviderContract(BaseModel):
"""Top-level provider entry in providers.json."""
@@ -73,6 +105,7 @@ class ProviderContract(BaseModel):
module_counts: dict[str, int] = Field(default_factory=dict)
categories: list[CategoryContract] = Field(default_factory=list)
connection_types: list[ConnectionTypeContract] =
Field(default_factory=list)
+ uri_schemes: list[UriSchemeContract] = Field(default_factory=list)
requires_python: str = ""
dependencies: list[str] = Field(default_factory=list)
optional_extras: dict[str, list[str]] = Field(default_factory=dict)
@@ -213,6 +246,7 @@ class ProviderVersionMetadataContract(BaseModel):
dependencies: list[str]
optional_extras: dict[str, list[str]]
connection_types: list[ConnectionTypeContract]
+ uri_schemes: list[UriSchemeContract] = Field(default_factory=list)
module_counts: dict[str, int]
modules: list[ModuleContract]
diff --git a/dev/registry/registry_tools/uri_schemes.py
b/dev/registry/registry_tools/uri_schemes.py
new file mode 100644
index 00000000000..95e65734a9e
--- /dev/null
+++ b/dev/registry/registry_tools/uri_schemes.py
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""URI schemes a provider registers, collected from its provider.yaml.
+
+Airflow resolves three kinds of provider handler by URI scheme at runtime:
+
+- ``filesystems``: modules exposing ``get_fs``, which ``airflow.sdk.io.fs``
+ maps to ``ObjectStoragePath`` schemes. The schemes are not in provider.yaml;
+ each module declares them in a module-level ``schemes`` list, read here with
AST.
+- ``asset-uris``: URI normalizers, Asset factories and OpenLineage converters.
+- ``remote-logging``: ``RemoteLogIO`` classes claiming a scheme of
+ ``[logging] remote_base_log_folder``.
+
+Shared by extract_metadata.py (reads sources from the working tree) and
+extract_versions.py (reads sources from a release tag).
+"""
+
+from __future__ import annotations
+
+import ast
+from collections.abc import Callable
+from typing import Any
+
+ReadModuleSource = Callable[[str], str | None]
+
+
+def read_filesystem_schemes(source: str) -> list[str]:
+ """Return the module-level ``schemes`` list declared by a filesystem
module."""
+ try:
+ for node in ast.parse(source).body:
+ if isinstance(node, ast.Assign):
+ targets, value = node.targets, node.value
+ elif isinstance(node, ast.AnnAssign) and node.value is not None:
+ targets, value = [node.target], node.value
+ else:
+ continue
+ if any(isinstance(target, ast.Name) and target.id == "schemes" for
target in targets):
+ return list(ast.literal_eval(value))
+ except (SyntaxError, ValueError) as e:
+ print(f" Warning: could not read filesystem schemes: {e}")
+ return []
+
+
+def collect_uri_schemes(
+ provider_yaml: dict[str, Any], read_module_source: ReadModuleSource
+) -> list[dict[str, Any]]:
+ """
+ Return one entry per URI scheme the provider registers, sorted by scheme.
+
+ Each entry has a ``scheme`` key plus whichever of ``filesystem`` (module
+ path), ``asset`` (handler, factory and OpenLineage converter paths) and
+ ``remote_logging`` (``RemoteLogIO`` class path) the provider registers for
it.
+
+ :param provider_yaml: Parsed provider.yaml.
+ :param read_module_source: Returns the source of a dotted module path, or
+ ``None`` when the file is missing.
+ """
+ by_scheme: dict[str, dict[str, Any]] = {}
+
+ def entry(scheme: str) -> dict[str, Any]:
+ return by_scheme.setdefault(scheme, {"scheme": scheme})
+
+ for module_path in provider_yaml.get("filesystems", []):
+ if (source := read_module_source(module_path)) is None:
+ continue
+ for scheme in read_filesystem_schemes(source):
+ entry(scheme)["filesystem"] = module_path
+
+ for spec in provider_yaml.get("asset-uris", []):
+ # ProvidersManager skips entries without a handler key; an explicit
null
+ # handler still registers the scheme, with a no-op normalizer.
+ if "handler" not in spec:
+ continue
+ for scheme in spec.get("schemes", []):
+ entry(scheme)["asset"] = {
+ "handler": spec["handler"],
+ "factory": spec.get("factory"),
+ "to_openlineage_converter":
spec.get("to_openlineage_converter"),
+ }
+
+ for spec in provider_yaml.get("remote-logging", []):
+ entry(spec["scheme"])["remote_logging"] = spec["classpath"]
+
+ return [by_scheme[scheme] for scheme in sorted(by_scheme)]
diff --git a/dev/registry/tests/test_extract_metadata.py
b/dev/registry/tests/test_extract_metadata.py
index faf17f6f9c8..49a7bebaba0 100644
--- a/dev/registry/tests/test_extract_metadata.py
+++ b/dev/registry/tests/test_extract_metadata.py
@@ -833,29 +833,61 @@ class TestVersionsListFiltering:
# ---------------------------------------------------------------------------
-# main() -- connection-types `external-services` propagation
-# ---------------------------------------------------------------------------
+# main() -- provider.yaml fields reaching providers.json
+# ---------------------------------------------------------------------------
+@patch("extract_metadata.fetch_provider_inventory", autospec=True,
return_value=None)
+@patch("extract_metadata.fetch_pypi_data_parallel", autospec=True,
return_value={})
+@patch("extract_metadata.load_release_tags", autospec=True, return_value=set())
+def _run_main_for_provider(
+ _load_release_tags,
+ _fetch_pypi_data_parallel,
+ _fetch_provider_inventory,
+ tmp_path,
+ provider_yaml,
+ src_files,
+):
+ """Run main() on one provider under tmp_path and return its providers.json
entry.
+
+ Network and filesystem dependencies are mocked or redirected, so the
+ written entry has also passed main()'s ProviderContract (extra="forbid")
+ validation, catching key-name drift between provider.yaml, main() and
+ registry_contract_models.py.
+ """
+ providers_dir = tmp_path / "providers"
+ provider_dir = providers_dir / "testprov"
+ provider_dir.mkdir(parents=True)
+ (provider_dir / "provider.yaml").write_text(provider_yaml)
+ for rel_path, content in src_files.items():
+ src_file = provider_dir / "src" / rel_path
+ src_file.parent.mkdir(parents=True, exist_ok=True)
+ src_file.write_text(content)
+ output_dir = tmp_path / "output"
+ script_dir = tmp_path / "script"
+ output_dir.mkdir()
+ script_dir.mkdir()
+
+ with (
+ patch("extract_metadata.PROVIDERS_DIR", providers_dir),
+ patch("extract_metadata.OUTPUT_DIR", output_dir),
+ patch("extract_metadata.SCRIPT_DIR", script_dir),
+ patch.object(sys, "argv", ["extract_metadata.py"]),
+ ):
+ main()
+
+ written = json.loads((output_dir / "providers.json").read_text())
+ return next(p for p in written["providers"] if p["id"] == "testprov")
+
+
class TestMainConnectionTypesExternalServices:
"""The connection-types extraction loop lives inline in main() rather than
- a standalone function, so this drives main() end-to-end (with network and
- filesystem dependencies mocked/redirected) to prove `external-services`
- from provider.yaml reaches the written providers.json, and round-trips
- through the `ConnectionTypeContract` (extra="forbid") validation main()
- already runs -- catching a key-name drift between provider.yaml, this
- script, and registry_contract_models.py.
+ a standalone function, so this drives main() end-to-end to prove
+ `external-services` from provider.yaml reaches the written providers.json.
"""
- @patch("extract_metadata.fetch_provider_inventory", autospec=True,
return_value=None)
- @patch("extract_metadata.fetch_pypi_data_parallel", autospec=True,
return_value={})
- @patch("extract_metadata.load_release_tags", autospec=True,
return_value=set())
- def test_external_services_propagates_to_providers_json(
- self, _load_release_tags, _fetch_pypi_data_parallel,
_fetch_provider_inventory, tmp_path
- ):
- providers_dir = tmp_path / "providers"
- provider_dir = providers_dir / "testprov"
- provider_dir.mkdir(parents=True)
- (provider_dir / "provider.yaml").write_text(
- textwrap.dedent("""\
+ def test_external_services_propagates_to_providers_json(self, tmp_path):
+ provider = _run_main_for_provider(
+ tmp_path=tmp_path,
+ provider_yaml=textwrap.dedent("""\
name: Test Provider
description: A test provider.
versions:
@@ -866,21 +898,39 @@ class TestMainConnectionTypesExternalServices:
external-services:
- openai
- anthropic
- """)
+ """),
+ src_files={},
)
- output_dir = tmp_path / "output"
- script_dir = tmp_path / "script"
- output_dir.mkdir()
- script_dir.mkdir()
-
- with (
- patch("extract_metadata.PROVIDERS_DIR", providers_dir),
- patch("extract_metadata.OUTPUT_DIR", output_dir),
- patch("extract_metadata.SCRIPT_DIR", script_dir),
- patch.object(sys, "argv", ["extract_metadata.py"]),
- ):
- main()
- written = json.loads((output_dir / "providers.json").read_text())
- provider = next(p for p in written["providers"] if p["id"] ==
"testprov")
assert provider["connection_types"][0]["external_services"] ==
["openai", "anthropic"]
+
+
+class TestMainUriSchemes:
+ """main() must read filesystem schemes from the provider's src/ tree, skip
a
+ listed module whose file is missing, and write the entries to
providers.json."""
+
+ def test_uri_schemes_propagate_to_providers_json(self, tmp_path):
+ provider = _run_main_for_provider(
+ tmp_path=tmp_path,
+ provider_yaml=textwrap.dedent("""\
+ name: Test Provider
+ description: A test provider.
+ versions:
+ - 1.0.0
+ filesystems:
+ - airflow.providers.testprov.fs.testfs
+ - airflow.providers.testprov.fs.missing
+ asset-uris:
+ - schemes: [testfs]
+ handler: null
+ """),
+ src_files={"airflow/providers/testprov/fs/testfs.py": 'schemes =
["testfs"]\n'},
+ )
+
+ assert provider["uri_schemes"] == [
+ {
+ "scheme": "testfs",
+ "filesystem": "airflow.providers.testprov.fs.testfs",
+ "asset": {"handler": None, "factory": None,
"to_openlineage_converter": None},
+ }
+ ]
diff --git a/dev/registry/tests/test_extract_versions.py
b/dev/registry/tests/test_extract_versions.py
index 061424eb918..6a3128ce01a 100644
--- a/dev/registry/tests/test_extract_versions.py
+++ b/dev/registry/tests/test_extract_versions.py
@@ -210,3 +210,59 @@ class TestExtractVersionDataConnectionTypes:
assert result is not None
assert result["connection_types"][0]["external_services"] ==
["openai", "anthropic"]
+
+
+class TestExtractVersionDataUriSchemes:
+ """A superseded release's page reads URI schemes from its metadata.json,
so the
+ filesystem module source must come from the release tag, not the working
tree."""
+
+ PROVIDER_YAML = textwrap.dedent("""\
+ name: Test Provider
+ filesystems:
+ - airflow.providers.test.fs.testfs
+ remote-logging:
+ - classpath: airflow.providers.test.log.TestRemoteLogIO
+ scheme: testfs
+ """)
+
+ @pytest.mark.parametrize(
+ ("layout", "yaml_path", "fs_source_path"),
+ [
+ pytest.param(
+ "new",
+ "providers/test/provider.yaml",
+ "providers/test/src/airflow/providers/test/fs/testfs.py",
+ id="new-layout",
+ ),
+ pytest.param(
+ "old",
+ "providers/src/airflow/providers/test/provider.yaml",
+ "providers/src/airflow/providers/test/fs/testfs.py",
+ id="old-layout",
+ ),
+ ],
+ )
+ @patch("extract_versions.extract_modules_from_yaml", autospec=True,
return_value=[])
+ @patch("extract_versions.fetch_provider_inventory", autospec=True,
return_value=None)
+ @patch("extract_versions.git_show", autospec=True)
+ @patch("extract_versions.detect_layout", autospec=True)
+ @patch("extract_versions.git_tag_exists", autospec=True, return_value=True)
+ def test_filesystem_schemes_read_from_release_tag(
+ self, _tag_exists, mock_layout, mock_git_show, _inventory, _modules,
layout, yaml_path, fs_source_path
+ ):
+ mock_layout.return_value = layout
+ sources = {yaml_path: self.PROVIDER_YAML, fs_source_path: 'schemes =
["testfs", "tfs"]\n'}
+ mock_git_show.side_effect = lambda tag, path: sources.get(path)
+
+ result = extract_version_data("test", "1.0.0", "test")
+
+ assert result is not None
+ assert result["uri_schemes"] == [
+ {
+ "scheme": "testfs",
+ "filesystem": "airflow.providers.test.fs.testfs",
+ "remote_logging": "airflow.providers.test.log.TestRemoteLogIO",
+ },
+ {"scheme": "tfs", "filesystem":
"airflow.providers.test.fs.testfs"},
+ ]
+ mock_git_show.assert_any_call("providers-test/1.0.0", fs_source_path)
diff --git a/dev/registry/tests/test_uri_schemes.py
b/dev/registry/tests/test_uri_schemes.py
new file mode 100644
index 00000000000..80bd402fe88
--- /dev/null
+++ b/dev/registry/tests/test_uri_schemes.py
@@ -0,0 +1,128 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Unit tests for dev/registry/registry_tools/uri_schemes.py."""
+
+from __future__ import annotations
+
+import textwrap
+
+import pytest
+from registry_contract_models import UriSchemeContract
+from registry_tools.uri_schemes import collect_uri_schemes,
read_filesystem_schemes
+
+S3_FS_SOURCE = textwrap.dedent("""\
+ from __future__ import annotations
+
+ import logging
+
+ S3_PROXY_URI = "proxy-uri"
+ log = logging.getLogger(__name__)
+
+ schemes = ["s3", "s3a"]
+
+
+ def get_fs(conn_id, storage_options=None):
+ ...
+ """)
+
+S3_ASSET = {
+ "handler": "airflow.providers.amazon.aws.assets.s3.sanitize_uri",
+ "factory": "airflow.providers.amazon.aws.assets.s3.create_asset",
+ "to_openlineage_converter":
"airflow.providers.amazon.aws.assets.s3.convert_asset_to_openlineage",
+}
+
+
+class TestReadFilesystemSchemes:
+ def test_reads_module_level_schemes_list(self):
+ assert read_filesystem_schemes(S3_FS_SOURCE) == ["s3", "s3a"]
+
+ def test_reads_annotated_schemes_list(self):
+ assert read_filesystem_schemes('schemes: list[str] = ["gs", "gcs"]\n')
== ["gs", "gcs"]
+
+ @pytest.mark.parametrize(
+ "source",
+ [
+ pytest.param("def get_fs():\n schemes = ['local']\n",
id="bound-inside-function"),
+ pytest.param('schemes = [*BASE_SCHEMES, "s3"]\n',
id="non-literal"),
+ pytest.param("schemes = [\n", id="syntax-error"),
+ ],
+ )
+ def test_returns_empty_when_no_literal_module_level_schemes(self, source):
+ assert read_filesystem_schemes(source) == []
+
+
+class TestCollectUriSchemes:
+ def test_merges_sections_into_one_entry_per_scheme_sorted(self):
+ provider_yaml = {
+ "filesystems": ["airflow.providers.amazon.aws.fs.s3"],
+ "asset-uris": [{"schemes": ["s3"], **S3_ASSET}],
+ "remote-logging": [
+ {
+ "classpath":
"airflow.providers.amazon.aws.log.s3_task_handler.S3RemoteLogIO",
+ "scheme": "s3",
+ },
+ {
+ "classpath":
"airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudWatchRemoteLogIO",
+ "scheme": "cloudwatch",
+ },
+ ],
+ }
+
+ result = collect_uri_schemes(provider_yaml,
{"airflow.providers.amazon.aws.fs.s3": S3_FS_SOURCE}.get)
+
+ assert result == [
+ {
+ "scheme": "cloudwatch",
+ "remote_logging":
"airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudWatchRemoteLogIO",
+ },
+ {
+ "scheme": "s3",
+ "filesystem": "airflow.providers.amazon.aws.fs.s3",
+ "asset": S3_ASSET,
+ "remote_logging":
"airflow.providers.amazon.aws.log.s3_task_handler.S3RemoteLogIO",
+ },
+ {"scheme": "s3a", "filesystem":
"airflow.providers.amazon.aws.fs.s3"},
+ ]
+ for entry in result:
+ UriSchemeContract.model_validate(entry)
+
+ def test_asset_scheme_with_null_handler_is_still_registered(self):
+ result = collect_uri_schemes({"asset-uris": [{"schemes": ["gcp"],
"handler": None}]}, {}.get)
+
+ assert result == [
+ {"scheme": "gcp", "asset": {"handler": None, "factory": None,
"to_openlineage_converter": None}}
+ ]
+
+ def test_skips_asset_entry_without_handler_key(self):
+ result = collect_uri_schemes(
+ {
+ "asset-uris": [
+ {"schemes": ["s3"], "factory":
"airflow.providers.amazon.aws.assets.s3.create_asset"}
+ ]
+ },
+ {}.get,
+ )
+
+ assert result == []
+
+ def test_skips_filesystem_module_whose_source_is_missing(self):
+ result = collect_uri_schemes({"filesystems":
["airflow.providers.gone.fs.gone"]}, {}.get)
+
+ assert result == []
+
+ def test_provider_without_scheme_sections_returns_empty(self):
+ assert collect_uri_schemes({"name": "No Schemes", "hooks": []},
{}.get) == []
diff --git a/registry/.eleventy.js b/registry/.eleventy.js
index 83496beb4f5..bf2e04927dd 100644
--- a/registry/.eleventy.js
+++ b/registry/.eleventy.js
@@ -131,6 +131,43 @@ module.exports = function(eleventyConfig) {
return collectExternalServices(provider);
});
+ // Groups a provider's uri_schemes by the Airflow feature that routes them to
+ // the provider. Listing only registered schemes per feature avoids a matrix
+ // whose empty cells read as "unsupported" when the feature just doesn't
apply
+ // (redshift:// is never an object store). Features with no schemes are
dropped.
+ eleventyConfig.addFilter("uriSchemeGroups", (uriSchemes) => {
+ const docs = "https://airflow.apache.org/docs/apache-airflow/stable";;
+ const features = [
+ {
+ label: "ObjectStoragePath",
+ docsUrl: `${docs}/core-concepts/objectstorage.html`,
+ detail: (s) => s.filesystem,
+ },
+ {
+ label: "Asset URIs",
+ docsUrl:
`${docs}/authoring-and-scheduling/assets.html#what-is-valid-uri`,
+ detail: (s) =>
+ s.asset &&
+ ([s.asset.handler, s.asset.factory,
s.asset.to_openlineage_converter].filter(Boolean).join(", ") ||
+ "No-op handler"),
+ },
+ {
+ label: "Remote logging",
+ docsUrl:
`${docs}/administration-and-deployment/logging-monitoring/logging-tasks.html`,
+ detail: (s) => s.remote_logging,
+ },
+ ];
+ return features
+ .map(({ label, docsUrl, detail }) => ({
+ label,
+ docsUrl,
+ schemes: (uriSchemes || [])
+ .filter((s) => detail(s))
+ .map((s) => ({ scheme: s.scheme, detail: detail(s) })),
+ }))
+ .filter((group) => group.schemes.length > 0);
+ });
+
eleventyConfig.addShortcode("year", () => `${new Date().getFullYear()}`);
return {
diff --git a/registry/src/css/main.css b/registry/src/css/main.css
index 14d7fd7d4f5..d5077f8c888 100644
--- a/registry/src/css/main.css
+++ b/registry/src/css/main.css
@@ -4071,6 +4071,83 @@ main {
height: 1rem;
}
+/* URI Schemes Card */
+.provider-detail-page .uri-schemes {
+ padding: var(--space-5);
+ margin-bottom: var(--space-6);
+}
+
+.provider-detail-page .uri-schemes > h3 {
+ font-size: var(--text-sm);
+ font-weight: var(--font-semibold);
+ color: var(--text-primary);
+ margin-bottom: var(--space-2);
+ display: flex;
+ align-items: center;
+ gap: var(--space-2);
+}
+
+.provider-detail-page .uri-schemes > h3 svg {
+ width: 1rem;
+ height: 1rem;
+ color: var(--color-blue-400);
+}
+
+.provider-detail-page .uri-schemes > p {
+ font-size: var(--text-sm);
+ color: var(--text-secondary);
+ margin-bottom: var(--space-3);
+}
+
+.provider-detail-page .uri-schemes dl {
+ display: grid;
+ gap: var(--space-3);
+}
+
+.provider-detail-page .uri-schemes dl > div {
+ display: grid;
+ gap: var(--space-2);
+}
+
+@media (min-width: 640px) {
+ .provider-detail-page .uri-schemes dl > div {
+ grid-template-columns: 10rem 1fr;
+ align-items: baseline;
+ }
+}
+
+.provider-detail-page .uri-schemes dt {
+ font-size: var(--text-sm);
+ font-weight: var(--font-semibold);
+ color: var(--text-secondary);
+}
+
+.provider-detail-page .uri-schemes dt a {
+ color: inherit;
+ text-decoration: underline dotted;
+ text-underline-offset: 3px;
+}
+
+.provider-detail-page .uri-schemes dt a:hover {
+ color: var(--accent-primary);
+}
+
+.provider-detail-page .uri-schemes dd {
+ display: flex;
+ flex-wrap: wrap;
+ gap: var(--space-2);
+}
+
+.provider-detail-page .uri-schemes dd code {
+ padding: var(--space-1) var(--space-3);
+ font-family: var(--font-mono);
+ font-size: var(--text-sm);
+ color: var(--color-blue-400);
+ background: rgb(from var(--color-blue-500) r g b / 0.05);
+ border: 1px solid rgb(from var(--color-blue-500) r g b / 0.2);
+ border-radius: var(--radius-full);
+}
+
/* Search Modal */
#search-modal {
display: none;
diff --git a/registry/src/provider-version.njk
b/registry/src/provider-version.njk
index 4d8c759e322..e28be209f1a 100644
--- a/registry/src/provider-version.njk
+++ b/registry/src/provider-version.njk
@@ -18,6 +18,7 @@ eleventyComputed:
{% set deps = pv.provider.dependencies or [] %}
{% set extras = pv.provider.optional_extras or {} %}
{% set conns = pv.provider.connection_types or [] %}
+ {% set uriSchemes = pv.provider.uri_schemes or [] %}
{% set providerModules = pv.modules %}
{% set currentVersion = pv.provider.version %}
{% set pythonVersion = pv.provider.requires_python %}
@@ -28,6 +29,7 @@ eleventyComputed:
{% set deps = pv.versionData.dependencies or [] %}
{% set extras = pv.versionData.optional_extras or {} %}
{% set conns = pv.versionData.connection_types or [] %}
+ {% set uriSchemes = pv.versionData.uri_schemes or [] %}
{% set providerModules = pv.versionData.modules or [] %}
{% set currentVersion = pv.version %}
{% set pythonVersion = pv.versionData.requires_python %}
@@ -306,6 +308,32 @@ eleventyComputed:
{% endif %}
</section>
+ {# URI schemes grouped by the Airflow feature that routes them to the
provider #}
+ {% set schemeGroups = uriSchemes | uriSchemeGroups %}
+ {% if schemeGroups.length > 0 %}
+ <section class="uri-schemes card">
+ <h3>
+ <svg fill="none" stroke="currentColor" viewBox="0 0 24 24"
aria-hidden="true">
+ <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M21 12a9 9 0 01-9 9m9-9a9 9 0 00-9-9m9 9H3m9 9a9 9 0 01-9-9m9 9c1.657 0
3-4.03 3-9s-1.343-9-3-9m0 18c-1.657 0-3-4.03-3-9s1.343-9 3-9m-9 9a9 9 0 019-9"
/>
+ </svg>
+ URI Schemes ({{ uriSchemes.length }})
+ </h3>
+ <p>Schemes this release registers in provider.yaml, grouped by the Airflow
feature that routes them to the provider.</p>
+ <dl>
+ {% for group in schemeGroups %}
+ <div>
+ <dt><a href="{{ group.docsUrl }}" target="_blank" rel="noopener">{{
group.label }}</a></dt>
+ <dd>
+ {% for s in group.schemes %}
+ <code title="{{ s.detail }}">{{ s.scheme }}://</code>
+ {% endfor %}
+ </dd>
+ </div>
+ {% endfor %}
+ </dl>
+ </section>
+ {% endif %}
+
{% if totalModules > 0 %}
{# Module Type Tabs: sorted by module count (busiest first), overflow into a
"More" menu #}
{% set moduleTabsVisibleCount = 8 %}
