This is an automated email from the ASF dual-hosted git repository.
Lee-W pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new fbac2f5ce19 Add a Retry Policies category to the provider registry
(#70499)
fbac2f5ce19 is described below
commit fbac2f5ce19f9273974af76031d3e04854b3694f
Author: Wei Lee <[email protected]>
AuthorDate: Fri Jul 31 14:41:00 2026 +0800
Add a Retry Policies category to the provider registry (#70499)
---
airflow-core/src/airflow/provider.yaml.schema.json | 24 +++++++++++
dev/registry/registry_tools/types.py | 8 ++++
providers/common/ai/provider.yaml | 5 +++
registry/src/_data/types.json | 5 +++
registry/src/css/main.css | 1 +
registry/src/css/tokens.css | 1 +
.../in_container/run_provider_yaml_files_check.py | 48 +++++++++++++++++++---
7 files changed, 86 insertions(+), 6 deletions(-)
diff --git a/airflow-core/src/airflow/provider.yaml.schema.json
b/airflow-core/src/airflow/provider.yaml.schema.json
index 4f3b53f1077..0a700aa88d5 100644
--- a/airflow-core/src/airflow/provider.yaml.schema.json
+++ b/airflow-core/src/airflow/provider.yaml.schema.json
@@ -397,6 +397,30 @@
]
}
},
+ "retry-policies": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "integration-name": {
+ "type": "string",
+ "description": "Integration name. It must have a
matching item in the 'integration' section of any provider."
+ },
+ "python-modules": {
+ "description": "List of python modules containing the
retry policies.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "integration-name",
+ "python-modules"
+ ]
+ }
+ },
"connection-types": {
"type": "array",
"description": "Array of connection types mapped to hook class
names",
diff --git a/dev/registry/registry_tools/types.py
b/dev/registry/registry_tools/types.py
index 859e70da1d5..cd91b70ed65 100644
--- a/dev/registry/registry_tools/types.py
+++ b/dev/registry/registry_tools/types.py
@@ -109,6 +109,13 @@ MODULE_TYPES: dict[str, dict] = {
"label": "Toolsets",
"icon": "U",
},
+ "retry_policy": {
+ "yaml_key": "retry-policies",
+ "level": "module",
+ "suffixes": ["RetryPolicy"],
+ "label": "Retry Policies",
+ "icon": "R",
+ },
}
# Runtime base class imports for issubclass checks (extract_parameters.py).
@@ -121,6 +128,7 @@ BASE_CLASS_IMPORTS: list[tuple[str, str]] = [
("bundle", "airflow.dag_processing.bundles.base.BaseDagBundle"),
("operator", "airflow.sdk.bases.operator.BaseOperator"),
("toolset", "pydantic_ai.toolsets.abstract.AbstractToolset"),
+ ("retry_policy", "airflow.sdk.definitions.retry_policy.RetryPolicy"),
]
# Derived lookups used by extraction scripts.
diff --git a/providers/common/ai/provider.yaml
b/providers/common/ai/provider.yaml
index 7431bcde2c6..62c9eeaa025 100644
--- a/providers/common/ai/provider.yaml
+++ b/providers/common/ai/provider.yaml
@@ -463,6 +463,11 @@ toolsets:
- airflow.providers.common.ai.toolsets.skills
- airflow.providers.common.ai.toolsets.langchain_bridge
+retry-policies:
+ - integration-name: Common AI
+ python-modules:
+ - airflow.providers.common.ai.policies.retry
+
task-decorators:
- class-name: airflow.providers.common.ai.decorators.agent.agent_task
name: agent
diff --git a/registry/src/_data/types.json b/registry/src/_data/types.json
index 52e5657d499..150f986b2e4 100644
--- a/registry/src/_data/types.json
+++ b/registry/src/_data/types.json
@@ -58,5 +58,10 @@
"id": "toolset",
"label": "Toolsets",
"icon": "U"
+ },
+ {
+ "id": "retry_policy",
+ "label": "Retry Policies",
+ "icon": "R"
}
]
diff --git a/registry/src/css/main.css b/registry/src/css/main.css
index c733a4dd5f1..8a262428b55 100644
--- a/registry/src/css/main.css
+++ b/registry/src/css/main.css
@@ -3066,6 +3066,7 @@ main {
.tab-icon.bundle { background: rgb(from var(--color-bundle) r g b / 0.2);
color: var(--color-bundle); }
.tab-icon.decorator { background: rgb(from var(--color-decorator) r g b /
0.2); color: var(--color-decorator); }
.tab-icon.toolset { background: rgb(from var(--color-toolset) r g b / 0.2);
color: var(--color-toolset); }
+.tab-icon.retry_policy { background: rgb(from var(--color-retry_policy) r g b
/ 0.2); color: var(--color-retry_policy); }
/* Modules Layout (sidebar + content) */
.modules-layout {
diff --git a/registry/src/css/tokens.css b/registry/src/css/tokens.css
index 30f885b4927..74a6c039e00 100644
--- a/registry/src/css/tokens.css
+++ b/registry/src/css/tokens.css
@@ -66,6 +66,7 @@
--color-bundle: #0ea5e9;
--color-decorator: #d946ef;
--color-toolset: #84cc16;
+ --color-retry_policy: #10b981;
/* Additional Colors */
--color-green-400: #4ade80;
diff --git a/scripts/in_container/run_provider_yaml_files_check.py
b/scripts/in_container/run_provider_yaml_files_check.py
index cefdaea8ad5..eb6f4421384 100755
--- a/scripts/in_container/run_provider_yaml_files_check.py
+++ b/scripts/in_container/run_provider_yaml_files_check.py
@@ -418,14 +418,18 @@ def
check_correctness_of_list_of_sensors_operators_hook_trigger_modules(
return num_modules, num_errors
-@run_check("Checking for duplicates in list of {sensors, hooks, operators,
triggers, bundles, toolsets}")
+@run_check(
+ "Checking for duplicates in list of "
+ "{sensors, hooks, operators, triggers, bundles, toolsets, retry-policies}"
+)
def check_duplicates_in_integrations_names_of_hooks_sensors_operators(
yaml_files: dict[str, dict],
) -> tuple[int, int]:
num_errors = 0
num_integrations = 0
for (yaml_file_path, provider_data), resource_type in itertools.product(
- yaml_files.items(), ["sensors", "operators", "hooks", "triggers",
"bundles", "toolsets"]
+ yaml_files.items(),
+ ["sensors", "operators", "hooks", "triggers", "bundles", "toolsets",
"retry-policies"],
):
resource_data = provider_data.get(resource_type, [])
count_integrations = Counter(r.get("integration-name", "") for r in
resource_data)
@@ -489,6 +493,26 @@ def
check_hook_class_name_entries_in_connection_types(yaml_files: dict[str, dict
return num_connection_types, num_errors
+@run_check("Checking that python-modules in retry-policies exist and belong to
provider package")
+def check_retry_policy_modules_exist_and_belong_to_package(yaml_files:
dict[str, dict]) -> tuple[int, int]:
+ # parse_module_data's glob-based completeness check assumes a category's
modules live in a
+ # directory named after its yaml key. Retry policies intentionally live
under policies/ instead,
+ # so that check cannot be reused and this existence-only check stands in
for it.
+ resource_type = "retry-policies"
+ num_errors = 0
+ num_retry_policy_modules = 0
+ for yaml_file_path, provider_data in yaml_files.items():
+ provider_package = _filepath_to_module(yaml_file_path)
+ resource_data = provider_data.get(resource_type)
+ if resource_data:
+ current_modules = {str(i) for r in resource_data for i in
r.get("python-modules", [])}
+ num_retry_policy_modules += len(current_modules)
+ num_errors += check_if_objects_exist_and_belong_to_package(
+ current_modules, provider_package, yaml_file_path,
resource_type, ObjectType.MODULE
+ )
+ return num_retry_policy_modules, num_errors
+
+
@run_check("Checking that conn-fields in provider.yaml match
get_connection_form_widgets() of the hook class")
def check_conn_fields_match_form_widgets(yaml_files: dict[str, dict]) ->
tuple[int, int]:
"""
@@ -606,13 +630,13 @@ def
check_hook_classes_with_conn_type_are_registered(yaml_files: dict[str, dict]
@run_check(
- "Checking that all provider
Hook/Operator/Sensor/Trigger/Executor/Notifier/Toolset"
+ "Checking that all provider
Hook/Operator/Sensor/Trigger/Executor/Notifier/Toolset/RetryPolicy"
" classes are registered in provider.yaml"
)
def check_all_provider_classes_are_registered(yaml_files: dict[str, dict]) ->
tuple[int, int]:
"""
Walk all provider source files, find
Hook/Operator/Sensor/Trigger/Executor/Notifier/
- SecretsBackend/AuthManager/LoggingHandler/DagBundle/DBManager/Toolset
subclasses, and
+
SecretsBackend/AuthManager/LoggingHandler/DagBundle/DBManager/Toolset/RetryPolicy
subclasses, and
verify they are registered in the appropriate provider.yaml section.
This catches classes placed in non-standard directories or modules that
were missed
@@ -626,6 +650,7 @@ def check_all_provider_classes_are_registered(yaml_files:
dict[str, dict]) -> tu
from airflow.models.baseoperator import BaseOperator
from airflow.sdk.bases.hook import BaseHook
from airflow.sdk.bases.notifier import BaseNotifier
+ from airflow.sdk.definitions.retry_policy import RetryPolicy
from airflow.secrets.base_secrets import BaseSecretsBackend
from airflow.sensors.base import BaseSensorOperator
from airflow.triggers.base import BaseTrigger
@@ -646,6 +671,7 @@ def check_all_provider_classes_are_registered(yaml_files:
dict[str, dict]) -> tu
(BaseDagBundle, "bundles"),
(BaseDBManager, "db-managers"),
(AbstractToolset, "toolsets"),
+ (RetryPolicy, "retry-policies"),
]
# Resource types where registration is by class path (not module)
@@ -670,7 +696,15 @@ def check_all_provider_classes_are_registered(yaml_files:
dict[str, dict]) -> tu
# Collect all modules registered in provider.yaml across all resource
types
registered_modules: set[str] = set()
- for resource_type in ("hooks", "operators", "sensors", "triggers",
"bundles", "toolsets"):
+ for resource_type in (
+ "hooks",
+ "operators",
+ "sensors",
+ "triggers",
+ "bundles",
+ "toolsets",
+ "retry-policies",
+ ):
for entry in provider_data.get(resource_type, []):
registered_modules.update(entry.get("python-modules", []))
for entry in provider_data.get("transfers", []):
@@ -905,7 +939,8 @@ def check_invalid_integration(yaml_files: dict[str, dict])
-> tuple[int, int]:
num_errors = 0
num_integrations = len(all_integration_names)
for (yaml_file_path, provider_data), resource_type in itertools.product(
- yaml_files.items(), ["sensors", "operators", "hooks", "triggers",
"bundles", "toolsets"]
+ yaml_files.items(),
+ ["sensors", "operators", "hooks", "triggers", "bundles", "toolsets",
"retry-policies"],
):
resource_data = provider_data.get(resource_type, [])
current_names = {r["integration-name"] for r in resource_data}
@@ -1123,6 +1158,7 @@ if __name__ == "__main__":
check_completeness_of_list_of_transfers(all_parsed_yaml_files)
check_hook_class_name_entries_in_connection_types(all_parsed_yaml_files)
+
check_retry_policy_modules_exist_and_belong_to_package(all_parsed_yaml_files)
check_conn_fields_match_form_widgets(all_parsed_yaml_files)
check_hook_classes_with_conn_type_are_registered(all_parsed_yaml_files)
check_executor_classes(all_parsed_yaml_files)
