kaxil commented on code in PR #74386: URL: https://github.com/apache/airflow/pull/74386#discussion_r4300821902
########## providers/databricks/pyproject.toml: ########## @@ -76,6 +77,9 @@ dependencies = [ # The optional dependencies should be modified in place in the generated file # Any change in the dependencies is preserved when the file is regenerated [project.optional-dependencies] +"common.ai" = [ + "apache-airflow-providers-common-ai>=0.10.0", # use next version Review Comment: `providers-common-ai/0.10.0` has no `managed_agents` package, and 0.11.0 was never released. Once common.ai 1.0.0 has its final tag, `_process_line_with_next_version_comment` leaves a line alone whose referenced version is already tagged, so a Databricks release after that ships `>=0.10.0`, and a user on 0.10.0 gets the "needs the 'common.ai' extra" error with the extra installed. Amazon and Google use `>=0.11.0` for this extra, and since this is a new line the selective-checks bump detector won't flag it. Could this match them? ########## providers/databricks/src/airflow/providers/databricks/operators/agent.py: ########## @@ -0,0 +1,200 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Operators for invoking Databricks agents.""" + +from __future__ import annotations + +import json +import time +from collections.abc import Sequence +from datetime import timedelta +from functools import cached_property +from typing import TYPE_CHECKING, Any +from uuid import NAMESPACE_URL, UUID, uuid5 + +from airflow.providers.common.compat.sdk import AirflowFailException, BaseOperator, conf +from airflow.providers.databricks.exceptions import ( + DatabricksAgentInvocationError, + DatabricksAgentInvocationTimeout, +) +from airflow.providers.databricks.hooks.agent import DatabricksAgentHook +from airflow.providers.databricks.triggers.agent import DatabricksAgentInvocationTrigger + +if TYPE_CHECKING: + from airflow.providers.common.compat.sdk import Context + + +class DatabricksAgentInvokeOperator(BaseOperator): + """ + Invoke a DurableAgentServer on Databricks Apps and optionally wait for its result. + + :param app_url: HTTPS base URL of the deployed app. (templated) + :param input: JSON-serializable agent input. (templated) + :param databricks_conn_id: Databricks connection using service principal OAuth. + Defaults to ``databricks_default``. (templated) + :param session_id: Conversation ID. CLI template agents require this field. + Defaults to ``None``. (templated) + :param invocation_id: Idempotency UUID. Defaults to a stable UUID for this task instance, + app URL, input and session, shared across retries and clears with unchanged input and session. + A stored failure requires a new UUID to run again. (templated) + :param wait_for_termination: Wait for completion or interruption. If false, return the submission response. + Defaults to ``True``. + :param polling_period_seconds: Seconds between status checks. Defaults to ``10``. + :param timeout: Maximum seconds to wait after submission. Timing out does not cancel the remote invocation. + Defaults to ``3600``. + :param deferrable: Release the worker while waiting. Defaults to ``[operators] default_deferrable``, + with a fallback of ``False``. + """ + + template_fields: Sequence[str] = ("app_url", "input", "databricks_conn_id", "session_id", "invocation_id") + template_fields_renderers = {"input": "json"} + ui_color = "#1CB1C2" + ui_fgcolor = "#fff" + + def __init__( + self, + *, + app_url: str, + input: Any, + databricks_conn_id: str = "databricks_default", + session_id: str | None = None, + invocation_id: str | None = None, + wait_for_termination: bool = True, + polling_period_seconds: float = 10, + timeout: float = 3600, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), + **kwargs, + ) -> None: + super().__init__(**kwargs) + if polling_period_seconds <= 0 or timeout <= 0: + raise ValueError("polling_period_seconds and timeout must be positive") + self.app_url = app_url + self.input = input + self.databricks_conn_id = databricks_conn_id + self.session_id = session_id + self.invocation_id = invocation_id + self.wait_for_termination = wait_for_termination + self.polling_period_seconds = polling_period_seconds + self.timeout = timeout + self.deferrable = deferrable + + @cached_property + def hook(self) -> DatabricksAgentHook: + return DatabricksAgentHook(self.app_url, self.databricks_conn_id) + + def _get_invocation_id(self, context: Context) -> str: + if self.invocation_id is not None: + return str(UUID(self.invocation_id)) + ti = context["ti"] + identity = json.dumps( + [ + self.app_url.rstrip("/"), + ti.dag_id, + ti.task_id, + ti.run_id, + ti.map_index, + self.input, + self.session_id, + ], + sort_keys=True, + ) + return str(uuid5(NAMESPACE_URL, identity)) + + def _get_result(self, result: dict[str, Any], invocation_id: str) -> dict[str, Any] | None: + status = result.get("status") + if status == "failed": + raise AirflowFailException( + f"Databricks agent invocation {invocation_id} failed; inspect the app logs " + "and supply a new invocation_id to run again" + ) + if status == "completed": + self.log.info("Databricks agent invocation %s completed.", invocation_id) + return result + if not status: + raise DatabricksAgentInvocationError( + f"Databricks agent invocation {invocation_id} response is missing its status" + ) + return None + + def execute(self, context: Context) -> dict[str, Any]: + invocation_id = self._get_invocation_id(context) + self.log.info("Submitting Databricks agent invocation %s.", invocation_id) + submitted = self.hook.create_invocation(invocation_id, self.input, self.session_id) + self.log.info("Databricks agent invocation %s was submitted.", invocation_id) + if not self.wait_for_termination: + return submitted Review Comment: With `wait_for_termination=False` this returns before the `failed` check below, so a clear with unchanged input re-POSTs the stored ID, gets `{"status": "failed"}` back, and the task goes green without the agent running. agent.rst:81 says a stored failure raises `AirflowFailException` with no such caveat. Moving the `failed` check above this return would match the docs. No operator test sends a failed submission from `create_invocation` today, so one here would pin it. ########## providers/databricks/src/airflow/providers/databricks/hooks/agent.py: ########## @@ -0,0 +1,405 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Invoke agents served by Databricks DurableAgentServer.""" + +from __future__ import annotations + +import json +import time +from typing import TYPE_CHECKING, Any, cast +from urllib.parse import urlsplit +from uuid import UUID, uuid4 + +import aiohttp +import requests +from tenacity import RetryError, stop_before_delay + +from airflow.providers.common.compat.sdk import AirflowException, AirflowOptionalProviderFeatureException +from airflow.providers.databricks.exceptions import DatabricksAgentInvocationTimeout, DatabricksApiError +from airflow.providers.databricks.hooks.databricks_base import BaseDatabricksHook, _TokenAuth + +if TYPE_CHECKING: + from tenacity.stop import stop_base + + from airflow.providers.common.ai.exceptions import ManagedAgentInvocationError + from airflow.providers.common.ai.managed_agents.base import ( + BaseManagedAgentHook, + ManagedAgentCapabilities, + ManagedAgentRef, + ManagedAgentRequest, + ManagedAgentResponse, + ) +else: + try: + from airflow.providers.common.ai.exceptions import ManagedAgentInvocationError + from airflow.providers.common.ai.managed_agents.base import ( + BaseManagedAgentHook, + ManagedAgentCapabilities, + ManagedAgentRef, + ManagedAgentResponse, + ) + except ImportError as e: + _common_ai_import_error = e + + def _needs_common_ai(*args: Any, **kwargs: Any) -> Any: + raise AirflowOptionalProviderFeatureException( + "Consulting a Databricks agent needs the 'common.ai' extra: " + "pip install 'apache-airflow-providers-databricks[common.ai]'." + ) from _common_ai_import_error + + class BaseManagedAgentHook: + """Keep background invocation available without the Common AI extra.""" + + agent = _needs_common_ai + + ManagedAgentCapabilities = ManagedAgentRef = ManagedAgentResponse = _needs_common_ai + ManagedAgentInvocationError = _needs_common_ai + + +_AGENT_OPTIONS = frozenset({"input_key", "invocation_id"}) + + +class DatabricksAgentHook(BaseDatabricksHook, BaseManagedAgentHook): + """ + Invoke a DurableAgentServer deployed on Databricks Apps. + + With the ``common.ai`` extra installed, ``hook.agent(app_url)`` exposes the synchronous + invocation API through the managed-agent contract. Background invocations remain available + through ``create_invocation`` and ``get_invocation`` without that extra. + + :param app_url: HTTPS base URL for background invocation methods. Defaults to ``None``; + managed-agent calls receive the app URL as their ``agent`` argument. + :param databricks_conn_id: Databricks workspace connection configured with + ``service_principal_oauth=true``, a client ID in login and a client secret in password. + Defaults to ``databricks_default``. + :param timeout_seconds: HTTP request timeout in seconds. Defaults to ``180``. + :param retry_limit: Maximum number of attempts for transient HTTP failures. Defaults to ``3``. + :param retry_delay: Minimum delay between HTTP retries in seconds. Defaults to ``1``. + """ + + def __init__( + self, + app_url: str | None = None, + databricks_conn_id: str = "databricks_default", + timeout_seconds: int = 180, + retry_limit: int = 3, + retry_delay: float = 1, + ) -> None: + super().__init__(databricks_conn_id, timeout_seconds, retry_limit, retry_delay) + self.app_url = self._validate_app_url(app_url) if app_url is not None else None + + agent_platform = "databricks.agent_runtime" + + @staticmethod + def _validate_app_url(app_url: str | None) -> str: + if app_url is None: + raise ValueError("app_url is required for background invocation methods") + parsed = urlsplit(app_url) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.path not in ("", "/") + or parsed.query + or parsed.fragment + ): + raise ValueError("app_url must be an HTTPS base URL without credentials, path, query or fragment") + return app_url.rstrip("/") + + def resolve_agent(self, agent: str) -> ManagedAgentRef: + """ + Resolve an app's HTTPS base URL without contacting Databricks. + + :param agent: HTTPS base URL of the deployed app. + :return: Platform-qualified app URL with its trailing slash removed. + """ + return ManagedAgentRef(platform=self.agent_platform, name=self._validate_app_url(agent)) + + def get_agent_capabilities(self, agent: str) -> ManagedAgentCapabilities: + """ + Report session, structured-output and invocation-ID support. + + :param agent: HTTPS base URL of the deployed app. + :return: Capabilities of the synchronous invocation API. + """ + self.resolve_agent(agent) + return ManagedAgentCapabilities(sessions=True, structured_output=True, trace=True) + + def invoke_agent(self, agent: str, request: ManagedAgentRequest) -> ManagedAgentResponse: + """ + Invoke an agent synchronously through the Common AI contract. + + :param agent: HTTPS base URL of the deployed app. + :param request: Prompt or messages, optional session and request timeout. Vendor options + accept ``input_key`` to send a prompt under an agent-specific key and ``invocation_id`` + to reuse a caller-provided UUID. By default, prompts are converted to messages and + a new UUID is generated for each call. Without a session, the invocation UUID + is also used as a one-shot session ID. + :return: Answer text, full invocation response, structured output, session and invocation ID. + """ + app_url = self.resolve_agent(agent).name + unknown = set(request.vendor_options) - _AGENT_OPTIONS + if unknown: + raise ValueError( + f"vendor_options {sorted(unknown)} are not accepted; expected {sorted(_AGENT_OPTIONS)}" + ) + input_key = request.vendor_options.get("input_key") + if input_key is not None and (not isinstance(input_key, str) or not input_key): + raise ValueError("vendor_options['input_key'] must be a nonempty string") + if input_key is not None and request.messages is not None: + raise ValueError("vendor_options['input_key'] only applies to a prompt") + invocation_id = ( + str(UUID(request.vendor_options["invocation_id"])) + if "invocation_id" in request.vendor_options + else str(uuid4()) + ) + session_id = request.session_id if request.session_id is not None else invocation_id + payload: dict[str, Any] = { + "id": invocation_id, + "session_id": session_id, + "input": {input_key: request.prompt} + if input_key is not None + else {"messages": request.as_messages()}, + } + if request.timeout is not None and request.timeout <= 0: + raise ValueError("timeout_seconds must be positive") + deadline = time.monotonic() + request.timeout if request.timeout is not None else None + try: + raw = self._do_agent_api_call( + "POST", + "api/invocations", + payload, + session_id, + timeout_seconds=self._get_remaining_timeout(deadline, invocation_id), + app_url=app_url, + ) + except DatabricksAgentInvocationTimeout: + raise + except AirflowException as exc: + cause = exc.__cause__ or exc.__context__ + if isinstance(cause, RetryError): + cause = cause.last_attempt.exception() + if ( + isinstance(exc, DatabricksApiError) + and isinstance(cause, requests.HTTPError) + and cause.response is not None + and cause.response.status_code >= 500 + ): + # A synchronous agent failure is HTTP 500; the stored status distinguishes it from an outage. + try: + state = self._do_agent_api_call( + "GET", + f"api/invocations/{invocation_id}", + None, + session_id, + timeout_seconds=self._get_remaining_timeout(deadline, invocation_id), + app_url=app_url, + ) + except DatabricksAgentInvocationTimeout: + raise + except (AirflowException, requests.RequestException, ValueError): + pass + else: + self._get_remaining_timeout(deadline, invocation_id) + if isinstance(state, dict) and state.get("status") == "failed": + raise ManagedAgentInvocationError( + f"Databricks agent invocation {invocation_id} failed; inspect the app logs" + ) from exc + if isinstance(exc.__cause__, RetryError) and cause is not None: + raise cause from None + if isinstance(cause, requests.HTTPError) and ( + cause.response.status_code == 429 or cause.response.status_code >= 500 + ): + raise cause from None + raise ManagedAgentInvocationError( + "Databricks managed-agent request failed; inspect the app logs" + ) from exc + self._get_remaining_timeout(deadline, invocation_id) + if not isinstance(raw, dict) or raw.get("status") != "completed": + raise ManagedAgentInvocationError( + f"Databricks agent invocation {invocation_id} did not complete; inspect the app logs" + ) + output = raw.get("output") + if isinstance(output, dict) and output.get("status") == "interrupted": + raise ManagedAgentInvocationError( + f"Databricks agent invocation {invocation_id} was interrupted; use the operator to handle its output" + ) + text = output if isinstance(output, str) else "" if output is None else json.dumps(output) Review Comment: The Databricks CLI templates return `{"output": [<messages>], "status": "completed"}` from the handler (`agent-openai/runtime/adapter.py:118-121` and the langgraph adapter in agentbricks 0.4.0), so `output["output"]` is a list and `text` becomes `json.dumps` of the whole envelope, tool-call args and tool outputs included. `ManagedAgentToolset` hands `text` straight to the calling model, and `ManagedAgentResponse` says the hook unwraps the vendor envelope to produce it. Could a list `output` use the last assistant message's `content` as `text`, keeping the JSON fallback for shapes it doesn't recognise? The fixture app returns its own dict, so only a unit test with the template shape would cover this. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
