kaxil commented on code in PR #76974: URL: https://github.com/apache/airflow/pull/76974#discussion_r4622874583
########## providers/common/ai/src/airflow/providers/common/ai/batch/pydantic_ai_backend.py: ########## @@ -0,0 +1,937 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Batch adapters built on pydantic-ai's public ``Model.batch_*`` methods. + +One backend drives every provider pydantic-ai can batch: it encodes each request with +``batch_encode``, submits them with ``batch_create``, polls with ``batch_status``, collects with +``batch_results`` and reads each result with ``batch_decode``. The provider subclasses set only +what differs between providers (the settings they accept and how a finished job is read). + +pydantic-ai's ``Batch`` handle is plain data. It is stored with the batch's state record, so the +trigger and a later worker rebuild a model from the connection and continue the same batch without +the original process. + +Importing this module never imports a provider SDK: the model classes are built by +:class:`~airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIHook`, and the SDK-specific +settings types are imported inside the methods that need them. +""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import dataclasses +import functools +import queue +import threading +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Coroutine, Generator, Iterator, Mapping +from contextlib import AsyncExitStack, aclosing, asynccontextmanager, closing, suppress +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar, cast + +from pydantic import TypeAdapter +from pydantic_ai.exceptions import ModelAPIError, ModelHTTPError, UserError +from pydantic_ai.messages import ( + ModelRequest, + ModelRequestPart, + ModelResponse, + SystemPromptPart, + UserPromptPart, +) +from pydantic_ai.models import ModelRequestParameters + +# pydantic-ai raises this for a job that ended with no results file. Nothing public tells it from an API failure +# (the handle carries no result destination), and a test pins that it stays a ``ModelHTTPError``. +from pydantic_ai.models._batch_codec import BatchJobError +from pydantic_ai.models.batch import ( + Batch, + BatchRequest as PydanticAIBatchRequest, + BatchResult, + EncodedBatchRequest, +) +from pydantic_ai.output import OutputObjectDefinition +from pydantic_ai.settings import ModelSettings +from pydantic_ai.tools import ToolDefinition + +from airflow.providers.common.ai.batch.base import ( + BatchAdapter, + BatchState, + ExtractedOutput, + RawResultItem, + SubmitResult, +) +from airflow.providers.common.ai.batch.output_schema import build_output_spec +from airflow.providers.common.ai.exceptions import ( + LLMBatchInputError, + LLMBatchJobError, + LLMBatchLimitExceededError, + LLMBatchModelMismatchError, + LLMBatchStateReadError, +) +from airflow.providers.common.ai.hooks.pydantic_ai import PydanticAIHook +from airflow.providers.common.ai.utils.task_logger import get_task_logger + +if TYPE_CHECKING: + from pydantic_ai.models import Model + + from airflow.providers.common.ai.batch.base import BatchRequest + from airflow.providers.common.ai.batch.output_schema import OutputSpec + from airflow.sdk import Connection + +log = get_task_logger() + +T = TypeVar("T") + +_BATCH_ADAPTER: TypeAdapter[Batch] = TypeAdapter(Batch) + +#: Sampling settings with the same meaning on every provider. A key in this set (or one of a +#: provider's own ``ModelSettings`` keys) becomes a typed setting; any other key is a raw +#: provider request field. +_GENERIC_SETTINGS = frozenset( + { + "temperature", + "top_p", + "top_k", + "seed", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "stop_sequences", + } +) + +#: Request fields this adapter sets itself. Taking them from ``request_params`` would replace the +#: prompt, the model or the structured-output constraint, so they are dropped with a warning. +_MANAGED_KEYS = frozenset( + { + "model", + "messages", + "input", + "system", + "system_instruction", + "contents", + "tools", + "tool_choice", + "stream", + "max_tokens", + "max_output_tokens", + "max_completion_tokens", + "output_config", + "text", + "response_format", + } +) + +#: pydantic-ai's status for a request that never ran, as this provider names it. +_UNRUN_STATUS: dict[str, Literal["cancelled", "expired"]] = {"canceled": "cancelled", "expired": "expired"} + +#: Gemini documents no request limit, and the ABC wants an int. +_NO_REQUEST_LIMIT = 2**63 - 1 + +#: Decoded rows a batch's results may run ahead of the task that reads them. +_RESULT_BUFFER = 64 + +#: Seconds a closed results stream waits for its helper thread to finish after cancelling it. +_STREAM_JOIN_TIMEOUT = 5.0 + + [email protected](frozen=True) +class _ResultsContext: + """What a reader needs, besides the raw results, to decode them.""" + + model: Model + parameters: ModelRequestParameters + + +def _run(coro: Coroutine[Any, Any, T]) -> T: + """ + Run ``coro`` to completion from synchronous code. + + ``asyncio.run`` refuses to start inside a thread that already runs a loop, which is where a + kill signal's ``on_kill`` can land while a poll is in flight, so that case uses a helper thread. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + # No loop in this thread. Run outside the ``except`` so an error from ``coro`` isn't chained + # onto this probe in the task log. + pass + else: + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(asyncio.run, coro).result() + return asyncio.run(coro) + + +def _stream(source: AsyncGenerator[T, None]) -> Generator[T, None, None]: + """ + Read an async generator from synchronous code without collecting it first. + + The generator runs as a task on its own event loop in a helper thread and hands items over through a + queue of :data:`_RESULT_BUFFER` items, so a slow reader holds the download back instead of buffering it. + An error from the generator is raised at the reader, whatever its type. + + When the reader stops (closes the iterator, is dropped, or fails), the task is cancelled from the reader's + thread. A read the generator is awaiting is interrupted rather than waited for, and the generator's own + cleanup still runs, so whatever it holds open is closed. The wait for the thread is bounded. + """ + items: queue.Queue[ + tuple[Literal["item"], T] | tuple[Literal["error"], BaseException] | tuple[Literal["end"], None] + ] + items = queue.Queue(maxsize=_RESULT_BUFFER) + closed = threading.Event() + started = threading.Event() + running: list[tuple[asyncio.AbstractEventLoop, asyncio.Task[None]]] = [] + + def put(message: tuple[Any, Any]) -> bool: + # A bounded put that gives up once the reader is gone, so a full queue never strands the thread. + while not closed.is_set(): + try: + items.put(message, timeout=0.1) + return True + except queue.Full: + continue + return False + + async def produce() -> None: + try: + async for item in source: + if not put(("item", item)): + return + put(("end", None)) + except BaseException as e: + # Also a cancellation the reader did not ask for: forwarding it keeps the reader from waiting forever. + put(("error", e)) + finally: + await source.aclose() + + async def main() -> None: + task = asyncio.ensure_future(produce()) + running.append((asyncio.get_running_loop(), task)) + started.set() + await asyncio.wait([task]) + + thread = threading.Thread(target=asyncio.run, args=(main(),), name="llm-batch-results", daemon=True) + thread.start() + try: + while True: + try: + message = items.get(timeout=1) + except queue.Empty: + if not thread.is_alive() and items.empty(): + raise RuntimeError("The batch results reader stopped without finishing.") from None + continue + if message[0] == "item": + yield message[1] + elif message[0] == "error": + raise message[1] + else: + return + finally: + closed.set() + if started.wait(timeout=1): + loop, task = running[0] + # A RuntimeError means the loop already finished, so there is nothing left to cancel. + with suppress(RuntimeError): + loop.call_soon_threadsafe(task.cancel) + thread.join(timeout=_STREAM_JOIN_TIMEOUT) + if thread.is_alive(): + log.debug( + "Batch results thread did not stop within the timeout; it is a daemon and will be left behind" + ) + + [email protected] +def _typed_settings(settings_type: type) -> frozenset[str]: + """Return the keys a request may set as typed settings: the generic ones and the provider's own.""" + return _GENERIC_SETTINGS | (set(settings_type.__annotations__) - set(ModelSettings.__annotations__)) + + +def _error_body(error: ModelHTTPError) -> Mapping[str, Any]: + """Return the ``error`` object of an HTTP error's body, which carries the provider's own type and message.""" + body = error.body + inner = body.get("error") if isinstance(body, Mapping) else None + return inner if isinstance(inner, Mapping) else {} + + +def _reason(error: ModelHTTPError) -> str: + """Return the provider's own explanation of an HTTP error; pydantic-ai's message wraps it in status and model.""" + return str(_error_body(error).get("message") or error.message) + + +def _error_info(status: str, message: str, provider_code: str | int | None = None) -> dict[str, Any]: + return {"type": status, "message": message, "provider_code": provider_code, "stage": "provider"} + + +def _failed( + custom_id: str, + index: int, + status: Literal["errored", "expired", "cancelled"], + error: Mapping[str, Any], +) -> RawResultItem: + return RawResultItem( + custom_id=custom_id, + index=index, + provider_status=status, + model=None, + usage=None, + finish_reason=None, + error=error, + raw=None, + ) + + +class PydanticAIBatchAdapter(BatchAdapter): + """ + Batch adapter for any provider whose pydantic-ai ``Model`` implements the ``batch_*`` methods. + + .. note:: + + Experimental: this can change or be removed in a minor release of this provider. + See :ref:`howto/stability`. + + The adapter is bound to one connection and one ``model_id``. It builds a fresh ``Model`` for + every operation and runs it in its own event loop, so no client crosses a loop, and it never + keeps a connection open between operations. + + :param hook: The hook the models are built from (credentials, ``provider_kwargs``, the + connection's extra). + :param model_id: The batch's ``"<provider>:<model>"`` id. + :param batch_handle: The serialized ``Batch`` handle recorded when the batch was submitted. + :param output_spec: The task's output spec. Needed only to read results. + :param model_factory: Builds the ``Model`` for a model id instead of the hook. For tests. + """ + + conn_types = frozenset({"pydanticai"}) + #: ``True`` when a key that is not a typed setting is passed to the provider as a raw request field. + accepts_raw_params: ClassVar[bool] = True + + def __init__( + self, + *, + hook: PydanticAIHook | None = None, + model_id: str | None = None, + batch_handle: str | None = None, + output_spec: OutputSpec | None = None, + model_factory: Callable[[str], Model] | None = None, + ) -> None: + super().__init__() + self._hook = hook + self._model_id = model_id + self._handle = batch_handle + self._output_spec = output_spec or build_output_spec(str) + self._model_factory = model_factory + + @classmethod + def from_connection( + cls, + conn: Connection, + *, + model_id: str | None = None, + batch_handle: str | None = None, + output_spec: OutputSpec | None = None, + ) -> PydanticAIBatchAdapter: + hook = PydanticAIHook.get_hook(conn.conn_id, {"model_id": model_id}) + if hook._get_fallback_conn_ids(): + log.warning( + "Connection has fallback connections configured; batch ignores them and uses the primary", + llm_conn_id=conn.conn_id, + ) + return cls(hook=hook, model_id=model_id, batch_handle=batch_handle, output_spec=output_spec) + + @classmethod + def _settings_type(cls) -> type: + """Return the provider's ``ModelSettings`` class, whose own keys a request may set directly.""" + raise NotImplementedError(f"{cls.__name__} does not name its ModelSettings type.") + + def _require_model_id(self) -> str: + if not self._model_id: + raise LLMBatchInputError( + f"The {self.name!r} batch adapter needs the batch's model_id ('<provider>:<model>') to " + "rebuild its model." + ) + return self._model_id + + async def _amodel(self, model_id: str) -> Model: + if self._model_factory is not None: + return self._model_factory(model_id) + if self._hook is None: + raise LLMBatchInputError(f"The {self.name!r} batch adapter was built without a connection.") + return await self._hook.aget_primary_model(model_id) + + @asynccontextmanager + async def _existing(self, batch_id: str) -> AsyncIterator[tuple[Model, Batch]]: + """ + Yield the model an already-submitted batch belongs to, and its handle. + + The model comes from the batch, not from the task's ``model_id``: pydantic-ai checks the + handle's group on every call, and OpenAI's and Gemini's groups name the model the batch was + submitted on. That differs from the task's model when every request overrode it, or when the + task's model changed since the batch was submitted. Only the provider prefix is taken from + ``model_id``, because the connection that submitted the batch is the one that resolves it. + """ + model_id = self._model_id + if not model_id: + raise LLMBatchStateReadError( + f"Cannot reach batch {batch_id!r}: it was deferred by an earlier version of this provider, " + "which did not record the model it needs. Retry the task so it re-attaches." + ) + if not self._handle: + async with await self._amodel(model_id) as model: + yield model, await self._unrecorded_batch(model, batch_id) + return + batch = _BATCH_ADAPTER.validate_json(self._handle) + provider = model_id.partition(":")[0] + async with await self._amodel(f"{provider}:{batch.model_name}") as model: + yield model, batch + + async def _unrecorded_batch(self, model: Model, batch_id: str) -> Batch: + """Return a handle for a batch recorded without one, by an earlier version of this provider.""" + raise LLMBatchStateReadError( + f"No pydantic-ai batch handle is recorded for batch {batch_id!r}, so it cannot be polled. " + "It was submitted before this provider recorded handles; retry the task so it re-attaches." + ) + + def _build_settings( + self, + model: Model, + *layers: Mapping[str, Any] | None, + max_tokens: int, + ) -> tuple[ModelSettings, set[str]]: + """ + Turn ``request_params`` and a request's ``params`` into ``ModelSettings``. + + Typed keys become settings. Any other key is a raw provider request field and goes through + ``extra_body``, which is how these parameters worked before the adapter used pydantic-ai. + Returns the settings and the managed keys that were dropped. + """ + typed = _typed_settings(self._settings_type()) + merged: dict[str, Any] = {} + for layer in layers: + merged.update(layer or {}) + extra_body: dict[str, Any] = {} + if "extra_body" in merged: + body = merged.pop("extra_body") + if not self.accepts_raw_params: + raise self._unsupported(model, "extra_body") + extra_body.update(body) + settings: dict[str, Any] = {} + for key, value in merged.items(): + if key in typed: + if key in model.unsupported_settings: + raise self._unsupported(model, key) + settings[key] = value + elif self.accepts_raw_params: + extra_body[key] = value + else: + raise LLMBatchInputError( + f"Unknown request parameter {key!r} for the {self.name!r} adapter. It accepts " + f"{sorted(typed)}." + ) + dropped = set(_MANAGED_KEYS & extra_body.keys()) + for key in dropped: + del extra_body[key] + settings["max_tokens"] = max_tokens + if extra_body: + settings["extra_body"] = extra_body + # The keys were checked against the provider's settings above; a TypedDict cannot be built from a dynamic dict. + return cast("ModelSettings", settings), dropped + + @staticmethod + def _unsupported(model: Model, key: str) -> LLMBatchInputError: + return LLMBatchInputError( + f"The request parameter {key!r} is not supported by {type(model).__name__}; " + "pydantic-ai would drop it without a warning." + ) + + @staticmethod + def _output_parameters(model: Model, spec: OutputSpec) -> ModelRequestParameters: + """ + Ask for ``spec``'s schema the strongest way the model supports. + + Native JSON-schema output where the model has it. Otherwise a single forced output tool, + which is a hard format constraint too. Neither is pydantic-ai's profile default + (``tool`` mode with no output tool sends no schema at all), so the mode is set explicitly. + """ + if not spec.is_structured: + return ModelRequestParameters() + if model.profile.get("supports_json_schema_output"): + return ModelRequestParameters( + output_mode="native", + output_object=OutputObjectDefinition(json_schema=spec.json_schema, name=spec.schema_name), + ) + return ModelRequestParameters( + output_mode="tool", + output_tools=[ + ToolDefinition( + name=spec.schema_name, + description="Return the result using this schema.", + parameters_json_schema=spec.json_schema, + kind="output", + ) + ], + allow_text_output=False, + ) + + async def _encode( + self, + stack: AsyncExitStack, + requests: list[BatchRequest], + *, + model: str, + idempotency_key: str, + output_spec: OutputSpec, + system_prompt: str, + max_tokens: int, + request_params: dict[str, Any] | None, + ) -> tuple[Model, str, list[EncodedBatchRequest]]: + """ + Encode every request locally, enforcing the provider's batch limits and a single group. + + Returns request 0's model, its id and the encoded requests. Nothing is sent. + """ + if not requests: + raise LLMBatchInputError("A batch needs at least one request.") + default_id = self._require_model_id() + # Per model id: the model, the output parameters and the settings of a request with no params of its own. + models: dict[str, tuple[Model, ModelRequestParameters, ModelSettings]] = {} + dropped: set[str] = set() + encoded_requests: list[EncodedBatchRequest] = [] + first_id = "" + total_bytes = 0 + for index, request in enumerate(requests): + model_id = default_id + if request_model := request.get("model"): + self.resolve_request_model(request_model, default_bare_model=model, request_index=index) + model_id = request_model + if model_id not in models: + current = await stack.enter_async_context(await self._amodel(model_id)) + settings, dropped_here = self._build_settings(current, request_params, max_tokens=max_tokens) + dropped |= dropped_here + models[model_id] = (current, self._output_parameters(current, output_spec), settings) + current, parameters, settings = models[model_id] + if request.get("params") or request.get("max_tokens"): + settings, dropped_here = self._build_settings( + current, + request_params, + request.get("params"), + max_tokens=request.get("max_tokens") or max_tokens, + ) + dropped |= dropped_here + parts: list[ModelRequestPart] = [] + if system := request.get("system_prompt") or system_prompt: + parts.append(SystemPromptPart(content=system)) + parts.append(UserPromptPart(content=request["prompt"])) + try: + encoded = await current.batch_encode( + PydanticAIBatchRequest( + custom_id=f"{idempotency_key}-{index}", + messages=[ModelRequest(parts=parts)], + model_settings=settings, + model_request_parameters=parameters, + ) + ) + except UserError as e: + raise LLMBatchInputError(f"Request {index} cannot be sent as a batch: {e}") from e + if index == 0: + first_id = model_id + elif encoded.group != encoded_requests[0].group: + raise LLMBatchModelMismatchError( + f"Request {index} cannot share a batch with request 0: they use different models, " + f"credentials or endpoints, and the {self.name!r} batch API takes one of each per batch. " + "Use one model for the whole task, or split the input across tasks." + ) + total_bytes += current.batch_request_size(encoded) + encoded_requests.append(encoded) + if dropped: + log.warning("Ignoring request parameters the batch sets itself", keys=sorted(dropped)) + first = models[first_id][0] + self._check_limits(first, len(requests), total_bytes) + return first, first_id, encoded_requests + + def _check_limits(self, model: Model, request_count: int, total_bytes: int) -> None: + max_requests = model.batch_max_requests + if max_requests is not None and request_count > max_requests: + raise LLMBatchLimitExceededError( + f"The {self.name!r} batch API accepts at most {max_requests} requests; got {request_count}. " + f"Split the input across mapped task instances with .expand() so each instance submits at " + f"most {max_requests} requests." + ) + max_bytes = model.batch_max_bytes + if max_bytes is not None and total_bytes > max_bytes: + raise LLMBatchLimitExceededError( + f"The {self.name!r} batch API accepts a payload of at most {max_bytes} bytes; this input is " + f"{total_bytes} bytes (the output_type schema is repeated in every request). Split the input " + "with .expand()." + ) + + @asynccontextmanager + async def _encoded( + self, + requests: list[BatchRequest], + *, + model: str, + idempotency_key: str, + output_spec: OutputSpec, + system_prompt: str, + max_tokens: int, + request_params: dict[str, Any] | None, + ) -> AsyncIterator[tuple[Model, list[EncodedBatchRequest]]]: + """Yield request 0's model and the encoded requests, with every model open until the block ends.""" + async with AsyncExitStack() as stack: + first, _, encoded = await self._encode( + stack, + requests, + model=model, + idempotency_key=idempotency_key, + output_spec=output_spec, + system_prompt=system_prompt, + max_tokens=max_tokens, + request_params=request_params, + ) + yield first, encoded + + def validate_requests( + self, + requests: list[BatchRequest], + *, + model: str, + output_spec: OutputSpec, + idempotency_key: str, + system_prompt: str = "", + max_tokens: int = 1024, + request_params: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + self.check_custom_id_length(idempotency_key, len(requests)) + + async def validate() -> None: + async with self._encoded( + requests, + model=model, + idempotency_key=idempotency_key, + output_spec=output_spec, + system_prompt=system_prompt, + max_tokens=max_tokens, + request_params=request_params, + ): + pass + + _run(validate()) + + def submit( + self, + requests: list[BatchRequest], + *, + model: str, + idempotency_key: str, + input_fingerprint: str, + output_spec: OutputSpec, + system_prompt: str = "", + max_tokens: int = 1024, + request_params: dict[str, Any] | None = None, + **kwargs: Any, + ) -> SubmitResult: + async def create() -> Batch: + async with self._encoded( + requests, + model=model, + idempotency_key=idempotency_key, + output_spec=output_spec, + system_prompt=system_prompt, + max_tokens=max_tokens, + request_params=request_params, + ) as (submitter, encoded): + return await submitter.batch_create(encoded) + + batch = _run(create()) + self._handle = _BATCH_ADAPTER.dump_json(batch).decode() + return SubmitResult( + batch_id=batch.id, + provider_input_ref=(batch.provider_details or {}).get("input_file"), + handle=self._handle, + ) + + async def aget_batch(self, batch_id: str) -> BatchState: + async with self._existing(batch_id) as (model, stored): + return await self._describe(model, await model.batch_status(stored)) + + async def _describe(self, model: Model, batch: Batch) -> BatchState: + """Reduce a refreshed handle to this adapter's status vocabulary; a provider with finer states extends it.""" + return BatchState( + status="completed" if batch.status == "ended" else "in_progress", counts=None, error_message=None + ) + + def get_batch(self, batch_id: str) -> BatchState: + return _run(self.aget_batch(batch_id)) + + async def acancel_batch(self, batch_id: str) -> None: + async with self._existing(batch_id) as (model, stored): + await model.batch_cancel(stored) + + def cancel_batch(self, batch_id: str) -> None: + _run(self.acancel_batch(batch_id)) + + def find_orphaned_batch( + self, idempotency_key: str, input_fingerprint: str, not_before: str + ) -> str | None: + """ + Return ``None`` unconditionally: ``batch_create`` takes no metadata to find a batch by. + + A crash between "submit sent" and "response recorded" falls through to ``on_orphaned_intent``. + """ + return None + + async def _collect(self, model: Model, batch: Batch) -> list[BatchResult]: + """Return the results of an ended batch; a provider that can end a job with no results file overrides it.""" + return await model.batch_results(batch) + + async def _stream_results(self, model: Model, batch: Batch) -> AsyncGenerator[BatchResult, None]: + """ + Yield the results of an ended batch one at a time. + + The default collects them as one list with ``batch_results``, which is all a provider whose SDK + downloads the results in one call can do (Gemini). A provider that can read its results file as a + stream overrides this. + """ + results = await self._collect(model, batch) + # Popping frees each result once it is handed on; the order does not matter, the merge is by index. + while results: + yield results.pop() + + async def _read_results(self, batch_id: str) -> AsyncGenerator[_ResultsContext | BatchResult, None]: + """Yield what decoding needs, then the raw results of an ended batch; decoding runs on the reader's thread.""" + async with self._existing(batch_id) as (model, stored): + # ``batch_results`` refuses a handle that has not ended, and the stored one is from submit time. + batch = await model.batch_status(stored) + if batch.status != "ended": + raise LLMBatchJobError(f"Batch {batch_id!r} has not ended (status {batch.status!r}).") + yield _ResultsContext(model, self._output_parameters(model, self._output_spec)) + async with aclosing(self._stream_results(model, batch)) as results: + async for result in results: + yield result + + def iter_results(self, batch_id: str) -> Iterator[RawResultItem]: + # Only the download crosses to the helper thread. Decoding is synchronous and does no I/O, so it + # stays here with the task's logging context. + with closing(_stream(self._read_results(batch_id))) as stream: + context = cast("_ResultsContext", next(stream)) + for result in stream: + row = self._decode_or_skip( + context.model, cast("BatchResult", result), context.parameters, batch_id + ) + if row is not None: + yield row + + def _decode_or_skip( + self, model: Model, result: BatchResult, parameters: ModelRequestParameters, batch_id: str + ) -> RawResultItem | None: + try: + return self._decode_one(model, result, parameters) + except Exception as e: + # One malformed item must not abort the whole batch; the merge counts the index as missing. + log.warning( + "Skipping unparsable result item for batch", + batch_id=batch_id, + custom_id=result.custom_id, + error=str(e), + ) + return None + + @staticmethod + def _decode_one(model: Model, result: BatchResult, parameters: ModelRequestParameters) -> RawResultItem: + custom_id = result.custom_id + index = int(custom_id.rsplit("-", 1)[1]) + if result.status in _UNRUN_STATUS: + return _failed( + custom_id, + index, + _UNRUN_STATUS[result.status], + _error_info(result.status, f"batch item {result.status}"), + ) + try: + response = model.batch_decode( + result, + PydanticAIBatchRequest(custom_id=custom_id, messages=[], model_request_parameters=parameters), + ) + except ModelAPIError as e: + provider_code = None + if isinstance(e, ModelHTTPError): + provider_code = _error_body(e).get("type") or e.status_code + return _failed(custom_id, index, "errored", _error_info(result.status, e.message, provider_code)) + details = response.provider_details or {} + return RawResultItem( + custom_id=custom_id, + index=index, + provider_status="success", + model=response.model_name, + usage={ + "input_tokens": response.usage.input_tokens, + "output_tokens": response.usage.output_tokens, + }, + finish_reason=details.get("finish_reason") or response.finish_reason, + error=None, + raw=response, + ) + + def extract_output(self, raw: RawResultItem, spec: OutputSpec) -> ExtractedOutput: + response: ModelResponse = raw.raw + if not spec.is_structured: + return ExtractedOutput(kind="text", text=response.text or None) + if response.tool_calls: + return ExtractedOutput(kind="json_value", value=response.tool_calls[0].args_as_dict()) + # Native structured output arrives as the response text, which is the JSON document. + if response.text: + return ExtractedOutput(kind="json_text", text=response.text) + return ExtractedOutput(kind="absent") + + +class AnthropicBatchAdapter(PydanticAIBatchAdapter): + """ + Batch adapter for Anthropic's Message Batches API. + + The batch's group leaves the model out, so one batch can hold requests for different Claude + models. Raw Messages API fields in ``request_params`` (``metadata``, ``thinking``, ...) go through + ``extra_body``. + """ + + name = "anthropic" + #: Anthropic's documented limit: 100,000 requests or 256 MB, whichever is reached first. + max_requests = 100_000 + max_payload_bytes = 256_000_000 + allows_per_request_model = True + + @classmethod + def _settings_type(cls) -> type: + # Heavy third-party SDK: importing the Anthropic model module imports ``anthropic``. + from pydantic_ai.models.anthropic import AnthropicModelSettings + + return AnthropicModelSettings + + async def _stream_results(self, model: Model, batch: Batch) -> AsyncGenerator[BatchResult, None]: + # Heavy third-party SDK: importing the Anthropic model module imports ``anthropic``. + from anthropic import AsyncAnthropic + from pydantic_ai.models.anthropic import AnthropicModel + + client = model.client if isinstance(model, AnthropicModel) else None + if not isinstance(client, AsyncAnthropic): + # Not a client pydantic-ai batches through (Bedrock, Vertex): its collect raises the right error. + async for collected in super()._stream_results(model, batch): + yield collected + return + # ``batch_status`` has already checked the credentials and the group, as ``batch_results`` would. + stream = await client.beta.messages.batches.results(batch.id) + try: + async for item in stream: + result = item.result + # Mirrors the loop in pydantic-ai's Anthropic ``batch_results``, one result at a time. + body = result.model_dump_json() if result.type in ("succeeded", "errored") else None + yield BatchResult(custom_id=item.custom_id, status=result.type, body=body) + finally: + await stream.close() + + async def _unrecorded_batch(self, model: Model, batch_id: str) -> Batch: + """ + Rebuild the handle of a batch an earlier version of this provider submitted. + + That version stored only the batch id. Anthropic's group is the endpoint and the credential + (not the model), so a throwaway encode on the connection's model yields the same group the + original submission had. ``created_at`` and ``request_count`` are placeholders: the status + call replaces them with the provider's own. + """ + try: + probe = await model.batch_encode( + PydanticAIBatchRequest( + custom_id="probe", messages=[ModelRequest(parts=[UserPromptPart(content="probe")])] + ) + ) + except UserError as e: + raise LLMBatchStateReadError( + f"Batch {batch_id!r} was recorded by an earlier version of this provider and cannot be " + f"re-attached: {e}" + ) from e + return Batch( + id=batch_id, + model_name=model.model_name, + provider_name=model.system, + group=probe.group, + status="in_progress", + created_at=datetime.now(UTC), + request_count=0, + ) + + +class GoogleBatchAdapter(PydanticAIBatchAdapter): + """ + Batch adapter for the Gemini Developer API (Batch Mode). + + Vertex AI is not supported, because its batch prediction reads and writes Cloud Storage or BigQuery. + The Gemini batch API has no raw request passthrough, so ``request_params`` may only hold typed settings. + A batch's group includes the model, so every request must use the same one. + """ + + name = "google" + max_requests = _NO_REQUEST_LIMIT + max_payload_bytes = 2_000_000_000 + allows_per_request_model = False + accepts_raw_params = False + + #: Job states that end a batch before it runs a request, as the status this provider reports. + _UNRUN_STATES: ClassVar[dict[str, Literal["cancelled", "expired"]]] = { + "JOB_STATE_CANCELLED": "cancelled", + "JOB_STATE_EXPIRED": "expired", + } + + @classmethod + def _settings_type(cls) -> type: + # Heavy third-party SDK: importing the Google model module imports ``google.genai``. + from pydantic_ai.models.google import GoogleModelSettings + + return GoogleModelSettings + + @staticmethod + def _job_state(batch: Batch) -> str | None: + return (batch.provider_details or {}).get("state") + + async def _describe(self, model: Model, batch: Batch) -> BatchState: + state = self._job_state(batch) + # ``ended`` covers every terminal state, so the outcome is in the job's own state. + if batch.status == "ended": + if state in self._UNRUN_STATES: + status = self._UNRUN_STATES[state] + return BatchState(status=status, counts={status: batch.request_count}, error_message=None) + if state == "JOB_STATE_FAILED": + message = f"Gemini batch job {batch.id} failed." + try: + await model.batch_results(batch) Review Comment: Google's batch docs don't say whether a failed job can have a results file, and pydantic-ai's comment allows for it, so I stopped assuming. `_describe` now does one `batches.get` and checks `job.dest`: with a results file the operator lands the rows (per-row errors included), without one it reports `failed` with the job's own error. It also no longer downloads anything in the triggerer just to get the error text. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
