kaxil commented on code in PR #76975: URL: https://github.com/apache/airflow/pull/76975#discussion_r4623389562
########## providers/common/ai/src/airflow/providers/common/ai/batch/pydantic_ai_backend.py: ########## @@ -0,0 +1,1128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Batch adapters built on pydantic-ai's public ``Model.batch_*`` methods. + +One backend drives every provider pydantic-ai can batch: it encodes each request with +``batch_encode``, submits them with ``batch_create``, polls with ``batch_status``, collects with +``batch_results`` and reads each result with ``batch_decode``. The provider subclasses set only +what differs between providers (the settings they accept and how a finished job is read). + +pydantic-ai's ``Batch`` handle is plain data. It is stored with the batch's state record, so the +trigger and a later worker rebuild a model from the connection and continue the same batch without +the original process. + +Importing this module never imports a provider SDK: the model classes are built by +:class:`~airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIHook`, and the SDK-specific +settings types are imported inside the methods that need them. +""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import dataclasses +import functools +import queue +import threading +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Coroutine, Generator, Iterator, Mapping +from contextlib import AsyncExitStack, aclosing, asynccontextmanager, closing, suppress +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar, cast + +from pydantic import BaseModel, JsonValue, TypeAdapter +from pydantic_ai.exceptions import ModelAPIError, ModelHTTPError, UserError +from pydantic_ai.messages import ( + ModelRequest, + ModelRequestPart, + ModelResponse, + SystemPromptPart, + UserPromptPart, +) +from pydantic_ai.models import ModelRequestParameters + +# pydantic-ai raises this for a job that ended with no results file. Nothing public tells it from an API failure +# (the handle carries no result destination), and a test pins that it stays a ``ModelHTTPError``. +from pydantic_ai.models._batch_codec import BatchJobError +from pydantic_ai.models.batch import ( + Batch, + BatchRequest as PydanticAIBatchRequest, + BatchResult, + EncodedBatchRequest, +) +from pydantic_ai.output import OutputObjectDefinition +from pydantic_ai.settings import ModelSettings +from pydantic_ai.tools import ToolDefinition +from pydantic_core import to_json + +from airflow.providers.common.ai.batch.base import ( + BatchAdapter, + BatchState, + ExtractedOutput, + RawResultItem, + SubmitResult, +) +from airflow.providers.common.ai.batch.output_schema import build_output_spec +from airflow.providers.common.ai.exceptions import ( + LLMBatchInputError, + LLMBatchJobError, + LLMBatchLimitExceededError, + LLMBatchModelMismatchError, + LLMBatchStateReadError, +) +from airflow.providers.common.ai.hooks.pydantic_ai import PydanticAIHook +from airflow.providers.common.ai.utils.task_logger import get_task_logger + +if TYPE_CHECKING: + from pydantic_ai.models import Model + from pydantic_ai.models.openai import OpenAIResponsesModel + + from airflow.providers.common.ai.batch.base import BatchRequest, BatchStatus + from airflow.providers.common.ai.batch.output_schema import OutputSpec + from airflow.sdk import Connection + +log = get_task_logger() + +T = TypeVar("T") + +_BATCH_ADAPTER: TypeAdapter[Batch] = TypeAdapter(Batch) + +#: Sampling settings with the same meaning on every provider. A key in this set (or one of a +#: provider's own ``ModelSettings`` keys) becomes a typed setting; any other key is a raw +#: provider request field. +_GENERIC_SETTINGS = frozenset( + { + "temperature", + "top_p", + "top_k", + "seed", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "stop_sequences", + } +) + +#: Request fields this adapter sets itself. Taking them from ``request_params`` would replace the +#: prompt, the model or the structured-output constraint, so they are dropped with a warning. +_MANAGED_KEYS = frozenset( + { + "model", + "messages", + "input", + "system", + "system_instruction", + "contents", + "tools", + "tool_choice", + "stream", + "max_tokens", + "max_output_tokens", + "max_completion_tokens", + "output_config", + "text", + "response_format", + } +) + +#: pydantic-ai's status for a request that never ran, as this provider names it. +_UNRUN_STATUS: dict[str, Literal["cancelled", "expired"]] = {"canceled": "cancelled", "expired": "expired"} + +#: The error codes of a request an OpenAI batch ended before running, and the status pydantic-ai gives each. +#: Mirrors ``_UNRUN_STATUSES`` in pydantic-ai's OpenAI batch module. +_OPENAI_UNRUN_STATUSES: dict[str, Literal["expired", "canceled"]] = { + "batch_expired": "expired", + "batch_cancelled": "canceled", +} + +#: Gemini documents no request limit, and the ABC wants an int. +_NO_REQUEST_LIMIT = 2**63 - 1 + +#: Decoded rows a batch's results may run ahead of the task that reads them. +_RESULT_BUFFER = 64 + +#: Seconds a closed results stream waits for its helper thread to finish after cancelling it. +_STREAM_JOIN_TIMEOUT = 5.0 + + [email protected](frozen=True) +class _ResultsContext: + """What a reader needs, besides the raw results, to decode them.""" + + model: Model + parameters: ModelRequestParameters + + +def _run(coro: Coroutine[Any, Any, T]) -> T: + """ + Run ``coro`` to completion from synchronous code. + + ``asyncio.run`` refuses to start inside a thread that already runs a loop, which is where a + kill signal's ``on_kill`` can land while a poll is in flight, so that case uses a helper thread. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + # No loop in this thread. Run outside the ``except`` so an error from ``coro`` isn't chained + # onto this probe in the task log. + pass + else: + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(asyncio.run, coro).result() + return asyncio.run(coro) + + +class _OpenAILineResponse(BaseModel): + status_code: int + body: JsonValue + + +class _OpenAILineError(BaseModel): + code: str | None = None + message: str | None = None + + +class _OpenAIOutputLine(BaseModel): + """A line of an OpenAI batch's output or error file.""" + + custom_id: str + response: _OpenAILineResponse | None = None + error: _OpenAILineError | None = None + + +class _OpenAIInputLine(BaseModel): + custom_id: str + + +def _openai_result(line: _OpenAIOutputLine) -> BatchResult: + """Mirror of ``_to_result`` in pydantic-ai's OpenAI batch module.""" + if (response := line.response) is not None: + if 200 <= response.status_code < 300: + return BatchResult( + custom_id=line.custom_id, status="succeeded", body=to_json(response.body).decode() + ) + return BatchResult(custom_id=line.custom_id, status="errored", body=response.model_dump_json()) + error = line.error or _OpenAILineError() + if status := _OPENAI_UNRUN_STATUSES.get(error.code or ""): + return BatchResult(custom_id=line.custom_id, status=status) + body = _OpenAILineResponse(status_code=500, body={"error": error.model_dump(mode="json")}) + return BatchResult(custom_id=line.custom_id, status="errored", body=body.model_dump_json()) + + +def _openai_unrun_result(details: Mapping[str, Any], custom_id: str, line_number: int) -> BatchResult: + """Mirror of ``_unrun_result`` in pydantic-ai's OpenAI batch module.""" + if details["status"] == "failed": + # The input file failed validation, so the batch ran none of its requests. A request gets the errors of its + # own line and those of no line; when they're all about other lines, it gets them all, which name them. + errors = details.get("errors", []) + own = [ + error + for error in errors + if not isinstance(error, Mapping) or error.get("line") in (line_number, None) + ] + body = _OpenAILineResponse(status_code=400, body={"errors": own or errors}) + return BatchResult(custom_id=custom_id, status="errored", body=body.model_dump_json()) + return BatchResult( + custom_id=custom_id, status="expired" if details["status"] == "expired" else "canceled" + ) + + +def _stream(source: AsyncGenerator[T, None]) -> Generator[T, None, None]: + """ + Read an async generator from synchronous code without collecting it first. + + The generator runs as a task on its own event loop in a helper thread and hands items over through a + queue of :data:`_RESULT_BUFFER` items, so a slow reader holds the download back instead of buffering it. + An error from the generator is raised at the reader, whatever its type. + + When the reader stops (closes the iterator, is dropped, or fails), the task is cancelled from the reader's + thread. A read the generator is awaiting is interrupted rather than waited for, and the generator's own + cleanup still runs, so whatever it holds open is closed. The wait for the thread is bounded. + """ + items: queue.Queue[ + tuple[Literal["item"], T] | tuple[Literal["error"], BaseException] | tuple[Literal["end"], None] + ] + items = queue.Queue(maxsize=_RESULT_BUFFER) + closed = threading.Event() + started = threading.Event() + running: list[tuple[asyncio.AbstractEventLoop, asyncio.Task[None]]] = [] + + def put(message: tuple[Any, Any]) -> bool: + # A bounded put that gives up once the reader is gone, so a full queue never strands the thread. + while not closed.is_set(): + try: + items.put(message, timeout=0.1) + return True + except queue.Full: + continue + return False + + async def produce() -> None: + try: + async for item in source: + if not put(("item", item)): + return + put(("end", None)) + except BaseException as e: + # Also a cancellation the reader did not ask for: forwarding it keeps the reader from waiting forever. + put(("error", e)) + finally: + await source.aclose() + + async def main() -> None: + task = asyncio.ensure_future(produce()) + running.append((asyncio.get_running_loop(), task)) + started.set() + await asyncio.wait([task]) + + thread = threading.Thread(target=asyncio.run, args=(main(),), name="llm-batch-results", daemon=True) + thread.start() + try: + while True: + try: + message = items.get(timeout=1) + except queue.Empty: + if not thread.is_alive() and items.empty(): + raise RuntimeError("The batch results reader stopped without finishing.") from None + continue + if message[0] == "item": + yield message[1] + elif message[0] == "error": + raise message[1] + else: + return + finally: + closed.set() + if started.wait(timeout=1): + loop, task = running[0] + # A RuntimeError means the loop already finished, so there is nothing left to cancel. + with suppress(RuntimeError): + loop.call_soon_threadsafe(task.cancel) + thread.join(timeout=_STREAM_JOIN_TIMEOUT) + if thread.is_alive(): + log.debug( + "Batch results thread did not stop within the timeout; it is a daemon and will be left behind" + ) + + [email protected] +def _typed_settings(settings_type: type) -> frozenset[str]: + """Return the keys a request may set as typed settings: the generic ones and the provider's own.""" + return _GENERIC_SETTINGS | (set(settings_type.__annotations__) - set(ModelSettings.__annotations__)) + + +def _error_body(error: ModelHTTPError) -> Mapping[str, Any]: + """Return the ``error`` object of an HTTP error's body, which carries the provider's own type and message.""" + body = error.body + inner = body.get("error") if isinstance(body, Mapping) else None + return inner if isinstance(inner, Mapping) else {} + + +def _reason(error: ModelHTTPError) -> str: + """Return the provider's own explanation of an HTTP error; pydantic-ai's message wraps it in status and model.""" + return str(_error_body(error).get("message") or error.message) + + +def _error_info(status: str, message: str, provider_code: str | int | None = None) -> dict[str, Any]: + return {"type": status, "message": message, "provider_code": provider_code, "stage": "provider"} + + +def _failed( + custom_id: str, + index: int, + status: Literal["errored", "expired", "cancelled"], + error: Mapping[str, Any], +) -> RawResultItem: + return RawResultItem( + custom_id=custom_id, + index=index, + provider_status=status, + model=None, + usage=None, + finish_reason=None, + error=error, + raw=None, + ) + + +class PydanticAIBatchAdapter(BatchAdapter): + """ + Batch adapter for any provider whose pydantic-ai ``Model`` implements the ``batch_*`` methods. + + .. note:: + + Experimental: this can change or be removed in a minor release of this provider. + See :ref:`howto/stability`. + + The adapter is bound to one connection and one ``model_id``. It builds a fresh ``Model`` for + every operation and runs it in its own event loop, so no client crosses a loop, and it never + keeps a connection open between operations. + + :param hook: The hook the models are built from (credentials, ``provider_kwargs``, the + connection's extra). + :param model_id: The batch's ``"<provider>:<model>"`` id. + :param batch_handle: The serialized ``Batch`` handle recorded when the batch was submitted. + :param output_spec: The task's output spec. Needed only to read results. + :param model_factory: Builds the ``Model`` for a model id instead of the hook. For tests. + """ + + conn_types = frozenset({"pydanticai"}) + #: ``True`` when a key that is not a typed setting is passed to the provider as a raw request field. + accepts_raw_params: ClassVar[bool] = True + #: The key of the batch's ``provider_details`` that names the uploaded input file. + input_file_key: ClassVar[str] = "input_file" + + def __init__( + self, + *, + hook: PydanticAIHook | None = None, + model_id: str | None = None, + batch_handle: str | None = None, + output_spec: OutputSpec | None = None, + model_factory: Callable[[str], Model] | None = None, + ) -> None: + super().__init__() + self._hook = hook + self._model_id = model_id + self._handle = batch_handle + self._output_spec = output_spec or build_output_spec(str) + self._model_factory = model_factory + + @classmethod + def from_connection( + cls, + conn: Connection, + *, + model_id: str | None = None, + batch_handle: str | None = None, + output_spec: OutputSpec | None = None, + ) -> PydanticAIBatchAdapter: + hook = PydanticAIHook.get_hook(conn.conn_id, {"model_id": model_id}) + if hook._get_fallback_conn_ids(): + log.warning( + "Connection has fallback connections configured; batch ignores them and uses the primary", + llm_conn_id=conn.conn_id, + ) + return cls(hook=hook, model_id=model_id, batch_handle=batch_handle, output_spec=output_spec) + + @classmethod + def _settings_type(cls) -> type: + """Return the provider's ``ModelSettings`` class, whose own keys a request may set directly.""" + raise NotImplementedError(f"{cls.__name__} does not name its ModelSettings type.") + + def _require_model_id(self) -> str: + if not self._model_id: + raise LLMBatchInputError( + f"The {self.name!r} batch adapter needs the batch's model_id ('<provider>:<model>') to " + "rebuild its model." + ) + return self._model_id + + async def _amodel(self, model_id: str) -> Model: + if self._model_factory is not None: + return self._model_factory(model_id) + if self._hook is None: + raise LLMBatchInputError(f"The {self.name!r} batch adapter was built without a connection.") + return await self._hook.aget_primary_model(model_id) + + @asynccontextmanager + async def _existing(self, batch_id: str) -> AsyncIterator[tuple[Model, Batch]]: + """ + Yield the model an already-submitted batch belongs to, and its handle. + + The model comes from the batch, not from the task's ``model_id``: pydantic-ai checks the + handle's group on every call, and OpenAI's and Gemini's groups name the model the batch was + submitted on. That differs from the task's model when every request overrode it, or when the + task's model changed since the batch was submitted. Only the provider prefix is taken from + ``model_id``, because the connection that submitted the batch is the one that resolves it. + """ + model_id = self._model_id + if not model_id: + raise LLMBatchStateReadError( + f"Cannot reach batch {batch_id!r}: it was deferred by an earlier version of this provider, " + "which did not record the model it needs. Retry the task so it re-attaches." + ) + if not self._handle: + async with await self._amodel(model_id) as model: + yield model, await self._unrecorded_batch(model, batch_id) + return + batch = _BATCH_ADAPTER.validate_json(self._handle) + provider = model_id.partition(":")[0] + async with await self._amodel(f"{provider}:{batch.model_name}") as model: + yield model, batch + + async def _unrecorded_batch(self, model: Model, batch_id: str) -> Batch: + """Return a handle for a batch recorded without one, by an earlier version of this provider.""" + raise LLMBatchStateReadError( + f"No pydantic-ai batch handle is recorded for batch {batch_id!r}, so it cannot be polled. " + "It was submitted before this provider recorded handles; retry the task so it re-attaches." + ) + + def _build_settings( + self, + model: Model, + *layers: Mapping[str, Any] | None, + max_tokens: int, + ) -> tuple[ModelSettings, set[str]]: + """ + Turn ``request_params`` and a request's ``params`` into ``ModelSettings``. + + Typed keys become settings. Any other key is a raw provider request field and goes through + ``extra_body``, which is how these parameters worked before the adapter used pydantic-ai. + Returns the settings and the managed keys that were dropped. + """ + typed = _typed_settings(self._settings_type()) + merged: dict[str, Any] = {} + for layer in layers: + merged.update(layer or {}) + extra_body: dict[str, Any] = {} + if "extra_body" in merged: + body = merged.pop("extra_body") + if not self.accepts_raw_params: + raise self._unsupported(model, "extra_body") + extra_body.update(body) + settings: dict[str, Any] = {} + for key, value in merged.items(): + if key in typed: + if key in model.unsupported_settings: + raise self._unsupported(model, key) + settings[key] = value + elif self.accepts_raw_params: + extra_body[key] = value + else: + raise LLMBatchInputError( + f"Unknown request parameter {key!r} for the {self.name!r} adapter. It accepts " + f"{sorted(typed)}." + ) + dropped = set(_MANAGED_KEYS & extra_body.keys()) + for key in dropped: + del extra_body[key] + settings["max_tokens"] = max_tokens + if extra_body: + settings["extra_body"] = extra_body + # The keys were checked against the provider's settings above; a TypedDict cannot be built from a dynamic dict. + return cast("ModelSettings", settings), dropped + + @staticmethod + def _unsupported(model: Model, key: str) -> LLMBatchInputError: + return LLMBatchInputError( + f"The request parameter {key!r} is not supported by {type(model).__name__}; " + "pydantic-ai would drop it without a warning." + ) + + @staticmethod + def _output_parameters(model: Model, spec: OutputSpec) -> ModelRequestParameters: + """ + Ask for ``spec``'s schema the strongest way the model supports. + + Native JSON-schema output where the model has it. Otherwise a single forced output tool, + which is a hard format constraint too. Neither is pydantic-ai's profile default + (``tool`` mode with no output tool sends no schema at all), so the mode is set explicitly. + """ + if not spec.is_structured: + return ModelRequestParameters() + if model.profile.get("supports_json_schema_output"): + return ModelRequestParameters( + output_mode="native", + output_object=OutputObjectDefinition(json_schema=spec.json_schema, name=spec.schema_name), + ) + return ModelRequestParameters( + output_mode="tool", + output_tools=[ + ToolDefinition( + name=spec.schema_name, + description="Return the result using this schema.", + parameters_json_schema=spec.json_schema, + kind="output", + ) + ], + allow_text_output=False, + ) + + async def _encode( + self, + stack: AsyncExitStack, + requests: list[BatchRequest], + *, + model: str, + idempotency_key: str, + output_spec: OutputSpec, + system_prompt: str, + max_tokens: int, + request_params: dict[str, Any] | None, + ) -> tuple[Model, str, list[EncodedBatchRequest]]: + """ + Encode every request locally, enforcing the provider's batch limits and a single group. + + Returns request 0's model, its id and the encoded requests. Nothing is sent. + """ + if not requests: + raise LLMBatchInputError("A batch needs at least one request.") + default_id = self._require_model_id() + # Per model id: the model, the output parameters and the settings of a request with no params of its own. + models: dict[str, tuple[Model, ModelRequestParameters, ModelSettings]] = {} + dropped: set[str] = set() + encoded_requests: list[EncodedBatchRequest] = [] + first_id = "" + total_bytes = 0 + for index, request in enumerate(requests): + model_id = default_id + if request_model := request.get("model"): + self.resolve_request_model(request_model, default_bare_model=model, request_index=index) + model_id = request_model + if model_id not in models: + current = await stack.enter_async_context(await self._amodel(model_id)) + settings, dropped_here = self._build_settings(current, request_params, max_tokens=max_tokens) + dropped |= dropped_here + models[model_id] = (current, self._output_parameters(current, output_spec), settings) + current, parameters, settings = models[model_id] + if request.get("params") or request.get("max_tokens"): + settings, dropped_here = self._build_settings( + current, + request_params, + request.get("params"), + max_tokens=request.get("max_tokens") or max_tokens, + ) + dropped |= dropped_here + parts: list[ModelRequestPart] = [] + if system := request.get("system_prompt") or system_prompt: + parts.append(SystemPromptPart(content=system)) + parts.append(UserPromptPart(content=request["prompt"])) + try: + encoded = await current.batch_encode( + PydanticAIBatchRequest( + custom_id=f"{idempotency_key}-{index}", + messages=[ModelRequest(parts=parts)], + model_settings=settings, + model_request_parameters=parameters, + ) + ) + except UserError as e: + raise LLMBatchInputError(f"Request {index} cannot be sent as a batch: {e}") from e + if index == 0: + first_id = model_id + elif encoded.group != encoded_requests[0].group: + raise LLMBatchModelMismatchError( + f"Request {index} cannot share a batch with request 0: they use different models, " + f"credentials or endpoints, and the {self.name!r} batch API takes one of each per batch. " + "Use one model for the whole task, or split the input across tasks." + ) + total_bytes += current.batch_request_size(encoded) + encoded_requests.append(encoded) + if dropped: + log.warning("Ignoring request parameters the batch sets itself", keys=sorted(dropped)) + first = models[first_id][0] + self._check_limits(first, len(requests), total_bytes) + return first, first_id, encoded_requests + + def _check_limits(self, model: Model, request_count: int, total_bytes: int) -> None: + max_requests = model.batch_max_requests + if max_requests is not None and request_count > max_requests: + raise LLMBatchLimitExceededError( + f"The {self.name!r} batch API accepts at most {max_requests} requests; got {request_count}. " + f"Split the input across mapped task instances with .expand() so each instance submits at " + f"most {max_requests} requests." + ) + max_bytes = model.batch_max_bytes + if max_bytes is not None and total_bytes > max_bytes: + raise LLMBatchLimitExceededError( + f"The {self.name!r} batch API accepts a payload of at most {max_bytes} bytes; this input is " + f"{total_bytes} bytes (the output_type schema is repeated in every request). Split the input " + "with .expand()." + ) + + @asynccontextmanager + async def _encoded( + self, + requests: list[BatchRequest], + *, + model: str, + idempotency_key: str, + output_spec: OutputSpec, + system_prompt: str, + max_tokens: int, + request_params: dict[str, Any] | None, + ) -> AsyncIterator[tuple[Model, list[EncodedBatchRequest]]]: + """Yield request 0's model and the encoded requests, with every model open until the block ends.""" + async with AsyncExitStack() as stack: + first, _, encoded = await self._encode( + stack, + requests, + model=model, + idempotency_key=idempotency_key, + output_spec=output_spec, + system_prompt=system_prompt, + max_tokens=max_tokens, + request_params=request_params, + ) + yield first, encoded + + def validate_requests( + self, + requests: list[BatchRequest], + *, + model: str, + output_spec: OutputSpec, + idempotency_key: str, + system_prompt: str = "", + max_tokens: int = 1024, + request_params: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + self.check_custom_id_length(idempotency_key, len(requests)) + + async def validate() -> None: + async with self._encoded( + requests, + model=model, + idempotency_key=idempotency_key, + output_spec=output_spec, + system_prompt=system_prompt, + max_tokens=max_tokens, + request_params=request_params, + ): + pass + + _run(validate()) + + def submit( + self, + requests: list[BatchRequest], + *, + model: str, + idempotency_key: str, + input_fingerprint: str, + output_spec: OutputSpec, + system_prompt: str = "", + max_tokens: int = 1024, + request_params: dict[str, Any] | None = None, + **kwargs: Any, + ) -> SubmitResult: + async def create() -> Batch: + async with self._encoded( + requests, + model=model, + idempotency_key=idempotency_key, + output_spec=output_spec, + system_prompt=system_prompt, + max_tokens=max_tokens, + request_params=request_params, + ) as (submitter, encoded): + return await submitter.batch_create(encoded) + + batch = _run(create()) + self._handle = _BATCH_ADAPTER.dump_json(batch).decode() + return SubmitResult( + batch_id=batch.id, + provider_input_ref=(batch.provider_details or {}).get(self.input_file_key), + handle=self._handle, + ) + + async def aget_batch(self, batch_id: str) -> BatchState: + async with self._existing(batch_id) as (model, stored): + return await self._describe(model, await model.batch_status(stored)) + + async def _describe(self, model: Model, batch: Batch) -> BatchState: + """Reduce a refreshed handle to this adapter's status vocabulary; a provider with finer states extends it.""" + return BatchState( + status="completed" if batch.status == "ended" else "in_progress", counts=None, error_message=None + ) + + def get_batch(self, batch_id: str) -> BatchState: + return _run(self.aget_batch(batch_id)) + + async def acancel_batch(self, batch_id: str) -> None: + async with self._existing(batch_id) as (model, stored): + await model.batch_cancel(stored) + + def cancel_batch(self, batch_id: str) -> None: + _run(self.acancel_batch(batch_id)) + + def find_orphaned_batch( + self, idempotency_key: str, input_fingerprint: str, not_before: str + ) -> str | None: + """ + Return ``None`` unconditionally: ``batch_create`` takes no metadata to find a batch by. + + A crash between "submit sent" and "response recorded" falls through to ``on_orphaned_intent``. + """ + return None + + async def _collect(self, model: Model, batch: Batch) -> list[BatchResult]: + """Return the results of an ended batch; a provider that can end a job with no results file overrides it.""" + return await model.batch_results(batch) + + async def _stream_results(self, model: Model, batch: Batch) -> AsyncGenerator[BatchResult, None]: + """ + Yield the results of an ended batch one at a time. + + The default collects them as one list with ``batch_results``, which is all a provider whose SDK + downloads the results in one call can do (Gemini). A provider that can read its results file as a + stream overrides this. + """ + results = await self._collect(model, batch) + # Popping frees each result once it is handed on; the order does not matter, the merge is by index. + while results: + yield results.pop() + + async def _read_results(self, batch_id: str) -> AsyncGenerator[_ResultsContext | BatchResult, None]: + """Yield what decoding needs, then the raw results of an ended batch; decoding runs on the reader's thread.""" + async with self._existing(batch_id) as (model, stored): + # ``batch_results`` refuses a handle that has not ended, and the stored one is from submit time. + batch = await model.batch_status(stored) + if batch.status != "ended": + raise LLMBatchJobError(f"Batch {batch_id!r} has not ended (status {batch.status!r}).") + yield _ResultsContext(model, self._output_parameters(model, self._output_spec)) + async with aclosing(self._stream_results(model, batch)) as results: + async for result in results: + yield result + + def iter_results(self, batch_id: str) -> Iterator[RawResultItem]: + # Only the download crosses to the helper thread. Decoding is synchronous and does no I/O, so it + # stays here with the task's logging context. + with closing(_stream(self._read_results(batch_id))) as stream: + context = cast("_ResultsContext", next(stream)) + for result in stream: + row = self._decode_or_skip( + context.model, cast("BatchResult", result), context.parameters, batch_id + ) + if row is not None: + yield row + + def _decode_or_skip( + self, model: Model, result: BatchResult, parameters: ModelRequestParameters, batch_id: str + ) -> RawResultItem | None: + try: + return self._decode_one(model, result, parameters) + except Exception as e: + # One malformed item must not abort the whole batch; the merge counts the index as missing. + log.warning( + "Skipping unparsable result item for batch", + batch_id=batch_id, + custom_id=result.custom_id, + error=str(e), + ) + return None + + @staticmethod + def _decode_one(model: Model, result: BatchResult, parameters: ModelRequestParameters) -> RawResultItem: + custom_id = result.custom_id + index = int(custom_id.rsplit("-", 1)[1]) + if result.status in _UNRUN_STATUS: + return _failed( + custom_id, + index, + _UNRUN_STATUS[result.status], + _error_info(result.status, f"batch item {result.status}"), + ) + try: + response = model.batch_decode( + result, + PydanticAIBatchRequest(custom_id=custom_id, messages=[], model_request_parameters=parameters), + ) + except ModelAPIError as e: + provider_code = None + if isinstance(e, ModelHTTPError): + provider_code = _error_body(e).get("type") or e.status_code + return _failed(custom_id, index, "errored", _error_info(result.status, e.message, provider_code)) + details = response.provider_details or {} + return RawResultItem( + custom_id=custom_id, + index=index, + provider_status="success", + model=response.model_name, + usage={ + "input_tokens": response.usage.input_tokens, + "output_tokens": response.usage.output_tokens, + }, + finish_reason=details.get("finish_reason") or response.finish_reason, + error=None, + raw=response, + ) + + def extract_output(self, raw: RawResultItem, spec: OutputSpec) -> ExtractedOutput: + response: ModelResponse = raw.raw + if not spec.is_structured: + return ExtractedOutput(kind="text", text=response.text or None) + if response.tool_calls: + return ExtractedOutput(kind="json_value", value=response.tool_calls[0].args_as_dict()) + # Native structured output arrives as the response text, which is the JSON document. + if response.text: + return ExtractedOutput(kind="json_text", text=response.text) + return ExtractedOutput(kind="absent") + + +class OpenAIBatchAdapter(PydanticAIBatchAdapter): + """ + Batch adapter for OpenAI's Batch API, on the Responses API (``/v1/responses``). + + pydantic-ai batches only its Responses model, so an ``openai-chat:`` model id is built as the + ``openai:`` model of the same name and logged. Azure OpenAI has no adapter. A batch's group + includes the model, so every request must use the same one. Raw Responses fields in + ``request_params`` (``user``, ``metadata``, ...) go through ``extra_body``; the typed form of + a setting is preferred where one exists (``openai_reasoning_effort``, not ``reasoning_effort``). + + A batch gateway that implements ``/v1/files`` and ``/v1/batches`` for ``/v1/responses`` works + through the connection's ``host``; one that implements those only for ``/v1/chat/completions`` + does not. + """ + + name = "openai" + #: OpenAI's documented limit: 50,000 requests or 200 MB, whichever is reached first. + max_requests = 50_000 + max_payload_bytes = 200_000_000 + allows_per_request_model = False + input_file_key = "input_file_id" + + #: Job statuses that end a batch, as the status this provider reports. A job that expired or was + #: cancelled still yields a row per request, and those rows carry the counts. + _ENDED_STATUSES: ClassVar[dict[str, BatchStatus]] = { + "failed": "failed", + "expired": "expired", + "cancelled": "cancelled", + } + + @classmethod + def _settings_type(cls) -> type: + # Heavy third-party SDK: importing the OpenAI model module imports ``openai``. + from pydantic_ai.models.openai import OpenAIResponsesModelSettings + + return OpenAIResponsesModelSettings + + @staticmethod + def _responses_model_id(model_id: str) -> str: + """Return ``model_id`` with pydantic-ai's Chat Completions prefix swapped for the Responses one.""" + prefix, sep, bare = model_id.partition(":") + if sep and prefix == "openai-chat": + log.info( + "Batches run on the OpenAI Responses API; using openai:<model> in place of openai-chat:<model>", + model=bare, + ) + return f"openai:{bare}" + return model_id + + async def _amodel(self, model_id: str) -> Model: + return await super()._amodel(self._responses_model_id(model_id)) + + async def _stream_results(self, model: Model, batch: Batch) -> AsyncGenerator[BatchResult, None]: + # Heavy third-party SDK: importing the OpenAI model module imports ``openai``. + from pydantic_ai.models.openai import OpenAIResponsesModel + + if not isinstance(model, OpenAIResponsesModel) or model.batch_max_requests is None: + # Not a model pydantic-ai batches through: its collect raises the right error. + async for collected in super()._stream_results(model, batch): + yield collected + return + # ``batch_status`` has already checked the credentials and the group, as ``batch_results`` would. + # The handle of an ended batch names its final files, as it does for pydantic-ai's own collect. + details = batch.provider_details or {} + # Only the ids seen are kept, to find the requests the batch never ran. + seen: set[str] = set() + for file_id in (details.get("output_file_id"), details.get("error_file_id")): + if file_id: + async with aclosing(self._file_lines(model, file_id)) as lines: + async for line in lines: + result = _openai_result(_OpenAIOutputLine.model_validate_json(line)) + seen.add(result.custom_id) + yield result + if details["status"] != "completed": + # The output and error files have no line for a request the batch didn't run, so the input file lists them. + async with aclosing(self._file_lines(model, details["input_file_id"])) as lines: + line_number = 0 + async for line in lines: + line_number += 1 + if (custom_id := _OpenAIInputLine.model_validate_json(line).custom_id) not in seen: + yield _openai_unrun_result(details, custom_id, line_number) + + @staticmethod + async def _file_lines(model: OpenAIResponsesModel, file_id: str) -> AsyncGenerator[str, None]: + """Yield a file's lines as they arrive; the response is closed when the iteration ends or is closed.""" + async with model.client.files.with_streaming_response.content(file_id) as response: + async for line in response.iter_lines(): + yield line + + async def _unrecorded_batch(self, model: Model, batch_id: str) -> Batch: Review Comment: Good point, cancel only needs the id. With no handle the OpenAI adapter now calls `client.batches.cancel(batch_id)` directly; polling and collecting such a batch still raise the state error. ########## providers/common/ai/src/airflow/providers/common/ai/batch/pydantic_ai_backend.py: ########## @@ -0,0 +1,1128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Batch adapters built on pydantic-ai's public ``Model.batch_*`` methods. + +One backend drives every provider pydantic-ai can batch: it encodes each request with +``batch_encode``, submits them with ``batch_create``, polls with ``batch_status``, collects with +``batch_results`` and reads each result with ``batch_decode``. The provider subclasses set only +what differs between providers (the settings they accept and how a finished job is read). + +pydantic-ai's ``Batch`` handle is plain data. It is stored with the batch's state record, so the +trigger and a later worker rebuild a model from the connection and continue the same batch without +the original process. + +Importing this module never imports a provider SDK: the model classes are built by +:class:`~airflow.providers.common.ai.hooks.pydantic_ai.PydanticAIHook`, and the SDK-specific +settings types are imported inside the methods that need them. +""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import dataclasses +import functools +import queue +import threading +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Coroutine, Generator, Iterator, Mapping +from contextlib import AsyncExitStack, aclosing, asynccontextmanager, closing, suppress +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar, cast + +from pydantic import BaseModel, JsonValue, TypeAdapter +from pydantic_ai.exceptions import ModelAPIError, ModelHTTPError, UserError +from pydantic_ai.messages import ( + ModelRequest, + ModelRequestPart, + ModelResponse, + SystemPromptPart, + UserPromptPart, +) +from pydantic_ai.models import ModelRequestParameters + +# pydantic-ai raises this for a job that ended with no results file. Nothing public tells it from an API failure +# (the handle carries no result destination), and a test pins that it stays a ``ModelHTTPError``. +from pydantic_ai.models._batch_codec import BatchJobError +from pydantic_ai.models.batch import ( + Batch, + BatchRequest as PydanticAIBatchRequest, + BatchResult, + EncodedBatchRequest, +) +from pydantic_ai.output import OutputObjectDefinition +from pydantic_ai.settings import ModelSettings +from pydantic_ai.tools import ToolDefinition +from pydantic_core import to_json + +from airflow.providers.common.ai.batch.base import ( + BatchAdapter, + BatchState, + ExtractedOutput, + RawResultItem, + SubmitResult, +) +from airflow.providers.common.ai.batch.output_schema import build_output_spec +from airflow.providers.common.ai.exceptions import ( + LLMBatchInputError, + LLMBatchJobError, + LLMBatchLimitExceededError, + LLMBatchModelMismatchError, + LLMBatchStateReadError, +) +from airflow.providers.common.ai.hooks.pydantic_ai import PydanticAIHook +from airflow.providers.common.ai.utils.task_logger import get_task_logger + +if TYPE_CHECKING: + from pydantic_ai.models import Model + from pydantic_ai.models.openai import OpenAIResponsesModel + + from airflow.providers.common.ai.batch.base import BatchRequest, BatchStatus + from airflow.providers.common.ai.batch.output_schema import OutputSpec + from airflow.sdk import Connection + +log = get_task_logger() + +T = TypeVar("T") + +_BATCH_ADAPTER: TypeAdapter[Batch] = TypeAdapter(Batch) + +#: Sampling settings with the same meaning on every provider. A key in this set (or one of a +#: provider's own ``ModelSettings`` keys) becomes a typed setting; any other key is a raw +#: provider request field. +_GENERIC_SETTINGS = frozenset( + { + "temperature", + "top_p", + "top_k", + "seed", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "stop_sequences", + } +) + +#: Request fields this adapter sets itself. Taking them from ``request_params`` would replace the +#: prompt, the model or the structured-output constraint, so they are dropped with a warning. +_MANAGED_KEYS = frozenset( + { + "model", + "messages", + "input", + "system", + "system_instruction", + "contents", + "tools", + "tool_choice", + "stream", + "max_tokens", + "max_output_tokens", + "max_completion_tokens", + "output_config", + "text", + "response_format", + } +) + +#: pydantic-ai's status for a request that never ran, as this provider names it. +_UNRUN_STATUS: dict[str, Literal["cancelled", "expired"]] = {"canceled": "cancelled", "expired": "expired"} + +#: The error codes of a request an OpenAI batch ended before running, and the status pydantic-ai gives each. +#: Mirrors ``_UNRUN_STATUSES`` in pydantic-ai's OpenAI batch module. +_OPENAI_UNRUN_STATUSES: dict[str, Literal["expired", "canceled"]] = { + "batch_expired": "expired", + "batch_cancelled": "canceled", +} + +#: Gemini documents no request limit, and the ABC wants an int. +_NO_REQUEST_LIMIT = 2**63 - 1 + +#: Decoded rows a batch's results may run ahead of the task that reads them. +_RESULT_BUFFER = 64 + +#: Seconds a closed results stream waits for its helper thread to finish after cancelling it. +_STREAM_JOIN_TIMEOUT = 5.0 + + [email protected](frozen=True) +class _ResultsContext: + """What a reader needs, besides the raw results, to decode them.""" + + model: Model + parameters: ModelRequestParameters + + +def _run(coro: Coroutine[Any, Any, T]) -> T: + """ + Run ``coro`` to completion from synchronous code. + + ``asyncio.run`` refuses to start inside a thread that already runs a loop, which is where a + kill signal's ``on_kill`` can land while a poll is in flight, so that case uses a helper thread. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + # No loop in this thread. Run outside the ``except`` so an error from ``coro`` isn't chained + # onto this probe in the task log. + pass + else: + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(asyncio.run, coro).result() + return asyncio.run(coro) + + +class _OpenAILineResponse(BaseModel): + status_code: int + body: JsonValue + + +class _OpenAILineError(BaseModel): + code: str | None = None + message: str | None = None + + +class _OpenAIOutputLine(BaseModel): + """A line of an OpenAI batch's output or error file.""" + + custom_id: str + response: _OpenAILineResponse | None = None + error: _OpenAILineError | None = None + + +class _OpenAIInputLine(BaseModel): + custom_id: str + + +def _openai_result(line: _OpenAIOutputLine) -> BatchResult: + """Mirror of ``_to_result`` in pydantic-ai's OpenAI batch module.""" + if (response := line.response) is not None: + if 200 <= response.status_code < 300: + return BatchResult( + custom_id=line.custom_id, status="succeeded", body=to_json(response.body).decode() + ) + return BatchResult(custom_id=line.custom_id, status="errored", body=response.model_dump_json()) + error = line.error or _OpenAILineError() + if status := _OPENAI_UNRUN_STATUSES.get(error.code or ""): + return BatchResult(custom_id=line.custom_id, status=status) + body = _OpenAILineResponse(status_code=500, body={"error": error.model_dump(mode="json")}) + return BatchResult(custom_id=line.custom_id, status="errored", body=body.model_dump_json()) + + +def _openai_unrun_result(details: Mapping[str, Any], custom_id: str, line_number: int) -> BatchResult: + """Mirror of ``_unrun_result`` in pydantic-ai's OpenAI batch module.""" + if details["status"] == "failed": + # The input file failed validation, so the batch ran none of its requests. A request gets the errors of its + # own line and those of no line; when they're all about other lines, it gets them all, which name them. + errors = details.get("errors", []) + own = [ + error + for error in errors + if not isinstance(error, Mapping) or error.get("line") in (line_number, None) + ] + body = _OpenAILineResponse(status_code=400, body={"errors": own or errors}) + return BatchResult(custom_id=custom_id, status="errored", body=body.model_dump_json()) + return BatchResult( + custom_id=custom_id, status="expired" if details["status"] == "expired" else "canceled" + ) + + +def _stream(source: AsyncGenerator[T, None]) -> Generator[T, None, None]: + """ + Read an async generator from synchronous code without collecting it first. + + The generator runs as a task on its own event loop in a helper thread and hands items over through a + queue of :data:`_RESULT_BUFFER` items, so a slow reader holds the download back instead of buffering it. + An error from the generator is raised at the reader, whatever its type. + + When the reader stops (closes the iterator, is dropped, or fails), the task is cancelled from the reader's + thread. A read the generator is awaiting is interrupted rather than waited for, and the generator's own + cleanup still runs, so whatever it holds open is closed. The wait for the thread is bounded. + """ + items: queue.Queue[ + tuple[Literal["item"], T] | tuple[Literal["error"], BaseException] | tuple[Literal["end"], None] + ] + items = queue.Queue(maxsize=_RESULT_BUFFER) + closed = threading.Event() + started = threading.Event() + running: list[tuple[asyncio.AbstractEventLoop, asyncio.Task[None]]] = [] + + def put(message: tuple[Any, Any]) -> bool: + # A bounded put that gives up once the reader is gone, so a full queue never strands the thread. + while not closed.is_set(): + try: + items.put(message, timeout=0.1) + return True + except queue.Full: + continue + return False + + async def produce() -> None: + try: + async for item in source: + if not put(("item", item)): + return + put(("end", None)) + except BaseException as e: + # Also a cancellation the reader did not ask for: forwarding it keeps the reader from waiting forever. + put(("error", e)) + finally: + await source.aclose() + + async def main() -> None: + task = asyncio.ensure_future(produce()) + running.append((asyncio.get_running_loop(), task)) + started.set() + await asyncio.wait([task]) + + thread = threading.Thread(target=asyncio.run, args=(main(),), name="llm-batch-results", daemon=True) + thread.start() + try: + while True: + try: + message = items.get(timeout=1) + except queue.Empty: + if not thread.is_alive() and items.empty(): + raise RuntimeError("The batch results reader stopped without finishing.") from None + continue + if message[0] == "item": + yield message[1] + elif message[0] == "error": + raise message[1] + else: + return + finally: + closed.set() + if started.wait(timeout=1): + loop, task = running[0] + # A RuntimeError means the loop already finished, so there is nothing left to cancel. + with suppress(RuntimeError): + loop.call_soon_threadsafe(task.cancel) + thread.join(timeout=_STREAM_JOIN_TIMEOUT) + if thread.is_alive(): + log.debug( + "Batch results thread did not stop within the timeout; it is a daemon and will be left behind" + ) + + [email protected] +def _typed_settings(settings_type: type) -> frozenset[str]: + """Return the keys a request may set as typed settings: the generic ones and the provider's own.""" + return _GENERIC_SETTINGS | (set(settings_type.__annotations__) - set(ModelSettings.__annotations__)) + + +def _error_body(error: ModelHTTPError) -> Mapping[str, Any]: + """Return the ``error`` object of an HTTP error's body, which carries the provider's own type and message.""" + body = error.body + inner = body.get("error") if isinstance(body, Mapping) else None + return inner if isinstance(inner, Mapping) else {} + + +def _reason(error: ModelHTTPError) -> str: + """Return the provider's own explanation of an HTTP error; pydantic-ai's message wraps it in status and model.""" + return str(_error_body(error).get("message") or error.message) + + +def _error_info(status: str, message: str, provider_code: str | int | None = None) -> dict[str, Any]: + return {"type": status, "message": message, "provider_code": provider_code, "stage": "provider"} + + +def _failed( + custom_id: str, + index: int, + status: Literal["errored", "expired", "cancelled"], + error: Mapping[str, Any], +) -> RawResultItem: + return RawResultItem( + custom_id=custom_id, + index=index, + provider_status=status, + model=None, + usage=None, + finish_reason=None, + error=error, + raw=None, + ) + + +class PydanticAIBatchAdapter(BatchAdapter): + """ + Batch adapter for any provider whose pydantic-ai ``Model`` implements the ``batch_*`` methods. + + .. note:: + + Experimental: this can change or be removed in a minor release of this provider. + See :ref:`howto/stability`. + + The adapter is bound to one connection and one ``model_id``. It builds a fresh ``Model`` for + every operation and runs it in its own event loop, so no client crosses a loop, and it never + keeps a connection open between operations. + + :param hook: The hook the models are built from (credentials, ``provider_kwargs``, the + connection's extra). + :param model_id: The batch's ``"<provider>:<model>"`` id. + :param batch_handle: The serialized ``Batch`` handle recorded when the batch was submitted. + :param output_spec: The task's output spec. Needed only to read results. + :param model_factory: Builds the ``Model`` for a model id instead of the hook. For tests. + """ + + conn_types = frozenset({"pydanticai"}) + #: ``True`` when a key that is not a typed setting is passed to the provider as a raw request field. + accepts_raw_params: ClassVar[bool] = True + #: The key of the batch's ``provider_details`` that names the uploaded input file. + input_file_key: ClassVar[str] = "input_file" + + def __init__( + self, + *, + hook: PydanticAIHook | None = None, + model_id: str | None = None, + batch_handle: str | None = None, + output_spec: OutputSpec | None = None, + model_factory: Callable[[str], Model] | None = None, + ) -> None: + super().__init__() + self._hook = hook + self._model_id = model_id + self._handle = batch_handle + self._output_spec = output_spec or build_output_spec(str) + self._model_factory = model_factory + + @classmethod + def from_connection( + cls, + conn: Connection, + *, + model_id: str | None = None, + batch_handle: str | None = None, + output_spec: OutputSpec | None = None, + ) -> PydanticAIBatchAdapter: + hook = PydanticAIHook.get_hook(conn.conn_id, {"model_id": model_id}) + if hook._get_fallback_conn_ids(): + log.warning( + "Connection has fallback connections configured; batch ignores them and uses the primary", + llm_conn_id=conn.conn_id, + ) + return cls(hook=hook, model_id=model_id, batch_handle=batch_handle, output_spec=output_spec) + + @classmethod + def _settings_type(cls) -> type: + """Return the provider's ``ModelSettings`` class, whose own keys a request may set directly.""" + raise NotImplementedError(f"{cls.__name__} does not name its ModelSettings type.") + + def _require_model_id(self) -> str: + if not self._model_id: + raise LLMBatchInputError( + f"The {self.name!r} batch adapter needs the batch's model_id ('<provider>:<model>') to " + "rebuild its model." + ) + return self._model_id + + async def _amodel(self, model_id: str) -> Model: + if self._model_factory is not None: + return self._model_factory(model_id) + if self._hook is None: + raise LLMBatchInputError(f"The {self.name!r} batch adapter was built without a connection.") + return await self._hook.aget_primary_model(model_id) + + @asynccontextmanager + async def _existing(self, batch_id: str) -> AsyncIterator[tuple[Model, Batch]]: + """ + Yield the model an already-submitted batch belongs to, and its handle. + + The model comes from the batch, not from the task's ``model_id``: pydantic-ai checks the + handle's group on every call, and OpenAI's and Gemini's groups name the model the batch was + submitted on. That differs from the task's model when every request overrode it, or when the + task's model changed since the batch was submitted. Only the provider prefix is taken from + ``model_id``, because the connection that submitted the batch is the one that resolves it. + """ + model_id = self._model_id + if not model_id: + raise LLMBatchStateReadError( + f"Cannot reach batch {batch_id!r}: it was deferred by an earlier version of this provider, " + "which did not record the model it needs. Retry the task so it re-attaches." + ) + if not self._handle: + async with await self._amodel(model_id) as model: + yield model, await self._unrecorded_batch(model, batch_id) + return + batch = _BATCH_ADAPTER.validate_json(self._handle) + provider = model_id.partition(":")[0] + async with await self._amodel(f"{provider}:{batch.model_name}") as model: + yield model, batch + + async def _unrecorded_batch(self, model: Model, batch_id: str) -> Batch: + """Return a handle for a batch recorded without one, by an earlier version of this provider.""" + raise LLMBatchStateReadError( + f"No pydantic-ai batch handle is recorded for batch {batch_id!r}, so it cannot be polled. " + "It was submitted before this provider recorded handles; retry the task so it re-attaches." + ) + + def _build_settings( + self, + model: Model, + *layers: Mapping[str, Any] | None, + max_tokens: int, + ) -> tuple[ModelSettings, set[str]]: + """ + Turn ``request_params`` and a request's ``params`` into ``ModelSettings``. + + Typed keys become settings. Any other key is a raw provider request field and goes through + ``extra_body``, which is how these parameters worked before the adapter used pydantic-ai. + Returns the settings and the managed keys that were dropped. + """ + typed = _typed_settings(self._settings_type()) + merged: dict[str, Any] = {} + for layer in layers: + merged.update(layer or {}) + extra_body: dict[str, Any] = {} + if "extra_body" in merged: + body = merged.pop("extra_body") + if not self.accepts_raw_params: + raise self._unsupported(model, "extra_body") + extra_body.update(body) + settings: dict[str, Any] = {} + for key, value in merged.items(): + if key in typed: + if key in model.unsupported_settings: + raise self._unsupported(model, key) + settings[key] = value + elif self.accepts_raw_params: + extra_body[key] = value + else: + raise LLMBatchInputError( + f"Unknown request parameter {key!r} for the {self.name!r} adapter. It accepts " + f"{sorted(typed)}." + ) + dropped = set(_MANAGED_KEYS & extra_body.keys()) + for key in dropped: + del extra_body[key] + settings["max_tokens"] = max_tokens + if extra_body: + settings["extra_body"] = extra_body + # The keys were checked against the provider's settings above; a TypedDict cannot be built from a dynamic dict. + return cast("ModelSettings", settings), dropped + + @staticmethod + def _unsupported(model: Model, key: str) -> LLMBatchInputError: + return LLMBatchInputError( + f"The request parameter {key!r} is not supported by {type(model).__name__}; " + "pydantic-ai would drop it without a warning." + ) + + @staticmethod + def _output_parameters(model: Model, spec: OutputSpec) -> ModelRequestParameters: + """ + Ask for ``spec``'s schema the strongest way the model supports. + + Native JSON-schema output where the model has it. Otherwise a single forced output tool, + which is a hard format constraint too. Neither is pydantic-ai's profile default + (``tool`` mode with no output tool sends no schema at all), so the mode is set explicitly. + """ + if not spec.is_structured: + return ModelRequestParameters() + if model.profile.get("supports_json_schema_output"): + return ModelRequestParameters( + output_mode="native", + output_object=OutputObjectDefinition(json_schema=spec.json_schema, name=spec.schema_name), + ) + return ModelRequestParameters( + output_mode="tool", + output_tools=[ + ToolDefinition( + name=spec.schema_name, + description="Return the result using this schema.", + parameters_json_schema=spec.json_schema, + kind="output", + ) + ], + allow_text_output=False, + ) + + async def _encode( + self, + stack: AsyncExitStack, + requests: list[BatchRequest], + *, + model: str, + idempotency_key: str, + output_spec: OutputSpec, + system_prompt: str, + max_tokens: int, + request_params: dict[str, Any] | None, + ) -> tuple[Model, str, list[EncodedBatchRequest]]: + """ + Encode every request locally, enforcing the provider's batch limits and a single group. + + Returns request 0's model, its id and the encoded requests. Nothing is sent. + """ + if not requests: + raise LLMBatchInputError("A batch needs at least one request.") + default_id = self._require_model_id() + # Per model id: the model, the output parameters and the settings of a request with no params of its own. + models: dict[str, tuple[Model, ModelRequestParameters, ModelSettings]] = {} + dropped: set[str] = set() + encoded_requests: list[EncodedBatchRequest] = [] + first_id = "" + total_bytes = 0 + for index, request in enumerate(requests): + model_id = default_id + if request_model := request.get("model"): + self.resolve_request_model(request_model, default_bare_model=model, request_index=index) + model_id = request_model + if model_id not in models: + current = await stack.enter_async_context(await self._amodel(model_id)) + settings, dropped_here = self._build_settings(current, request_params, max_tokens=max_tokens) + dropped |= dropped_here + models[model_id] = (current, self._output_parameters(current, output_spec), settings) + current, parameters, settings = models[model_id] + if request.get("params") or request.get("max_tokens"): + settings, dropped_here = self._build_settings( + current, + request_params, + request.get("params"), + max_tokens=request.get("max_tokens") or max_tokens, + ) + dropped |= dropped_here + parts: list[ModelRequestPart] = [] + if system := request.get("system_prompt") or system_prompt: + parts.append(SystemPromptPart(content=system)) + parts.append(UserPromptPart(content=request["prompt"])) + try: + encoded = await current.batch_encode( + PydanticAIBatchRequest( + custom_id=f"{idempotency_key}-{index}", + messages=[ModelRequest(parts=parts)], + model_settings=settings, + model_request_parameters=parameters, + ) + ) + except UserError as e: + raise LLMBatchInputError(f"Request {index} cannot be sent as a batch: {e}") from e + if index == 0: + first_id = model_id + elif encoded.group != encoded_requests[0].group: + raise LLMBatchModelMismatchError( + f"Request {index} cannot share a batch with request 0: they use different models, " + f"credentials or endpoints, and the {self.name!r} batch API takes one of each per batch. " + "Use one model for the whole task, or split the input across tasks." + ) + total_bytes += current.batch_request_size(encoded) + encoded_requests.append(encoded) + if dropped: + log.warning("Ignoring request parameters the batch sets itself", keys=sorted(dropped)) + first = models[first_id][0] + self._check_limits(first, len(requests), total_bytes) + return first, first_id, encoded_requests + + def _check_limits(self, model: Model, request_count: int, total_bytes: int) -> None: + max_requests = model.batch_max_requests + if max_requests is not None and request_count > max_requests: + raise LLMBatchLimitExceededError( + f"The {self.name!r} batch API accepts at most {max_requests} requests; got {request_count}. " + f"Split the input across mapped task instances with .expand() so each instance submits at " + f"most {max_requests} requests." + ) + max_bytes = model.batch_max_bytes + if max_bytes is not None and total_bytes > max_bytes: + raise LLMBatchLimitExceededError( + f"The {self.name!r} batch API accepts a payload of at most {max_bytes} bytes; this input is " + f"{total_bytes} bytes (the output_type schema is repeated in every request). Split the input " + "with .expand()." + ) + + @asynccontextmanager + async def _encoded( + self, + requests: list[BatchRequest], + *, + model: str, + idempotency_key: str, + output_spec: OutputSpec, + system_prompt: str, + max_tokens: int, + request_params: dict[str, Any] | None, + ) -> AsyncIterator[tuple[Model, list[EncodedBatchRequest]]]: + """Yield request 0's model and the encoded requests, with every model open until the block ends.""" + async with AsyncExitStack() as stack: + first, _, encoded = await self._encode( + stack, + requests, + model=model, + idempotency_key=idempotency_key, + output_spec=output_spec, + system_prompt=system_prompt, + max_tokens=max_tokens, + request_params=request_params, + ) + yield first, encoded + + def validate_requests( + self, + requests: list[BatchRequest], + *, + model: str, + output_spec: OutputSpec, + idempotency_key: str, + system_prompt: str = "", + max_tokens: int = 1024, + request_params: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + self.check_custom_id_length(idempotency_key, len(requests)) + + async def validate() -> None: + async with self._encoded( + requests, + model=model, + idempotency_key=idempotency_key, + output_spec=output_spec, + system_prompt=system_prompt, + max_tokens=max_tokens, + request_params=request_params, + ): + pass + + _run(validate()) + + def submit( + self, + requests: list[BatchRequest], + *, + model: str, + idempotency_key: str, + input_fingerprint: str, + output_spec: OutputSpec, + system_prompt: str = "", + max_tokens: int = 1024, + request_params: dict[str, Any] | None = None, + **kwargs: Any, + ) -> SubmitResult: + async def create() -> Batch: + async with self._encoded( + requests, + model=model, + idempotency_key=idempotency_key, + output_spec=output_spec, + system_prompt=system_prompt, + max_tokens=max_tokens, + request_params=request_params, + ) as (submitter, encoded): + return await submitter.batch_create(encoded) + + batch = _run(create()) + self._handle = _BATCH_ADAPTER.dump_json(batch).decode() + return SubmitResult( + batch_id=batch.id, + provider_input_ref=(batch.provider_details or {}).get(self.input_file_key), + handle=self._handle, + ) + + async def aget_batch(self, batch_id: str) -> BatchState: + async with self._existing(batch_id) as (model, stored): + return await self._describe(model, await model.batch_status(stored)) + + async def _describe(self, model: Model, batch: Batch) -> BatchState: + """Reduce a refreshed handle to this adapter's status vocabulary; a provider with finer states extends it.""" + return BatchState( + status="completed" if batch.status == "ended" else "in_progress", counts=None, error_message=None + ) + + def get_batch(self, batch_id: str) -> BatchState: + return _run(self.aget_batch(batch_id)) + + async def acancel_batch(self, batch_id: str) -> None: + async with self._existing(batch_id) as (model, stored): + await model.batch_cancel(stored) + + def cancel_batch(self, batch_id: str) -> None: + _run(self.acancel_batch(batch_id)) + + def find_orphaned_batch( + self, idempotency_key: str, input_fingerprint: str, not_before: str + ) -> str | None: + """ + Return ``None`` unconditionally: ``batch_create`` takes no metadata to find a batch by. + + A crash between "submit sent" and "response recorded" falls through to ``on_orphaned_intent``. + """ + return None + + async def _collect(self, model: Model, batch: Batch) -> list[BatchResult]: + """Return the results of an ended batch; a provider that can end a job with no results file overrides it.""" + return await model.batch_results(batch) + + async def _stream_results(self, model: Model, batch: Batch) -> AsyncGenerator[BatchResult, None]: + """ + Yield the results of an ended batch one at a time. + + The default collects them as one list with ``batch_results``, which is all a provider whose SDK + downloads the results in one call can do (Gemini). A provider that can read its results file as a + stream overrides this. + """ + results = await self._collect(model, batch) + # Popping frees each result once it is handed on; the order does not matter, the merge is by index. + while results: + yield results.pop() + + async def _read_results(self, batch_id: str) -> AsyncGenerator[_ResultsContext | BatchResult, None]: + """Yield what decoding needs, then the raw results of an ended batch; decoding runs on the reader's thread.""" + async with self._existing(batch_id) as (model, stored): + # ``batch_results`` refuses a handle that has not ended, and the stored one is from submit time. + batch = await model.batch_status(stored) + if batch.status != "ended": + raise LLMBatchJobError(f"Batch {batch_id!r} has not ended (status {batch.status!r}).") + yield _ResultsContext(model, self._output_parameters(model, self._output_spec)) + async with aclosing(self._stream_results(model, batch)) as results: + async for result in results: + yield result + + def iter_results(self, batch_id: str) -> Iterator[RawResultItem]: + # Only the download crosses to the helper thread. Decoding is synchronous and does no I/O, so it + # stays here with the task's logging context. + with closing(_stream(self._read_results(batch_id))) as stream: + context = cast("_ResultsContext", next(stream)) + for result in stream: + row = self._decode_or_skip( + context.model, cast("BatchResult", result), context.parameters, batch_id + ) + if row is not None: + yield row + + def _decode_or_skip( + self, model: Model, result: BatchResult, parameters: ModelRequestParameters, batch_id: str + ) -> RawResultItem | None: + try: + return self._decode_one(model, result, parameters) + except Exception as e: + # One malformed item must not abort the whole batch; the merge counts the index as missing. + log.warning( + "Skipping unparsable result item for batch", + batch_id=batch_id, + custom_id=result.custom_id, + error=str(e), + ) + return None + + @staticmethod + def _decode_one(model: Model, result: BatchResult, parameters: ModelRequestParameters) -> RawResultItem: + custom_id = result.custom_id + index = int(custom_id.rsplit("-", 1)[1]) + if result.status in _UNRUN_STATUS: + return _failed( + custom_id, + index, + _UNRUN_STATUS[result.status], + _error_info(result.status, f"batch item {result.status}"), + ) + try: + response = model.batch_decode( + result, + PydanticAIBatchRequest(custom_id=custom_id, messages=[], model_request_parameters=parameters), + ) + except ModelAPIError as e: + provider_code = None + if isinstance(e, ModelHTTPError): + provider_code = _error_body(e).get("type") or e.status_code + return _failed(custom_id, index, "errored", _error_info(result.status, e.message, provider_code)) + details = response.provider_details or {} + return RawResultItem( + custom_id=custom_id, + index=index, + provider_status="success", + model=response.model_name, + usage={ + "input_tokens": response.usage.input_tokens, + "output_tokens": response.usage.output_tokens, + }, + finish_reason=details.get("finish_reason") or response.finish_reason, + error=None, + raw=response, + ) + + def extract_output(self, raw: RawResultItem, spec: OutputSpec) -> ExtractedOutput: + response: ModelResponse = raw.raw + if not spec.is_structured: + return ExtractedOutput(kind="text", text=response.text or None) + if response.tool_calls: + return ExtractedOutput(kind="json_value", value=response.tool_calls[0].args_as_dict()) + # Native structured output arrives as the response text, which is the JSON document. + if response.text: + return ExtractedOutput(kind="json_text", text=response.text) + return ExtractedOutput(kind="absent") + + +class OpenAIBatchAdapter(PydanticAIBatchAdapter): + """ + Batch adapter for OpenAI's Batch API, on the Responses API (``/v1/responses``). + + pydantic-ai batches only its Responses model, so an ``openai-chat:`` model id is built as the + ``openai:`` model of the same name and logged. Azure OpenAI has no adapter. A batch's group + includes the model, so every request must use the same one. Raw Responses fields in + ``request_params`` (``user``, ``metadata``, ...) go through ``extra_body``; the typed form of + a setting is preferred where one exists (``openai_reasoning_effort``, not ``reasoning_effort``). + + A batch gateway that implements ``/v1/files`` and ``/v1/batches`` for ``/v1/responses`` works + through the connection's ``host``; one that implements those only for ``/v1/chat/completions`` + does not. + """ + + name = "openai" + #: OpenAI's documented limit: 50,000 requests or 200 MB, whichever is reached first. + max_requests = 50_000 + max_payload_bytes = 200_000_000 + allows_per_request_model = False + input_file_key = "input_file_id" + + #: Job statuses that end a batch, as the status this provider reports. A job that expired or was + #: cancelled still yields a row per request, and those rows carry the counts. + _ENDED_STATUSES: ClassVar[dict[str, BatchStatus]] = { + "failed": "failed", + "expired": "expired", + "cancelled": "cancelled", + } + + @classmethod + def _settings_type(cls) -> type: + # Heavy third-party SDK: importing the OpenAI model module imports ``openai``. + from pydantic_ai.models.openai import OpenAIResponsesModelSettings + + return OpenAIResponsesModelSettings + + @staticmethod + def _responses_model_id(model_id: str) -> str: Review Comment: Moved it out: the rewrite is pure now and the log only fires from validate/submit. Test checks there's nothing on poll, cancel or landing. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
