Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package python-langsmith for
openSUSE:Factory checked in at 2026-07-17 01:43:23
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-langsmith (Old)
and /work/SRC/openSUSE:Factory/.python-langsmith.new.24530 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-langsmith"
Fri Jul 17 01:43:23 2026 rev:10 rq:1366100 version:0.10.5
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-langsmith/python-langsmith.changes
2026-07-15 16:57:50.839499239 +0200
+++
/work/SRC/openSUSE:Factory/.python-langsmith.new.24530/python-langsmith.changes
2026-07-17 01:44:05.782508310 +0200
@@ -1,0 +2,11 @@
+Thu Jul 16 05:54:10 UTC 2026 - Martin Pluskal <[email protected]>
+
+- Update to 0.10.5:
+ * Capture realtime/voice model costs for the optional OpenAI
+ Agents realtime and Google ADK Live (Gemini) integrations
+ (extras not enabled in this package)
+ * Capture the realtime user transcript for the optional LiveKit
+ voice integration
+ * Sync the bundled LangSmith backend API schema definitions
+
+-------------------------------------------------------------------
Old:
----
langsmith-0.10.3.tar.gz
New:
----
langsmith-0.10.5.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-langsmith.spec ++++++
--- /var/tmp/diff_new_pack.uzb0uG/_old 2026-07-17 01:44:06.354527406 +0200
+++ /var/tmp/diff_new_pack.uzb0uG/_new 2026-07-17 01:44:06.358527539 +0200
@@ -17,7 +17,7 @@
Name: python-langsmith
-Version: 0.10.3
+Version: 0.10.5
Release: 0
Summary: Client library for the LangSmith LLM tracing and evaluation
platform
License: MIT
++++++ langsmith-0.10.3.tar.gz -> langsmith-0.10.5.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.3/.bumpversion.cfg
new/langsmith-0.10.5/.bumpversion.cfg
--- old/langsmith-0.10.3/.bumpversion.cfg 2020-02-02 01:00:00.000000000
+0100
+++ new/langsmith-0.10.5/.bumpversion.cfg 2020-02-02 01:00:00.000000000
+0100
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 0.10.3
+current_version = 0.10.5
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
serialize = {major}.{minor}.{patch}
search = {current_version}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.3/PKG-INFO
new/langsmith-0.10.5/PKG-INFO
--- old/langsmith-0.10.3/PKG-INFO 2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/PKG-INFO 2020-02-02 01:00:00.000000000 +0100
@@ -1,6 +1,6 @@
Metadata-Version: 2.4
Name: langsmith
-Version: 0.10.3
+Version: 0.10.5
Summary: Client library to connect to the LangSmith Observability and
Evaluation Platform.
Project-URL: Homepage, https://smith.langchain.com/
Project-URL: Documentation, https://docs.smith.langchain.com/
@@ -29,7 +29,7 @@
Requires-Dist: google-adk>=1.0.0; extra == 'google-adk'
Requires-Dist: wrapt>=1.16.0; extra == 'google-adk'
Provides-Extra: google-adk-live
-Requires-Dist: google-adk>=1.14.0; extra == 'google-adk-live'
+Requires-Dist: google-adk>=2.3.0; extra == 'google-adk-live'
Provides-Extra: langsmith-pyo3
Requires-Dist: langsmith-pyo3>=0.1.0rc2; extra == 'langsmith-pyo3'
Provides-Extra: livekit
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.3/langsmith/__init__.py
new/langsmith-0.10.5/langsmith/__init__.py
--- old/langsmith-0.10.3/langsmith/__init__.py 2020-02-02 01:00:00.000000000
+0100
+++ new/langsmith-0.10.5/langsmith/__init__.py 2020-02-02 01:00:00.000000000
+0100
@@ -45,7 +45,7 @@
# Avoid calling into importlib on every call to __version__
-__version__ = "0.10.3"
+__version__ = "0.10.5"
version = __version__ # for backwards compatibility
# Metadata key to hide a traced run from LangSmith's Messages View.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/langsmith/_internal/voice/session.py
new/langsmith-0.10.5/langsmith/_internal/voice/session.py
--- old/langsmith-0.10.3/langsmith/_internal/voice/session.py 2020-02-02
01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/langsmith/_internal/voice/session.py 2020-02-02
01:00:00.000000000 +0100
@@ -309,6 +309,11 @@
prompt* — the conversation transcript so far minus the response being
recorded (the trailing assistant message) — so the ``llm`` run reads
like
a normal model call instead of having empty inputs.
+
+ ``usage_metadata`` must be passed here, not patched on afterwards: cost
+ is derived when the run is finalized, so usage that lands on a later
+ patch (after ``end``) is not reflected. Services that report tokens on
a
+ separate, later event should defer this call until the counts arrive.
"""
parent = parent or self._current_turn or self.run
if inputs is None:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/langsmith/_openapi_client/resources/sandboxes/boxes.py
new/langsmith-0.10.5/langsmith/_openapi_client/resources/sandboxes/boxes.py
--- old/langsmith-0.10.3/langsmith/_openapi_client/resources/sandboxes/boxes.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/langsmith/_openapi_client/resources/sandboxes/boxes.py
2020-02-02 01:00:00.000000000 +0100
@@ -60,6 +60,7 @@
mem_bytes: int | Omit = omit,
mount_config: box_create_params.MountConfig | Omit = omit,
name: str | Omit = omit,
+ preserve_memory_on_stop: bool | Omit = omit,
proxy_config: box_create_params.ProxyConfig | Omit = omit,
restore_memory: bool | Omit = omit,
snapshot_id: str | Omit = omit,
@@ -83,6 +84,12 @@
vCPU); takes precedence over VCPUs. Fractional (sub-vCPU) values
are not
available for every sandbox.
+ preserve_memory_on_stop: PreserveMemoryOnStop, when true, suspends
the sandbox's memory on a voluntary
+ stop (idle timeout or explicit stop) so the next start resumes
from where it
+ left off. Default false discards memory and keeps only the
filesystem, so the
+ next start is a cold boot. Restarts triggered by infrastructure
maintenance
+ always preserve memory regardless of this setting.
+
restore_memory:
RestoreMemory selects how the sandbox handles a snapshot's
captured memory:
@@ -112,6 +119,7 @@
"mem_bytes": mem_bytes,
"mount_config": mount_config,
"name": name,
+ "preserve_memory_on_stop": preserve_memory_on_stop,
"proxy_config": proxy_config,
"restore_memory": restore_memory,
"snapshot_id": snapshot_id,
@@ -557,6 +565,7 @@
mem_bytes: int | Omit = omit,
mount_config: box_create_params.MountConfig | Omit = omit,
name: str | Omit = omit,
+ preserve_memory_on_stop: bool | Omit = omit,
proxy_config: box_create_params.ProxyConfig | Omit = omit,
restore_memory: bool | Omit = omit,
snapshot_id: str | Omit = omit,
@@ -580,6 +589,12 @@
vCPU); takes precedence over VCPUs. Fractional (sub-vCPU) values
are not
available for every sandbox.
+ preserve_memory_on_stop: PreserveMemoryOnStop, when true, suspends
the sandbox's memory on a voluntary
+ stop (idle timeout or explicit stop) so the next start resumes
from where it
+ left off. Default false discards memory and keeps only the
filesystem, so the
+ next start is a cold boot. Restarts triggered by infrastructure
maintenance
+ always preserve memory regardless of this setting.
+
restore_memory:
RestoreMemory selects how the sandbox handles a snapshot's
captured memory:
@@ -609,6 +624,7 @@
"mem_bytes": mem_bytes,
"mount_config": mount_config,
"name": name,
+ "preserve_memory_on_stop": preserve_memory_on_stop,
"proxy_config": proxy_config,
"restore_memory": restore_memory,
"snapshot_id": snapshot_id,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/langsmith/_openapi_client/resources/threads.py
new/langsmith-0.10.5/langsmith/_openapi_client/resources/threads.py
--- old/langsmith-0.10.3/langsmith/_openapi_client/resources/threads.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/langsmith/_openapi_client/resources/threads.py
2020-02-02 01:00:00.000000000 +0100
@@ -71,6 +71,9 @@
"FIRST_TOKEN_TIME",
"INPUTS_PREVIEW",
"OUTPUTS_PREVIEW",
+ "INPUTS",
+ "OUTPUTS",
+ "ERROR",
"PROMPT_COST",
"COMPLETION_COST",
"TOTAL_COST",
@@ -339,6 +342,9 @@
"FIRST_TOKEN_TIME",
"INPUTS_PREVIEW",
"OUTPUTS_PREVIEW",
+ "INPUTS",
+ "OUTPUTS",
+ "ERROR",
"PROMPT_COST",
"COMPLETION_COST",
"TOTAL_COST",
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/langsmith/_openapi_client/types/sandbox_response.py
new/langsmith-0.10.5/langsmith/_openapi_client/types/sandbox_response.py
--- old/langsmith-0.10.3/langsmith/_openapi_client/types/sandbox_response.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/langsmith/_openapi_client/types/sandbox_response.py
2020-02-02 01:00:00.000000000 +0100
@@ -532,6 +532,8 @@
name: Optional[str] = None
+ preserve_memory_on_stop: Optional[bool] = None
+
proxy_config: Optional[ProxyConfig] = None
size_class: Optional[str] = None
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/langsmith/_openapi_client/types/sandboxes/box_create_params.py
new/langsmith-0.10.5/langsmith/_openapi_client/types/sandboxes/box_create_params.py
---
old/langsmith-0.10.3/langsmith/_openapi_client/types/sandboxes/box_create_params.py
2020-02-02 01:00:00.000000000 +0100
+++
new/langsmith-0.10.5/langsmith/_openapi_client/types/sandboxes/box_create_params.py
2020-02-02 01:00:00.000000000 +0100
@@ -81,6 +81,15 @@
name: str
+ preserve_memory_on_stop: bool
+ """
+ PreserveMemoryOnStop, when true, suspends the sandbox's memory on a
voluntary
+ stop (idle timeout or explicit stop) so the next start resumes from where
it
+ left off. Default false discards memory and keeps only the filesystem, so
the
+ next start is a cold boot. Restarts triggered by infrastructure maintenance
+ always preserve memory regardless of this setting.
+ """
+
proxy_config: ProxyConfig
restore_memory: bool
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/langsmith/_openapi_client/types/thread_list_traces_params.py
new/langsmith-0.10.5/langsmith/_openapi_client/types/thread_list_traces_params.py
---
old/langsmith-0.10.3/langsmith/_openapi_client/types/thread_list_traces_params.py
2020-02-02 01:00:00.000000000 +0100
+++
new/langsmith-0.10.5/langsmith/_openapi_client/types/thread_list_traces_params.py
2020-02-02 01:00:00.000000000 +0100
@@ -47,6 +47,9 @@
"FIRST_TOKEN_TIME",
"INPUTS_PREVIEW",
"OUTPUTS_PREVIEW",
+ "INPUTS",
+ "OUTPUTS",
+ "ERROR",
"PROMPT_COST",
"COMPLETION_COST",
"TOTAL_COST",
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/langsmith/_openapi_client/types/thread_trace.py
new/langsmith-0.10.5/langsmith/_openapi_client/types/thread_trace.py
--- old/langsmith-0.10.3/langsmith/_openapi_client/types/thread_trace.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/langsmith/_openapi_client/types/thread_trace.py
2020-02-02 01:00:00.000000000 +0100
@@ -76,6 +76,12 @@
JSON null if the run is still in progress. Omitted unless included in
`selects`.
"""
+ error: Optional[str] = None
+ """`error` is the full root run error message when the run failed.
+
+ Omitted unless included in `selects`.
+ """
+
error_preview: Optional[str] = None
"""`error_preview` is a short error summary when the run failed.
@@ -89,6 +95,12 @@
in `selects`.
"""
+ inputs: Optional[object] = None
+ """`inputs` is the full root run input payload.
+
+ Omitted unless included in `selects`.
+ """
+
inputs_preview: Optional[str] = None
"""`inputs_preview` is a truncated text preview of inputs.
@@ -117,6 +129,12 @@
included in `selects`.
"""
+ outputs: Optional[object] = None
+ """`outputs` is the full root run output payload.
+
+ Omitted unless included in `selects`.
+ """
+
outputs_preview: Optional[str] = None
"""`outputs_preview` is a truncated text preview of outputs.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/langsmith/integrations/google_adk_live/_plugin.py
new/langsmith-0.10.5/langsmith/integrations/google_adk_live/_plugin.py
--- old/langsmith-0.10.3/langsmith/integrations/google_adk_live/_plugin.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/langsmith/integrations/google_adk_live/_plugin.py
2020-02-02 01:00:00.000000000 +0100
@@ -159,23 +159,70 @@
return getattr(model, "model", None) or None
-def _usage_metadata(event: Any) -> dict[str, int] | None:
+def _audio_tokens(details: Any) -> int | None:
+ """Sum the ``AUDIO`` modality token count from a ``ModalityTokenCount``
list.
+
+ Gemini reports per-modality breakdowns as a list of ``{modality,
token_count}``
+ entries; the audio bucket is what the cost engine prices at the (much
higher)
+ audio rate. ``MediaModality`` has a single audio member (``AUDIO``), so an
+ exact match is enough. Returns ``None`` when no audio modality is present.
+ """
+ if not isinstance(details, (list, tuple)):
+ return None
+ total = 0
+ found = False
+ for entry in details:
+ modality = getattr(entry, "modality", None)
+ modality = getattr(modality, "value", modality)
+ count = getattr(entry, "token_count", None)
+ if str(modality).upper() == "AUDIO" and isinstance(count, int):
+ total += count
+ found = True
+ return total if found else None
+
+
+def _usage_metadata(event: Any) -> dict[str, Any] | None:
"""Map an ADK event's ``usage_metadata`` to LangSmith token usage, if any.
- ADK inherits genai's ``GenerateContentResponseUsageMetadata`` (prompt /
- candidates / total token counts). Live events may carry no usage at all, in
- which case the ``llm`` span is still recorded without token counts. Only
+ ADK inherits genai's ``GenerateContentResponseUsageMetadata``. Beyond the
+ aggregate prompt / candidates / total counts, the ``audio`` modality split
+ (from ``prompt_tokens_details`` / ``candidates_tokens_details``), the
+ ``cache_read`` count, and the ``reasoning`` count are captured — for Gemini
+ Live the audio tokens dominate and are priced very differently from text,
so
+ dropping the split would mis-price the turn. Live events may carry no usage
+ at all, in which case the ``llm`` span is recorded without token counts.
Only
the fields ADK actually reports are included.
"""
um = getattr(event, "usage_metadata", None)
if um is None:
return None
- mapping = {
- "input_tokens": getattr(um, "prompt_token_count", None),
- "output_tokens": getattr(um, "candidates_token_count", None),
- "total_tokens": getattr(um, "total_token_count", None),
- }
- usage = {k: v for k, v in mapping.items() if isinstance(v, int)}
+ usage: dict[str, Any] = {}
+ for key, attr in (
+ ("input_tokens", "prompt_token_count"),
+ ("output_tokens", "candidates_token_count"),
+ ("total_tokens", "total_token_count"),
+ ):
+ if isinstance(v := getattr(um, attr, None), int):
+ usage[key] = v
+
+ input_details: dict[str, int] = {}
+ if (audio := _audio_tokens(getattr(um, "prompt_tokens_details", None))) is
not None:
+ input_details["audio"] = audio
+ if isinstance(cached := getattr(um, "cached_content_token_count", None),
int):
+ input_details["cache_read"] = cached
+ if input_details:
+ usage["input_token_details"] = input_details
+
+ output_details: dict[str, int] = {}
+ if (
+ audio := _audio_tokens(getattr(um, "candidates_tokens_details", None))
+ ) is not None:
+ output_details["audio"] = audio
+ if isinstance(reasoning := getattr(um, "thoughts_token_count", None), int):
+ output_details["reasoning"] = reasoning
+ if output_details:
+ usage["output_token_details"] = output_details
+
return usage or None
@@ -196,6 +243,17 @@
# FunctionCall.id (falling back to the tool name when ADK omits an id).
# A FIFO list per key disambiguates repeated/parallel identical calls.
self._open_tools: dict[str, list[RunTree]] = {}
+ # ADK reports token usage on its own event, separate from (and just
+ # after) the agent transcript, and before turn_complete. The priced
+ # ``output_transcription`` llm span must carry that usage on the run's
+ # finalizing patch (cost is derived at finalize, not from a later
+ # patch), so the transcript is held here until its usage arrives — then
+ # both are recorded in one shot. ``_pending_event`` is the raw agent
+ # event (kept for the span's raw_event metadata). ``_pending_usage``
+ # covers the rarer case where usage lands before the transcript.
+ self._pending_text: str | None = None
+ self._pending_event: Any = None
+ self._pending_usage: dict[str, Any] | None = None
@property
def session(self) -> EventSession:
@@ -233,29 +291,37 @@
for response in view.function_responses:
self._end_tool(response, event, t)
- # A finalized agent utterance: transcript rollup plus a point-in-time
- # ``output_transcription`` span (raw event payload, token usage when
ADK
- # reports it). Anchored at the event's arrival and left flat rather
than
- # grouped into a synthetic turn: Gemini finalizes transcripts late
(often
- # after the audio it describes), so a turn's boundaries can't be placed
- # against the audio timeline without reading as misaligned.
+ # A finalized agent utterance. ADK reports its token usage on a
+ # *separate*, later event (before turn_complete), and cost is derived
+ # when the ``llm`` run is finalized — so usage patched on after the run
+ # ends is lost. The transcript is therefore held until its usage
+ # arrives, then recorded in one shot (see ``_emit_agent_turn``). Flat
+ # and point-in-time, not turn-grouped: Gemini finalizes transcripts
late
+ # (often after the audio they describe), so turn boundaries can't be
+ # placed against the audio timeline without reading as misaligned.
+ um = _usage_metadata(event)
fa = view.final_agent_transcript
if fa:
self._trace.add_message("assistant", fa)
- # llm-kind (not chain) so LangSmith prices it; ls_provider +
- # ls_model_name let it match a Gemini pricing entry. Still flat and
- # point-in-time at event arrival — no turn grouping. Raw event kept
- # in metadata so nothing is lost.
- self._trace.record_llm(
- name="output_transcription",
- outputs={"role": "assistant", "content": fa},
- usage_metadata=_usage_metadata(event),
- metadata={
- "raw_event": scrub(dump_event(event)),
- "ls_provider": "google",
- "ls_model_name": self._model,
- },
- )
+ if um is not None:
+ # Usage already on this event — record immediately.
+ self._emit_agent_turn(event, fa, um)
+ elif self._pending_usage is not None:
+ # Usage arrived first this turn — pair them now.
+ self._emit_agent_turn(event, fa, self._pending_usage)
+ self._pending_usage = None
+ else:
+ # Hold for the usage event that follows within the turn.
+ self._flush_pending_agent_turn()
+ self._pending_text = fa
+ self._pending_event = event
+ elif um is not None:
+ if self._pending_text is not None:
+ self._emit_agent_turn(self._pending_event, self._pending_text,
um)
+ self._pending_text = None
+ self._pending_event = None
+ else:
+ self._pending_usage = um
# Barge-in and end-of-turn markers, each a point-in-time span.
if view.interrupted:
@@ -264,6 +330,40 @@
if view.turn_complete:
with self._trace.event_span(event, t, name="turn_complete",
inbound=False):
pass
+ # Turn boundary: a held transcript never got usage — record it
as-is
+ # rather than lose it. Drop stray usage that never found a
transcript
+ # so it can't attach to the next turn.
+ self._flush_pending_agent_turn()
+ self._pending_usage = None
+
+ def _emit_agent_turn(
+ self, event: Any, text: str, usage: dict[str, Any] | None
+ ) -> None:
+ """Record one ``output_transcription`` ``llm`` run with its usage.
+
+ llm-kind (not chain) so LangSmith prices it; ls_provider +
ls_model_name
+ let it match a Gemini pricing entry. ``usage`` is passed in (not
patched
+ later) so it lands on the run's finalizing patch. Raw event kept in
+ metadata so nothing is lost.
+ """
+ self._trace.record_llm(
+ name="output_transcription",
+ outputs={"role": "assistant", "content": text},
+ usage_metadata=usage,
+ metadata={
+ "raw_event": scrub(dump_event(event)),
+ "ls_provider": "google",
+ "ls_model_name": self._model,
+ },
+ )
+ logger.debug("ADK Live: recorded output_transcription (usage=%s)",
usage)
+
+ def _flush_pending_agent_turn(self) -> None:
+ """Record a held transcript that never received usage (usage stays
None)."""
+ if self._pending_text is not None:
+ self._emit_agent_turn(self._pending_event, self._pending_text,
None)
+ self._pending_text = None
+ self._pending_event = None
def _start_tool(self, call: Any, event: Any) -> None:
"""Open a held-open ``tool`` span for one ``function_call``.
@@ -328,12 +428,15 @@
self._open_tools = {k: v for k, v in self._open_tools.items() if v}
def flush_pending(self) -> None:
- """Close any tool span that never saw its response, with an error.
+ """Close anything left open at teardown.
- Called at teardown: a tool that raised (ADK routes that through its own
- error handling, not a function_response event) or a session torn down
- mid-call would otherwise leave a span dangling open forever.
+ A held agent transcript whose turn never completed (e.g. a Ctrl-C
+ mid-turn) is recorded without usage rather than dropped. A tool that
+ raised (ADK routes that through its own error handling, not a
+ function_response event) or a session torn down mid-call would
otherwise
+ leave a span dangling open forever, so each is closed with an error.
"""
+ self._flush_pending_agent_turn()
for queue in self._open_tools.values():
for run in queue:
run.error = "tool did not complete before the session ended"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/langsmith/integrations/livekit/__init__.py
new/langsmith-0.10.5/langsmith/integrations/livekit/__init__.py
--- old/langsmith-0.10.3/langsmith/integrations/livekit/__init__.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/langsmith/integrations/livekit/__init__.py
2020-02-02 01:00:00.000000000 +0100
@@ -48,6 +48,11 @@
and applies it to every span in the trace — so it holds even for spans
finished on a background task, and concurrent conversations stay separated.
+ For a realtime (speech-to-speech) model, also call
+ :meth:`LiveKitLangSmithSpanProcessor.instrument_session` on the returned
+ processor: the user transcript arrives as a session event rather than on a
+ span, so without it the trace shows only the agent's turns.
+
Args:
audio_path_provider: zero-arg callable returning a local recording path
whose bytes are embedded in the root span (console/dev only). For
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/langsmith/integrations/livekit/processor.py
new/langsmith-0.10.5/langsmith/integrations/livekit/processor.py
--- old/langsmith-0.10.3/langsmith/integrations/livekit/processor.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/langsmith/integrations/livekit/processor.py
2020-02-02 01:00:00.000000000 +0100
@@ -53,6 +53,7 @@
_SESSION_SPAN = "agent_session"
_TOOL_SPAN = "function_tool"
_REALTIME_METRICS_SPAN = "realtime_metrics"
+_USER_SPEAKING_SPAN = "user_speaking"
# ``llm_request`` emits one ``gen_ai.<role>.message`` event per chat item,
plus a
# ``gen_ai.choice`` for the reply.
@@ -103,7 +104,9 @@
return TTLCache(maxsize=DEFAULT_STATE_MAXSIZE,
ttl=state_ttl_seconds)
# trace_id -> running transcript, rolled up onto the root at session
end.
- self._transcript_by_trace: MutableMapping[int, list[dict]] = _cache()
+ self._transcript_by_trace: MutableMapping[int, list[tuple[Any, dict]]]
= (
+ _cache()
+ )
# trace_id -> root span held open until the session (and any egress)
ends.
self._deferred_root_by_trace: MutableMapping[int, TranslatedSpan] =
_cache()
# trace_ids whose ``agent_session`` end span has arrived (used as a
set).
@@ -114,12 +117,49 @@
self._pending_audio_by_thread: MutableMapping[str, dict] = _cache()
# thread id -> trace_id, so ``complete_recording`` can find the trace.
self._trace_by_thread: MutableMapping[str, int] = _cache()
+ self._deferred_user_speaking: MutableMapping[str,
list[TranslatedSpan]] = (
+ _cache()
+ )
+ self._pending_user_transcripts: MutableMapping[str, list[str]] =
_cache()
def _remember_thread_id(self, trace_id: int, thread_id: str) -> None:
"""Also index thread→trace (so ``complete_recording`` can find the
trace)."""
super()._remember_thread_id(trace_id, thread_id)
self._trace_by_thread[thread_id] = trace_id
+ # -- realtime session instrumentation ------------------------------------
+
+ def instrument_session(self, session: Any, thread_id: str) -> None:
+ """Subscribe this processor to a LiveKit ``AgentSession``'s events.
+
+ A realtime (speech-to-speech) model's user transcript arrives via the
+ ``user_input_transcribed`` session event — never on a span — so the
+ processor can't see it from spans alone, and the trace ends up with
only
+ the agent's turns. Call this once after creating the session to wire
the
+ transcript in::
+
+ processor = configure_livekit(...)
+ session = AgentSession(llm=...)
+ set_thread_id(conversation_id)
+ processor.instrument_session(session, conversation_id)
+
+ Each final transcript is paired FIFO with the next ``user_speaking``
span
+ that has no transcript yet (we have no id to match a transcript to its
+ exact span). No-op for the cascade pipeline, where the transcript
already
+ rides the STT ``user_turn`` span.
+
+ Args:
+ session: the LiveKit ``AgentSession`` to subscribe to.
+ thread_id: the conversation id, matching :func:`set_thread_id`.
+ """
+
+ @session.on("user_input_transcribed")
+ def _on_user_input_transcribed(ev: Any) -> None:
+ if getattr(ev, "is_final", False):
+ self._record_user_transcript(
+ str(thread_id), getattr(ev, "transcript", "") or ""
+ )
+
# -- production recording (egress) ---------------------------------------
def expect_recording(self, thread_id: str) -> None:
@@ -173,10 +213,13 @@
tspan.set_kind("chain") # wrappers: no fabricated I/O
elif name == _TURN_SPAN:
self._handle_turn(tspan)
+ elif name == _USER_SPEAKING_SPAN:
+ return self._handle_user_speaking(tspan)
elif name == _SESSION_SPAN:
# Session end: release the deferred root, then export this span.
tspan.set_kind("chain")
self._ended_session_traces[trace_id] = True
+ self._flush_user_speaking(self._thread_id_by_trace.get(trace_id))
self._maybe_release(trace_id)
elif name == "eou_detection":
tspan.set_kind("chain") # framework step
@@ -280,18 +323,99 @@
user_input = tspan.attributes.get("lk.user_input")
response = tspan.attributes.get("lk.response.text")
trace_id = tspan.span.context.trace_id
- conversation = self._transcript_by_trace.get(trace_id) or []
+ start = tspan.span.start_time
if user_input:
msg = build_user_message(str(user_input))
tspan.set_messages(prompt=[msg])
- conversation.append(msg)
+ self._append_transcript(trace_id, msg, start)
if response:
msg = build_assistant_message(str(response))
tspan.set_messages(completion=[msg])
- conversation.append(msg)
+ self._append_transcript(trace_id, msg, start)
+
+ def _append_transcript(self, trace_id: int, message: dict, sort_key: Any)
-> None:
+ """Append a message to the transcript the root rolls up, keyed for
ordering."""
+ conversation = self._transcript_by_trace.get(trace_id) or []
+ conversation.append((sort_key, message))
# Re-assign (not just mutate) so each turn refreshes the cache TTL.
- if conversation:
- self._transcript_by_trace[trace_id] = conversation
+ self._transcript_by_trace[trace_id] = conversation
+
+ # -- realtime user transcript (speech-to-speech) --------------------------
+
+ def _handle_user_speaking(self, tspan: TranslatedSpan) -> bool:
+ """Handle a ``user_speaking`` span — the realtime user turn.
+
+ Deferred (``False``): stamp+export now if the transcript was already
+ buffered, else hold it until one is fed (or flushed untouched at
session
+ end). Exported as-is (``True``) when there's no thread id to pair
against.
+ """
+ tspan.set_kind("chain")
+ thread = tspan.attributes.get("langsmith.metadata.thread_id")
+ if thread is None:
+ return True
+ thread = str(thread)
+
+ pending = self._pending_user_transcripts.get(thread)
+ if pending:
+ transcript = pending.pop(0)
+ if pending:
+ self._pending_user_transcripts[thread] = pending
+ else:
+ self._pending_user_transcripts.pop(thread, None)
+ self._apply_user_transcript(tspan, transcript)
+ self._export(tspan)
+ return False
+
+ held = self._deferred_user_speaking.get(thread) or []
+ held.append(tspan)
+ self._deferred_user_speaking[thread] = held
+ return False
+
+ def _record_user_transcript(self, thread_id: str, transcript: str) -> None:
+ """Pair a realtime transcript (from ``instrument_session``) with its
span.
+
+ Applies it to the oldest held ``user_speaking`` span for the thread, or
+ buffers it if that span hasn't ended yet.
+ """
+ tid = str(thread_id)
+ held = self._deferred_user_speaking.get(tid)
+ if held:
+ tspan = held.pop(0)
+ if held:
+ self._deferred_user_speaking[tid] = held
+ else:
+ self._deferred_user_speaking.pop(tid, None)
+ self._apply_user_transcript(tspan, transcript)
+ self._export(tspan)
+ return
+ pending = self._pending_user_transcripts.get(tid) or []
+ pending.append(transcript)
+ self._pending_user_transcripts[tid] = pending
+
+ def _apply_user_transcript(self, tspan: TranslatedSpan, transcript: str)
-> None:
+ """Render a fed transcript onto a ``user_speaking`` span as the user's
turn.
+
+ Unlike the cascade STT ``user_turn`` (which is excluded), this is the
only
+ record of the realtime user's words, so it's shown — as a plain
``user``
+ message. An empty transcript renders no fabricated I/O.
+ """
+ tspan.set_kind("llm")
+ if transcript:
+ tspan.attributes["lk.user_transcript"] = transcript
+ msg = build_user_message(transcript)
+ tspan.set_messages(prompt=[msg])
+ self._append_transcript(
+ tspan.span.context.trace_id, msg, tspan.span.start_time
+ )
+
+ def _flush_user_speaking(self, thread_id: Optional[str]) -> None:
+ """Export held ``user_speaking`` spans untranscribed (no transcript
arrived)."""
+ if thread_id is None:
+ return
+ tid = str(thread_id)
+ for tspan in self._deferred_user_speaking.pop(tid, None) or []:
+ self._export(tspan)
+ self._pending_user_transcripts.pop(tid, None)
def _handle_tool(self, tspan: TranslatedSpan) -> None:
"""``function_tool``: render as a proper ``tool`` run with its I/O."""
@@ -347,10 +471,11 @@
self._cleanup_trace(trace_id)
def _render_conversation(self, tspan: TranslatedSpan) -> bool:
- """Set the whole transcript as the root's input messages; return if
any."""
- messages = self._transcript_by_trace.get(tspan.span.context.trace_id,
[])
- if not messages:
+ """Set the whole transcript (ordered by span start_time) as the root's
input."""
+ entries = self._transcript_by_trace.get(tspan.span.context.trace_id,
[])
+ if not entries:
return False
+ messages = [msg for _, msg in sorted(entries, key=lambda e: e[0])]
tspan.set_messages(prompt=messages)
return True
@@ -364,15 +489,18 @@
self._trace_by_thread.pop(thread, None)
self._threads_awaiting_recording.pop(thread, None)
self._pending_audio_by_thread.pop(thread, None)
+ self._flush_user_speaking(thread)
def shutdown(self) -> None:
- """Force-export any still-held roots (last resort), then shut down.
+ """Force-export any still-held roots and user_speaking spans, then
shut down.
``force_flush`` deliberately does not — a still-held root there is
legitimately in progress, not a buffered export waiting to drain.
"""
for trace_id in list(self._deferred_root_by_trace):
self._maybe_release(trace_id, force=True)
+ for thread in list(self._deferred_user_speaking.keys()):
+ self._flush_user_speaking(thread)
super().shutdown()
def force_flush(self, timeout_millis: int = 30000) -> bool:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/langsmith/integrations/openai_realtime/_connection.py
new/langsmith-0.10.5/langsmith/integrations/openai_realtime/_connection.py
--- old/langsmith-0.10.3/langsmith/integrations/openai_realtime/_connection.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/langsmith/integrations/openai_realtime/_connection.py
2020-02-02 01:00:00.000000000 +0100
@@ -156,15 +156,14 @@
return base
-def response_usage_metadata(response: Any) -> dict[str, Any] | None:
- """Map Realtime token ``usage`` onto LangSmith ``usage_metadata`` (for
cost).
+def usage_metadata_from_usage(usage: Any) -> dict[str, Any] | None:
+ """Map a Realtime ``usage`` (object or dict) onto LangSmith
``usage_metadata``.
Delegates to the shared OpenAI usage mapper so the Realtime integration and
the Chat/Agents wrappers agree on the canonical shape (and so audio/cached
- token detail is captured, not dropped). Returns ``None`` when the response
- carries no usage (e.g. a cancelled turn).
+ token detail is captured, not dropped). Returns ``None`` when there is no
+ usage or it carries no token counts (e.g. a cancelled turn).
"""
- usage = getattr(response, "usage", None)
if usage is None:
return None
usage_dict = _usage_to_dict(usage)
@@ -176,6 +175,11 @@
return dict(_create_usage_metadata(usage_dict))
+def response_usage_metadata(response: Any) -> dict[str, Any] | None:
+ """Map a ``response.done`` payload's token ``usage`` onto
``usage_metadata``."""
+ return usage_metadata_from_usage(getattr(response, "usage", None))
+
+
class _RealtimeTracer:
"""Turns the Realtime event stream into a LangSmith trace, one event at a
time.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/langsmith/integrations/openai_realtime/_session.py
new/langsmith-0.10.5/langsmith/integrations/openai_realtime/_session.py
--- old/langsmith-0.10.3/langsmith/integrations/openai_realtime/_session.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/langsmith/integrations/openai_realtime/_session.py
2020-02-02 01:00:00.000000000 +0100
@@ -15,8 +15,11 @@
curated ``user_message`` or an assistant ``model`` ``llm`` span), grouped into
turns. Each SDK-run tool becomes a single ``tool`` span spanning its
``tool_start``→``tool_end`` pair (so the span duration is the real tool
latency);
-``audio_interrupted`` flags the turn. (Token usage isn't available — the SDK
has
-no per-response usage events yet.)
+``audio_interrupted`` flags the turn. Token usage — which the SDK exposes no
+*semantic* event for — is recovered from the raw ``response.done`` the SDK
still
+forwards as a ``raw_model_event`` (``raw_server_event``), then attached to that
+turn's assistant ``model`` span so the conversation is priced (audio tokens
+included).
Trace shape — one conversation = one trace::
@@ -43,6 +46,9 @@
EventSession,
start_session,
)
+from langsmith.integrations.openai_realtime._connection import (
+ usage_metadata_from_usage,
+)
from langsmith.run_helpers import tracing_context
if TYPE_CHECKING:
@@ -175,6 +181,25 @@
return getattr(data, "item_id", None), transcript
+def raw_response_usage(event: Any) -> dict[str, Any] | None:
+ """Token usage from a ``raw_model_event`` wrapping a ``response.done``.
+
+ The Agents SDK emits no semantic usage event, but still forwards the raw
+ ``response.done`` (as a ``raw_server_event``), which carries
``response.usage``
+ — the only place per-turn token counts appear on this backend. Maps it via
the
+ shared Realtime usage mapper (so audio/cached detail is kept). Returns
``None``
+ for any other event or a payload without usage.
+ """
+ data = getattr(event, "data", None)
+ raw = getattr(data, "data", None)
+ if not isinstance(raw, dict) or raw.get("type") != "response.done":
+ return None
+ response = raw.get("response")
+ if not isinstance(response, dict):
+ return None
+ return usage_metadata_from_usage(response.get("usage"))
+
+
def history_item(event: Any) -> tuple[str | None, str | None, str | None]:
"""``(item_id, role, text)`` for a ``history_added`` event's single
item."""
item = getattr(event, "item", None)
@@ -278,6 +303,9 @@
self._notified: set[tuple[str, str]] = set() # (role, text) already
sent out
# Per-turn latency: when the current user turn opened; None = not
armed.
self._latency_since: float | None = None
+ # Usage from the most recent raw response.done, awaiting the assistant
+ # model span it belongs to (see observe / _record_item). None = none
seen.
+ self._pending_usage: dict[str, Any] | None = None
# Open tool spans, keyed by (tool_name, arguments). The SDK's
# tool_start/tool_end events carry no call_id, so start→end is matched
on
# that pair; a FIFO list per key disambiguates the (rare) concurrent or
@@ -295,6 +323,8 @@
self.record_first_audio(received_at)
return
if etype == "raw_model_event":
+ if (usage := raw_response_usage(event)) is not None:
+ self._pending_usage = usage
rec = raw_input_transcript(event)
if rec:
item_id, transcript = rec
@@ -464,8 +494,10 @@
else:
self._trace.record_llm(
outputs={"role": "assistant", "content": text},
+ usage_metadata=self._pending_usage,
metadata={"ls_provider": "openai", "ls_model_name":
self._model},
)
+ self._pending_usage = None
class _TracedRealtimeSession:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.3/pyproject.toml
new/langsmith-0.10.5/pyproject.toml
--- old/langsmith-0.10.3/pyproject.toml 2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/pyproject.toml 2020-02-02 01:00:00.000000000 +0100
@@ -96,9 +96,12 @@
google-adk = ["google-adk>=1.0.0", "wrapt>=1.16.0"]
# Enabled via `pip install langsmith[google-adk-live]`
-# ADK Live (Gemini Live voice) tracing via LangSmithGoogleADKLivePlugin.
Requires the
-# plugin API (``google.adk.plugins.base_plugin``), first shipped in 1.7.0.
-google-adk-live = ["google-adk>=1.14.0"]
+# ADK Live (Gemini Live voice) tracing via LangSmithGoogleADKLivePlugin. Needs
the
+# plugin API (``google.adk.plugins.base_plugin``, first shipped in 1.7.0); the
+# >=2.3.0 floor is for model-cost capture — earlier releases pass Gemini's live
+# usage through under `response_token_count`, which is dropped when coerced to
+# the response type, silently losing output-token counts. 2.3.0 remaps it.
+google-adk-live = ["google-adk>=2.3.0"]
# Enabled via `pip install langsmith[strands-agents]`
strands-agents = [
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/tests/unit_tests/wrappers/test_google_adk_live.py
new/langsmith-0.10.5/tests/unit_tests/wrappers/test_google_adk_live.py
--- old/langsmith-0.10.3/tests/unit_tests/wrappers/test_google_adk_live.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/tests/unit_tests/wrappers/test_google_adk_live.py
2020-02-02 01:00:00.000000000 +0100
@@ -264,6 +264,45 @@
)
assert _usage_metadata(_event(usage_metadata=um)) == {"input_tokens":
10}
+ def test_captures_audio_cache_and_reasoning_detail(self):
+ # Gemini Live's dominant cost is audio tokens, split out by modality;
the
+ # audio bucket, cache_read, and reasoning must all be captured or the
+ # audio gets mis-priced as text.
+ def _mtc(modality, count):
+ return SimpleNamespace(
+ modality=SimpleNamespace(value=modality), token_count=count
+ )
+
+ um = SimpleNamespace(
+ prompt_token_count=100,
+ candidates_token_count=40,
+ total_token_count=140,
+ cached_content_token_count=15,
+ thoughts_token_count=8,
+ prompt_tokens_details=[_mtc("AUDIO", 90), _mtc("TEXT", 10)],
+ candidates_tokens_details=[_mtc("AUDIO", 35)],
+ )
+ assert _usage_metadata(_event(usage_metadata=um)) == {
+ "input_tokens": 100,
+ "output_tokens": 40,
+ "total_tokens": 140,
+ "input_token_details": {"audio": 90, "cache_read": 15},
+ "output_token_details": {"audio": 35, "reasoning": 8},
+ }
+
+ def test_audio_modality_as_plain_string(self):
+ # The modality can arrive as a bare string rather than an enum.
+ um = SimpleNamespace(
+ prompt_token_count=50,
+ prompt_tokens_details=[
+ SimpleNamespace(modality="AUDIO", token_count=50),
+ ],
+ )
+ assert _usage_metadata(_event(usage_metadata=um)) == {
+ "input_tokens": 50,
+ "input_token_details": {"audio": 50},
+ }
+
# --------------------------------------------------------------------------- #
# _session_key — per-conversation key precedence.
@@ -445,25 +484,27 @@
session.add_message.assert_not_called()
session.event_span.assert_not_called()
- def test_final_agent_transcript_records_llm_output_transcription(self,
make_plugin):
- # The agent response is a flat, point-in-time ``llm`` run named
- # ``output_transcription`` (so LangSmith prices it), carrying token
usage
- # and ls_provider/ls_model_name — not a chain event span, not
turn-grouped.
+ def test_transcript_then_usage_records_one_priced_llm_run(self,
make_plugin):
+ # ADK reports usage on a separate event just after the transcript. The
+ # transcript is held until that usage arrives, then recorded once as a
+ # flat ``output_transcription`` llm run carrying the usage — so it
lands
+ # on the run's finalizing patch (patching usage after the run ends is
+ # not reflected in cost). Not a chain event span, not turn-grouped.
plugin, ctx, session = self._start(make_plugin)
- um = SimpleNamespace(
- prompt_token_count=3, candidates_token_count=2, total_token_count=5
- )
self._emit(
- plugin,
- ctx,
- _event(
- output_transcription=_txn("sunny", finished=True),
usage_metadata=um
- ),
+ plugin, ctx, _event(output_transcription=_txn("sunny",
finished=True))
)
+ # Held, not yet recorded — waiting for its usage.
session.add_message.assert_called_once_with("assistant", "sunny")
session.set_title.assert_not_called() # title only set from user
speech
session.event_span.assert_not_called() # llm run, not a chain event
span
session.start_turn.assert_not_called()
+ session.record_llm.assert_not_called()
+
+ um = SimpleNamespace(
+ prompt_token_count=3, candidates_token_count=2, total_token_count=5
+ )
+ self._emit(plugin, ctx, _event(usage_metadata=um))
session.record_llm.assert_called_once()
kwargs = session.record_llm.call_args.kwargs
assert kwargs["name"] == "output_transcription"
@@ -476,21 +517,95 @@
assert kwargs["metadata"]["ls_provider"] == "google"
assert "ls_model_name" in kwargs["metadata"]
+ def test_usage_on_same_event_records_immediately(self, make_plugin):
+ # If usage happens to ride the transcript event, record in one shot.
+ plugin, ctx, session = self._start(make_plugin)
+ um = SimpleNamespace(
+ prompt_token_count=3, candidates_token_count=2, total_token_count=5
+ )
+ self._emit(
+ plugin,
+ ctx,
+ _event(
+ output_transcription=_txn("sunny", finished=True),
usage_metadata=um
+ ),
+ )
+ session.record_llm.assert_called_once()
+ assert session.record_llm.call_args.kwargs["usage_metadata"] == {
+ "input_tokens": 3,
+ "output_tokens": 2,
+ "total_tokens": 5,
+ }
+
def test_agent_model_is_resolved_onto_llm_span(self, make_plugin):
# The run's agent model becomes ls_model_name for cost lookup.
factory, created = make_plugin
plugin = factory()
ctx = _ctx(session_id="A", model="gemini-2.0-flash-live")
_run(plugin.before_run_callback(invocation_context=ctx))
+ um = SimpleNamespace(
+ prompt_token_count=3, candidates_token_count=2, total_token_count=5
+ )
_run(
plugin.on_event_callback(
invocation_context=ctx,
- event=_event(output_transcription=_txn("hi", finished=True)),
+ event=_event(
+ output_transcription=_txn("hi", finished=True),
usage_metadata=um
+ ),
)
)
meta = created[0].record_llm.call_args.kwargs["metadata"]
assert meta["ls_model_name"] == "gemini-2.0-flash-live"
+ def test_usage_before_transcript_is_paired_when_transcript_arrives(
+ self, make_plugin
+ ):
+ # Usage can also arrive before the transcript within a turn; it's held
+ # and paired when the transcript lands, still recorded in one shot.
+ plugin, ctx, session = self._start(make_plugin)
+ um = SimpleNamespace(
+ prompt_token_count=7, candidates_token_count=1, total_token_count=8
+ )
+ self._emit(plugin, ctx, _event(usage_metadata=um))
+ session.record_llm.assert_not_called() # nothing to record yet
+
+ self._emit(plugin, ctx, _event(output_transcription=_txn("hi",
finished=True)))
+ session.record_llm.assert_called_once()
+ assert session.record_llm.call_args.kwargs["usage_metadata"] == {
+ "input_tokens": 7,
+ "output_tokens": 1,
+ "total_tokens": 8,
+ }
+
+ def test_turn_complete_flushes_transcript_that_got_no_usage(self,
make_plugin):
+ # A held transcript whose turn ends with no usage is recorded as-is
+ # (usage=None) rather than dropped.
+ plugin, ctx, session = self._start(make_plugin)
+ self._emit(
+ plugin, ctx, _event(output_transcription=_txn("sunny",
finished=True))
+ )
+ session.record_llm.assert_not_called() # held, awaiting usage
+ self._emit(plugin, ctx, _event(turn_complete=True))
+ session.record_llm.assert_called_once()
+ assert session.record_llm.call_args.kwargs["usage_metadata"] is None
+
+ def test_stray_usage_after_turn_complete_is_not_recorded(self,
make_plugin):
+ # Usage in a turn that never produced a transcript is dropped, not
+ # misattributed to a later turn.
+ plugin, ctx, session = self._start(make_plugin)
+ self._emit(
+ plugin, ctx, _event(output_transcription=_txn("sunny",
finished=True))
+ )
+ self._emit(plugin, ctx, _event(turn_complete=True)) # flushes "sunny"
+ session.record_llm.reset_mock()
+
+ um = SimpleNamespace(
+ prompt_token_count=3, candidates_token_count=2, total_token_count=5
+ )
+ self._emit(plugin, ctx, _event(usage_metadata=um))
+ self._emit(plugin, ctx, _event(turn_complete=True))
+ session.record_llm.assert_not_called()
+
def test_tool_call_is_one_span_start_to_end(self, make_plugin):
# function_call opens a held-open ``tool`` span; the matching
# function_response closes it (real tool latency = the gap between).
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/tests/unit_tests/wrappers/test_livekit.py
new/langsmith-0.10.5/tests/unit_tests/wrappers/test_livekit.py
--- old/langsmith-0.10.3/tests/unit_tests/wrappers/test_livekit.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/tests/unit_tests/wrappers/test_livekit.py
2020-02-02 01:00:00.000000000 +0100
@@ -27,6 +27,7 @@
span_id=0x1,
parent=True,
scope="livekit-agents",
+ start_time=None,
):
"""Build a mock LiveKit span.
@@ -46,6 +47,9 @@
span.context = MagicMock()
span.context.trace_id = trace_id
span.context.span_id = span_id
+ # start_time (ns) is the root's conversation-ordering key; default to
span_id
+ # so spans sort in a stable, explicit order without every test setting it.
+ span.start_time = span_id if start_time is None else start_time
span.parent = MagicMock() if parent else None
span.instrumentation_scope = MagicMock()
span.instrumentation_scope.name = scope
@@ -132,6 +136,296 @@
assert len(proc._transcript_by_trace) == 0
+class TestRealtimeUserTranscript:
+ """Realtime (speech-to-speech) user transcripts fed via instrument_session.
+
+ In the realtime pipeline there is no STT ``user_turn`` span; LiveKit emits
a
+ bare ``user_speaking`` span and delivers the transcript out of band via the
+ ``user_input_transcribed`` event. The host forwards it to the processor,
which
+ holds the span until the transcript arrives, then stamps it on.
+ """
+
+ def teardown_method(self):
+ set_thread_id(None)
+
+ def _speaking(self, *, thread="call-1", trace_id=0xABC, span_id=0x2):
+ return _make_span(
+ "user_speaking",
+ {"langsmith.metadata.thread_id": thread},
+ trace_id=trace_id,
+ span_id=span_id,
+ )
+
+ def _exported(self, proc, name):
+ return [
+ c.args[0]
+ for c in proc.downstream.on_end.call_args_list
+ if c.args[0].name == name
+ ]
+
+ def test_transcript_after_span_stamps_and_exports(self):
+ # Span ends empty first (held), then the transcript arrives.
+ proc = _processor()
+ proc.on_end(self._speaking())
+ # Held — nothing exported yet.
+ assert self._exported(proc, "user_speaking") == []
+
+ proc._record_user_transcript("call-1", "what's the weather?")
+
+ exported = self._exported(proc, "user_speaking")
+ assert len(exported) == 1
+ span = exported[0]
+ assert span._attributes["lk.user_transcript"] == "what's the weather?"
+ assert span._attributes["langsmith.span.kind"] == "llm"
+ # Rendered as the user's turn (not excluded, not attributed to
assistant).
+ assert "langsmith.metadata.ls_message_view_exclude" not in
span._attributes
+ assert json.loads(span._attributes["gen_ai.prompt"])["messages"][0] ==
{
+ "role": "user",
+ "content": "what's the weather?",
+ }
+ assert "gen_ai.completion" not in span._attributes
+ # No state left behind.
+ assert len(proc._deferred_user_speaking) == 0
+ assert len(proc._pending_user_transcripts) == 0
+
+ def test_transcript_before_span_is_buffered(self):
+ # Transcript can race ahead of the span's on_end — buffer, then apply.
+ proc = _processor()
+ proc._record_user_transcript("call-1", "hello there")
+ assert self._exported(proc, "user_speaking") == []
+
+ proc.on_end(self._speaking())
+
+ exported = self._exported(proc, "user_speaking")
+ assert len(exported) == 1
+ assert exported[0]._attributes["lk.user_transcript"] == "hello there"
+ assert len(proc._pending_user_transcripts) == 0
+
+ def test_fifo_pairing_within_conversation(self):
+ # Two utterances, two transcripts — paired in order.
+ proc = _processor()
+ proc.on_end(self._speaking(span_id=0x2))
+ proc.on_end(self._speaking(span_id=0x3))
+ proc._record_user_transcript("call-1", "first")
+ proc._record_user_transcript("call-1", "second")
+
+ transcripts = [
+ s._attributes["lk.user_transcript"]
+ for s in self._exported(proc, "user_speaking")
+ ]
+ assert transcripts == ["first", "second"]
+
+ def test_no_thread_id_exports_untouched(self):
+ # Without a thread id there is nothing to pair against — export as-is.
+ proc = _processor()
+ proc.on_end(_make_span("user_speaking", trace_id=0xABC, span_id=0x2))
+ exported = self._exported(proc, "user_speaking")
+ assert len(exported) == 1
+ assert exported[0]._attributes["langsmith.span.kind"] == "chain"
+ assert "lk.user_transcript" not in exported[0]._attributes
+
+ def test_empty_transcript_consumes_slot_without_fake_io(self):
+ # A final-but-empty transcript still pairs (keeping FIFO aligned) but
+ # renders no fabricated I/O.
+ proc = _processor()
+ proc.on_end(self._speaking())
+ proc._record_user_transcript("call-1", "")
+
+ exported = self._exported(proc, "user_speaking")
+ assert len(exported) == 1
+ assert "gen_ai.completion" not in exported[0]._attributes
+ assert "lk.user_transcript" not in exported[0]._attributes
+
+ def test_session_end_flushes_untranscribed_span(self):
+ # Realtime input transcription disabled: no transcript ever arrives, so
+ # the held span must still be exported (untouched) at session end.
+ proc = _processor()
+ tid = 0xABC
+ # Production flow: set_thread_id + on_start caches the trace's thread
id,
+ # so the session-end flush can resolve which held spans to release.
+ set_thread_id("call-1")
+ proc.on_start(_make_span("job_entrypoint", parent=None, trace_id=tid))
+ set_thread_id(None)
+ proc.on_end(_make_span("job_entrypoint", parent=None, trace_id=tid))
+ proc.on_end(self._speaking(trace_id=tid))
+ assert self._exported(proc, "user_speaking") == [] # held
+
+ proc.on_end(_make_span("agent_session", trace_id=tid, span_id=0x3))
+
+ assert len(self._exported(proc, "user_speaking")) == 1
+ assert len(proc._deferred_user_speaking) == 0
+
+ def test_shutdown_flushes_untranscribed_span(self):
+ proc = _processor()
+ proc.on_end(self._speaking())
+ proc.shutdown()
+ assert len(self._exported(proc, "user_speaking")) == 1
+
+
+class TestRealtimeRootRollup:
+ """A late realtime user transcript sorts into its place in the root rollup.
+
+ The transcript arrives after its turn's ``agent_turn`` reply is already
+ recorded, so ordering keys off each message's source-span start_time — the
+ ``user_speaking`` span (earlier) before the ``agent_turn`` reply (later).
+ """
+
+ def _root(self, proc):
+ return next(
+ c.args[0]
+ for c in proc.downstream.on_end.call_args_list
+ if c.args[0]._attributes.get("langsmith.root_span")
+ )
+
+ def test_user_sorts_before_reply_despite_late_arrival(self):
+ proc = _processor()
+ tid = 0xABC
+ proc.on_end(
+ _make_span(
+ "job_entrypoint",
+ {"langsmith.metadata.thread_id": "call-1"},
+ parent=None,
+ trace_id=tid,
+ )
+ )
+ # User speaks (early span), then the reply turn lands and is
recorded...
+ proc.on_end(
+ _make_span(
+ "user_speaking",
+ {"langsmith.metadata.thread_id": "call-1"},
+ trace_id=tid,
+ span_id=0x2,
+ start_time=10,
+ )
+ )
+ proc.on_end(
+ _make_span(
+ "agent_turn",
+ {"lk.response.text": "sunny"},
+ trace_id=tid,
+ span_id=0x3,
+ start_time=20,
+ )
+ )
+ # ...only *then* does the transcript arrive (out of order).
+ proc._record_user_transcript("call-1", "weather?")
+ proc.on_end(_make_span("agent_session", trace_id=tid, span_id=0x4))
+
+ root = self._root(proc)
+ prompt = json.loads(root._attributes["gen_ai.prompt"])["messages"]
+ assert [m["content"] for m in prompt] == ["weather?", "sunny"]
+ assert "gen_ai.completion" not in root._attributes
+
+ def test_greeting_then_user_turn_ordered(self):
+ # Agent greets first (agent_turn, no user_speaking), then a user turn.
+ # The greeting must not steal the user's transcript, and order holds.
+ proc = _processor()
+ tid = 0xABC
+ proc.on_end(
+ _make_span(
+ "job_entrypoint",
+ {"langsmith.metadata.thread_id": "call-1"},
+ parent=None,
+ trace_id=tid,
+ )
+ )
+ proc.on_end(
+ _make_span(
+ "agent_turn",
+ {"lk.response.text": "hi there!"},
+ trace_id=tid,
+ span_id=0x2,
+ start_time=5,
+ )
+ )
+ proc.on_end(
+ _make_span(
+ "user_speaking",
+ {"langsmith.metadata.thread_id": "call-1"},
+ trace_id=tid,
+ span_id=0x3,
+ start_time=10,
+ )
+ )
+ proc.on_end(
+ _make_span(
+ "agent_turn",
+ {"lk.response.text": "sunny"},
+ trace_id=tid,
+ span_id=0x4,
+ start_time=20,
+ )
+ )
+ proc._record_user_transcript("call-1", "weather?")
+ proc.on_end(_make_span("agent_session", trace_id=tid, span_id=0x5))
+
+ root = self._root(proc)
+ contents = [
+ m["content"]
+ for m in json.loads(root._attributes["gen_ai.prompt"])["messages"]
+ ]
+ assert contents == ["hi there!", "weather?", "sunny"]
+
+
+class TestInstrumentSession:
+ """instrument_session subscribes the processor to a session's events
itself."""
+
+ class _FakeSession:
+ """Minimal stand-in for a LiveKit AgentSession's event emitter."""
+
+ def __init__(self):
+ self.handlers = {}
+
+ def on(self, name):
+ def _register(fn):
+ self.handlers[name] = fn
+ return fn
+
+ return _register
+
+ @staticmethod
+ def _event(*, is_final, transcript):
+ ev = MagicMock()
+ ev.is_final = is_final
+ ev.transcript = transcript
+ return ev
+
+ def test_final_transcript_wired_to_processor(self):
+ proc = _processor()
+ session = self._FakeSession()
+ proc.instrument_session(session, "call-1")
+
+ proc.on_end(
+ _make_span(
+ "user_speaking",
+ {"langsmith.metadata.thread_id": "call-1"},
+ span_id=0x2,
+ )
+ )
+ session.handlers["user_input_transcribed"](
+ self._event(is_final=True, transcript="hello there")
+ )
+
+ exported = [
+ c.args[0]
+ for c in proc.downstream.on_end.call_args_list
+ if c.args[0].name == "user_speaking"
+ ]
+ assert len(exported) == 1
+ assert exported[0]._attributes["lk.user_transcript"] == "hello there"
+
+ def test_interim_transcript_ignored(self):
+ proc = _processor()
+ session = self._FakeSession()
+ proc.instrument_session(session, "call-1")
+
+ session.handlers["user_input_transcribed"](
+ self._event(is_final=False, transcript="partial")
+ )
+ # Interim result buffered nothing; a later span has no transcript to
pair.
+ assert len(proc._pending_user_transcripts) == 0
+
+
class TestForceFlush:
"""force_flush must not finalize conversations still in progress."""
@@ -259,7 +553,8 @@
"agent_turn", {"lk.response.text": "two"}, trace_id=tid,
span_id=0x3
)
)
- assert [m["content"] for m in proc._transcript_by_trace[tid]] == [
+ # The store holds (sort_key, message) tuples.
+ assert [m["content"] for _, m in proc._transcript_by_trace[tid]] == [
"one",
"two",
]
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/tests/unit_tests/wrappers/test_openai_realtime_agents.py
new/langsmith-0.10.5/tests/unit_tests/wrappers/test_openai_realtime_agents.py
---
old/langsmith-0.10.3/tests/unit_tests/wrappers/test_openai_realtime_agents.py
2020-02-02 01:00:00.000000000 +0100
+++
new/langsmith-0.10.5/tests/unit_tests/wrappers/test_openai_realtime_agents.py
2020-02-02 01:00:00.000000000 +0100
@@ -25,6 +25,7 @@
_stringify,
describe_event,
raw_input_transcript,
+ raw_response_usage,
wrap_realtime_session,
)
@@ -130,6 +131,35 @@
is None
)
+ def test_raw_response_usage(self):
+ ev = NS(
+ type="raw_model_event",
+ data=NS(
+ type="raw_server_event",
+ data={
+ "type": "response.done",
+ "response": {"usage": {"input_tokens": 7, "output_tokens":
3}},
+ },
+ ),
+ )
+ usage = raw_response_usage(ev)
+ assert usage["input_tokens"] == 7
+ assert usage["output_tokens"] == 3
+ # Non-usage raw events and unrelated events return None.
+ assert raw_response_usage(NS(data=NS(type="raw_server_event",
data={}))) is None
+ assert raw_response_usage(NS(data=NS(type="other"))) is None
+ assert (
+ raw_response_usage(
+ NS(
+ data=NS(
+ type="raw_server_event",
+ data={"type": "response.done", "response": {}},
+ )
+ )
+ )
+ is None
+ )
+
def test_describe_event_variants(self):
name, payload, inbound = describe_event(
NS(type="tool_start", tool=NS(name="lookup"), arguments='{"q": 1}')
@@ -332,6 +362,90 @@
assert session.messages == [{"role": "user", "content": "cancel that"}]
assert any(n == "user_message" for n, _ in created)
+ def test_response_done_usage_attached_to_assistant_span(self, monkeypatch):
+ # The SDK has no semantic usage event, but forwards the raw
response.done
+ # (as raw_server_event); its usage must ride on the assistant model
span,
+ # with audio/cached detail preserved for correct pricing.
+ session = start_session(
+ thread_id="t", sample_rate=24_000,
integration="openai-agents-realtime"
+ )
+ created = _spy_children(monkeypatch)
+ tracer = _tracer(session)
+
+ tracer.observe(_history_updated(_item("u1", "user", "weather?")))
+ tracer.observe(
+ NS(
+ type="raw_model_event",
+ data=NS(
+ type="raw_server_event",
+ data={
+ "type": "response.done",
+ "response": {
+ "usage": {
+ "input_tokens": 300,
+ "output_tokens": 120,
+ "total_tokens": 420,
+ "input_token_details": {
+ "audio_tokens": 250,
+ "cached_tokens": 20,
+ "text_tokens": 30,
+ },
+ "output_token_details": {
+ "audio_tokens": 100,
+ "text_tokens": 20,
+ },
+ }
+ },
+ },
+ ),
+ )
+ )
+ tracer.observe(
+ _history_updated(
+ _item("u1", "user", "weather?"),
+ _item("a1", "assistant", "It's sunny."),
+ )
+ )
+ tracer.flush_pending()
+ session.finalize()
+
+ model = next(c for n, c in created if n == "model")
+ usage = (model.extra or {}).get("metadata", {}).get("usage_metadata")
+ assert usage["input_tokens"] == 300
+ assert usage["output_tokens"] == 120
+ assert usage["input_token_details"]["audio"] == 250
+ assert usage["input_token_details"]["cache_read"] == 20
+ assert usage["output_token_details"]["audio"] == 100
+
+ def test_response_done_usage_consumed_once(self, monkeypatch):
+ # Usage is cleared after it's attached, so a later assistant turn
without
+ # its own response.done doesn't inherit the previous turn's counts.
+ session = start_session(
+ thread_id="t", sample_rate=24_000,
integration="openai-agents-realtime"
+ )
+ created = _spy_children(monkeypatch)
+ tracer = _tracer(session)
+
+ tracer.observe(
+ NS(
+ type="raw_model_event",
+ data=NS(
+ type="raw_server_event",
+ data={
+ "type": "response.done",
+ "response": {"usage": {"input_tokens": 10,
"output_tokens": 5}},
+ },
+ ),
+ )
+ )
+ tracer.observe(_history_updated(_item("a1", "assistant", "one")))
+ tracer.observe(_history_updated(_item("a2", "assistant", "two")))
+ tracer.flush_pending()
+
+ models = [c for n, c in created if n == "model"]
+ assert (models[0].extra or {}).get("metadata",
{}).get("usage_metadata")
+ assert (models[1].extra or {}).get("metadata",
{}).get("usage_metadata") is None
+
def test_on_message_called_once_per_finalized_line(self):
session = start_session(
thread_id="t", sample_rate=24_000,
integration="openai-agents-realtime"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.3/tests/unit_tests/wrappers/test_voice_session.py
new/langsmith-0.10.5/tests/unit_tests/wrappers/test_voice_session.py
--- old/langsmith-0.10.3/tests/unit_tests/wrappers/test_voice_session.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/tests/unit_tests/wrappers/test_voice_session.py
2020-02-02 01:00:00.000000000 +0100
@@ -223,6 +223,37 @@
assert len(captured) == 1
assert captured[0].inputs == {"messages": [{"role": "user", "content":
"hi"}]}
+ def test_usage_lands_on_the_run_before_it_is_finalized(self):
+ # Cost is derived when the run is finalized, so usage must be set on
the
+ # run's extra.metadata before end/patch — never on a later patch.
+ s = _session()
+ captured: list = []
+ real_create = session_mod.RunTree.create_child
+
+ def spy(self, **kwargs):
+ child = real_create(self, **kwargs)
+ if kwargs.get("run_type") == "llm":
+ captured.append(child)
+ return child
+
+ with mock.patch.object(session_mod.RunTree, "create_child", spy):
+ s.record_llm(
+ outputs={"role": "assistant", "content": "hi"},
+ usage_metadata={
+ "input_tokens": 3,
+ "output_tokens": 2,
+ "total_tokens": 5,
+ },
+ )
+ assert len(captured) == 1
+ run = captured[0]
+ assert run.end_time is not None # finalized
+ assert run.extra["metadata"]["usage_metadata"] == {
+ "input_tokens": 3,
+ "output_tokens": 2,
+ "total_tokens": 5,
+ }
+
class TestFinalizeFailOpen:
def test_wav_build_failure_does_not_break_finalize(self, monkeypatch):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.3/uv.lock new/langsmith-0.10.5/uv.lock
--- old/langsmith-0.10.3/uv.lock 2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.5/uv.lock 2020-02-02 01:00:00.000000000 +0100
@@ -1784,7 +1784,7 @@
{ name = "claude-agent-sdk", marker = "python_full_version >= '3.10' and
extra == 'claude-agent-sdk'", specifier = ">=0.1.0" },
{ name = "distro", specifier = ">=1.7.0" },
{ name = "google-adk", marker = "extra == 'google-adk'", specifier =
">=1.0.0" },
- { name = "google-adk", marker = "extra == 'google-adk-live'", specifier =
">=1.14.0" },
+ { name = "google-adk", marker = "extra == 'google-adk-live'", specifier =
">=2.3.0" },
{ name = "httpx", specifier = ">=0.23.0,<1" },
{ name = "langsmith-pyo3", marker = "extra == 'langsmith-pyo3'", specifier
= ">=0.1.0rc2" },
{ name = "livekit-agents", marker = "extra == 'livekit'", specifier =
">=1.0" },
