This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch feature/CAMEL-23860-token-tracking in repository https://gitbox.apache.org/repos/asf/camel.git
commit a719791c6cd0bbe7ee0f094565c7bab3d1e97a7a Author: Claus Ibsen <[email protected]> AuthorDate: Wed Jul 1 12:48:11 2026 +0200 CAMEL-23860: Track LLM token usage in CLI ask command and TUI AI panel Co-Authored-By: Claude <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../apache/camel/dsl/jbang/core/commands/Ask.java | 13 +++ .../camel/dsl/jbang/core/commands/LlmClient.java | 92 ++++++++++++++++++---- .../dsl/jbang/core/commands/tui/ActionsPopup.java | 4 +- .../camel/dsl/jbang/core/commands/tui/AiPanel.java | 34 ++++++-- .../dsl/jbang/core/commands/tui/CamelMonitor.java | 3 +- .../dsl/jbang/core/commands/tui/McpLogPopup.java | 17 +++- .../dsl/jbang/core/commands/tui/TuiMcpServer.java | 6 ++ 7 files changed, 141 insertions(+), 28 deletions(-) diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Ask.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Ask.java index f6a3ad5d70ba..c8dc8cd6a6c7 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Ask.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/Ask.java @@ -195,12 +195,14 @@ public class Ask extends CamelCommand { String userQuestion) { messages.add(LlmClient.Message.user(userQuestion)); + LlmClient.TokenUsage totalUsage = LlmClient.TokenUsage.EMPTY; for (int i = 0; i < maxIterations; i++) { LlmClient.ChatResponse response = client.chatWithTools(systemPrompt, messages, tools); if (response == null) { printer().printErr("Failed to get response from LLM"); return 1; } + totalUsage = totalUsage.add(response.usage()); if (response.toolCalls() != null && !response.toolCalls().isEmpty()) { messages.add(LlmClient.Message.assistantWithToolCalls(response.text(), response.toolCalls())); @@ -222,14 +224,25 @@ public class Ask extends CamelCommand { printer().println(response.text()); } messages.add(LlmClient.Message.assistantWithToolCalls(response.text(), List.of())); + printTokenUsage(totalUsage); return 0; } } + printTokenUsage(totalUsage); printer().printErr("Reached maximum iterations (" + maxIterations + ") without a final answer."); return 1; } + private void printTokenUsage(LlmClient.TokenUsage usage) { + if (usage.totalTokens() > 0) { + printer().println(); + printer().println("Tokens: " + usage.inputTokens() + " input / " + + usage.outputTokens() + " output / " + + usage.totalTokens() + " total"); + } + } + // ---- Process discovery (delegates to RuntimeHelper) ---- private RuntimeHelper.ProcessInfo findProcess(String nameOrPid) { diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/LlmClient.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/LlmClient.java index d2abee4f8f44..68d167745b74 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/LlmClient.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/LlmClient.java @@ -90,7 +90,19 @@ public class LlmClient { } } - public record ChatResponse(String text, List<ToolCall> toolCalls, String stopReason, boolean streamed) { + public record TokenUsage(int inputTokens, int outputTokens, int totalTokens) { + public static final TokenUsage EMPTY = new TokenUsage(0, 0, 0); + + public TokenUsage add(TokenUsage other) { + return new TokenUsage( + inputTokens + other.inputTokens, + outputTokens + other.outputTokens, + totalTokens + other.totalTokens); + } + } + + public record ChatResponse(String text, List<ToolCall> toolCalls, String stopReason, boolean streamed, + TokenUsage usage) { } // -- Configuration -- @@ -388,12 +400,13 @@ public class LlmClient { if (response.statusCode() != 200) { String errorBody = response.body().collect(Collectors.joining("\n")); handleErrorStatus(response.statusCode(), errorBody); - return new ChatResponse(null, List.of(), "error", false); + return new ChatResponse(null, List.of(), "error", false, TokenUsage.EMPTY); } StringBuilder fullText = new StringBuilder(); List<ToolCall> toolCalls = new ArrayList<>(); String[] doneReasonHolder = { null }; + int[] tokenHolder = { 0, 0 }; response.body().forEach(line -> { if (line.isBlank()) { @@ -443,6 +456,8 @@ public class LlmClient { if (Boolean.TRUE.equals(chunk.get("done"))) { doneReasonHolder[0] = chunk.getString("done_reason"); + tokenHolder[0] = getIntValue(chunk, "prompt_eval_count"); + tokenHolder[1] = getIntValue(chunk, "eval_count"); } } catch (Exception e) { // skip malformed chunks @@ -457,17 +472,19 @@ public class LlmClient { String stopReason = !toolCalls.isEmpty() ? "tool_calls" : (doneReasonHolder[0] != null ? doneReasonHolder[0] : "stop"); + TokenUsage usage = new TokenUsage(tokenHolder[0], tokenHolder[1], tokenHolder[0] + tokenHolder[1]); if (verbose) { printer.println("[verbose] Streamed Ollama: text=" + (text != null ? truncateVerbose(text) : "null") - + ", toolCalls=" + toolCalls.size() + ", doneReason=" + doneReasonHolder[0]); + + ", toolCalls=" + toolCalls.size() + ", doneReason=" + doneReasonHolder[0] + + ", tokens=" + usage.totalTokens()); } - return new ChatResponse(text, toolCalls, stopReason, true); + return new ChatResponse(text, toolCalls, stopReason, true, usage); } catch (HttpTimeoutException e) { printer.println("\nRequest timed out after " + timeout + " seconds."); - return new ChatResponse(null, List.of(), "error", false); + return new ChatResponse(null, List.of(), "error", false, TokenUsage.EMPTY); } catch (Exception e) { printer.println("\nError during streaming: " + e.getMessage()); - return new ChatResponse(null, List.of(), "error", false); + return new ChatResponse(null, List.of(), "error", false, TokenUsage.EMPTY); } } @@ -699,20 +716,21 @@ public class LlmClient { if (verbose) { printer.println("[verbose] parseOpenAiChatResponse: response is null"); } - return new ChatResponse(null, List.of(), "error", false); + return new ChatResponse(null, List.of(), "error", false, TokenUsage.EMPTY); } + TokenUsage usage = extractOpenAiUsage(response); JsonArray choices = (JsonArray) response.get("choices"); if (choices == null || choices.isEmpty()) { if (verbose) { printer.println("[verbose] parseOpenAiChatResponse: no choices in response. Keys: " + response.keySet()); } - return new ChatResponse(null, List.of(), "error", false); + return new ChatResponse(null, List.of(), "error", false, usage); } JsonObject firstChoice = (JsonObject) choices.get(0); String finishReason = firstChoice.getString("finish_reason"); JsonObject message = (JsonObject) firstChoice.get("message"); if (message == null) { - return new ChatResponse(null, List.of(), finishReason, false); + return new ChatResponse(null, List.of(), finishReason, false, usage); } String content = message.getString("content"); @@ -746,7 +764,7 @@ public class LlmClient { printer.println("[verbose] Parsed: text=" + (content != null ? truncateVerbose(content) : "null") + ", toolCalls=" + toolCalls.size() + ", finishReason=" + finishReason); } - return new ChatResponse(content, toolCalls, finishReason, false); + return new ChatResponse(content, toolCalls, finishReason, false, usage); } private ChatResponse parseOllamaChatResponse(JsonObject response) { @@ -754,14 +772,14 @@ public class LlmClient { if (verbose) { printer.println("[verbose] parseOllamaChatResponse: response is null"); } - return new ChatResponse(null, List.of(), "error", false); + return new ChatResponse(null, List.of(), "error", false, TokenUsage.EMPTY); } JsonObject message = (JsonObject) response.get("message"); if (message == null) { if (verbose) { printer.println("[verbose] parseOllamaChatResponse: no message in response. Keys: " + response.keySet()); } - return new ChatResponse(null, List.of(), "error", false); + return new ChatResponse(null, List.of(), "error", false, TokenUsage.EMPTY); } String content = message.getString("content"); @@ -796,21 +814,27 @@ public class LlmClient { String stopReason = !toolCalls.isEmpty() ? "tool_calls" : (doneReason != null ? doneReason : "stop"); + int inputTokens = getIntValue(response, "prompt_eval_count"); + int outputTokens = getIntValue(response, "eval_count"); + TokenUsage usage = new TokenUsage(inputTokens, outputTokens, inputTokens + outputTokens); + if (verbose) { printer.println("[verbose] Parsed Ollama: text=" + (content != null ? truncateVerbose(content) : "null") - + ", toolCalls=" + toolCalls.size() + ", doneReason=" + doneReason); + + ", toolCalls=" + toolCalls.size() + ", doneReason=" + doneReason + + ", tokens=" + usage.totalTokens()); } - return new ChatResponse(content, toolCalls, stopReason, false); + return new ChatResponse(content, toolCalls, stopReason, false, usage); } private ChatResponse parseAnthropicChatResponse(JsonObject response) { if (response == null) { - return new ChatResponse(null, List.of(), "error", false); + return new ChatResponse(null, List.of(), "error", false, TokenUsage.EMPTY); } String stopReason = response.getString("stop_reason"); + TokenUsage usage = extractAnthropicUsage(response); JsonArray contentBlocks = (JsonArray) response.get("content"); if (contentBlocks == null) { - return new ChatResponse(null, List.of(), stopReason, false); + return new ChatResponse(null, List.of(), stopReason, false, usage); } StringBuilder text = new StringBuilder(); @@ -828,7 +852,41 @@ public class LlmClient { } } String textContent = text.length() > 0 ? text.toString() : null; - return new ChatResponse(textContent, toolCalls, stopReason, false); + return new ChatResponse(textContent, toolCalls, stopReason, false, usage); + } + + // ---- Token usage extraction ---- + + private TokenUsage extractOpenAiUsage(JsonObject response) { + JsonObject usage = (JsonObject) response.get("usage"); + if (usage == null) { + return TokenUsage.EMPTY; + } + int prompt = getIntValue(usage, "prompt_tokens"); + int completion = getIntValue(usage, "completion_tokens"); + int total = getIntValue(usage, "total_tokens"); + if (total == 0) { + total = prompt + completion; + } + return new TokenUsage(prompt, completion, total); + } + + private TokenUsage extractAnthropicUsage(JsonObject response) { + JsonObject usage = (JsonObject) response.get("usage"); + if (usage == null) { + return TokenUsage.EMPTY; + } + int input = getIntValue(usage, "input_tokens"); + int output = getIntValue(usage, "output_tokens"); + return new TokenUsage(input, output, input + output); + } + + private static int getIntValue(JsonObject obj, String key) { + Object val = obj.get(key); + if (val instanceof Number n) { + return n.intValue(); + } + return 0; } private String extractOpenAiContent(JsonObject response) { diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ActionsPopup.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ActionsPopup.java index d90a0efa92eb..cce76250427d 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ActionsPopup.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/ActionsPopup.java @@ -212,12 +212,14 @@ class ActionsPopup { } void setMcpEnabled( - boolean enabled, int port, Supplier<String> connectedClient, Supplier<List<TuiMcpServer.LogEntry>> activityLog) { + boolean enabled, int port, Supplier<String> connectedClient, + Supplier<List<TuiMcpServer.LogEntry>> activityLog, Supplier<Integer> toolCallCount) { this.mcpEnabled = enabled; this.mcpPort = port; this.mcpConnectedClient = connectedClient; this.mcpActivityLog = activityLog; mcpLogPopup.setActivityLog(activityLog); + mcpLogPopup.setToolCallCount(toolCallCount); } void setAiActivityLog(Supplier<List<AiPanel.LogEntry>> activityLog) { diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/AiPanel.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/AiPanel.java index f700cddaee5b..548ca3ce124f 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/AiPanel.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/AiPanel.java @@ -90,9 +90,9 @@ class AiPanel { // Activity log for AI Log popup private final List<LogEntry> activityLog = new ArrayList<>(); - record ConversationEntry(String role, String text, long elapsedSeconds) { + record ConversationEntry(String role, String text, long elapsedSeconds, int totalTokens) { ConversationEntry(String role, String text) { - this(role, text, -1); + this(role, text, -1, 0); } } @@ -127,6 +127,14 @@ class AiPanel { return "assistant".equals(last.role()) ? last.elapsedSeconds() : -1; } + private int lastResponseTokens() { + if (thinking.get() || conversation.isEmpty()) { + return 0; + } + ConversationEntry last = conversation.get(conversation.size() - 1); + return "assistant".equals(last.role()) ? last.totalTokens() : 0; + } + void cycleHeight() { splitIndex = (splitIndex + 1) % SPLIT_PERCENTS.length; } @@ -285,6 +293,7 @@ class AiPanel { } messages.add(LlmClient.Message.user(question)); + LlmClient.TokenUsage totalUsage = LlmClient.TokenUsage.EMPTY; for (int i = 0; i < MAX_ITERATIONS; i++) { if (Thread.interrupted()) { throw new InterruptedException(); @@ -297,6 +306,7 @@ class AiPanel { log(LogLevel.ERROR, "Error", err); return; } + totalUsage = totalUsage.add(response.usage()); // check for error response (null text, no tool calls, error stop reason) if ("error".equals(response.stopReason()) @@ -326,8 +336,11 @@ class AiPanel { String text = response.text(); if (text != null && !text.isBlank()) { long elapsed = (System.currentTimeMillis() - thinkingStartTime) / 1000; - conversation.add(new ConversationEntry("assistant", text, elapsed)); - log(LogLevel.RESPONSE, "Response (" + elapsed + "s)", text); + conversation.add(new ConversationEntry("assistant", text, elapsed, totalUsage.totalTokens())); + String tokenInfo = totalUsage.totalTokens() > 0 + ? ", " + totalUsage.totalTokens() + " tokens" + : ""; + log(LogLevel.RESPONSE, "Response (" + elapsed + "s" + tokenInfo + ")", text); } else { String err = "Empty response from LLM."; conversation.add(new ConversationEntry("error", err)); @@ -344,13 +357,15 @@ class AiPanel { } void render(Frame frame, Rect area) { - // At 25% show elapsed in the title bar to save space + // At 25% show elapsed and tokens in the title bar to save space long titleElapsed = lastResponseElapsed(); + int titleTokens = lastResponseTokens(); Line titleLine; if (splitIndex == 0 && titleElapsed >= 0) { + String tokenSuffix = titleTokens > 0 ? ", " + titleTokens + " tokens" : ""; titleLine = Line.from( Span.styled(" AI ", Style.EMPTY.bold()), - Span.styled("(" + titleElapsed + "s) ", Style.EMPTY.dim())); + Span.styled("(" + titleElapsed + "s" + tokenSuffix + ") ", Style.EMPTY.dim())); } else { titleLine = Line.from(Span.styled(" AI ", Style.EMPTY.bold())); } @@ -419,12 +434,14 @@ class AiPanel { md.append(".".repeat((int) dots + 1)).append("*\n"); } - // Show elapsed time as a dimmed line below the markdown when at the bottom + // Show elapsed time and token count as a dimmed line below the markdown when at the bottom long lastElapsed = -1; + int lastTokens = 0; if (!thinking.get() && !conversation.isEmpty()) { ConversationEntry last = conversation.get(conversation.size() - 1); if ("assistant".equals(last.role()) && last.elapsedSeconds() >= 0) { lastElapsed = last.elapsedSeconds(); + lastTokens = last.totalTokens(); } } @@ -478,8 +495,9 @@ class AiPanel { } if (elapsedArea != null && lastElapsed >= 0) { + String tokenSuffix = lastTokens > 0 ? ", " + lastTokens + " tokens" : ""; frame.renderWidget( - Paragraph.from(Line.from(Span.styled("(" + lastElapsed + "s)", Style.EMPTY.dim()))), + Paragraph.from(Line.from(Span.styled("(" + lastElapsed + "s" + tokenSuffix + ")", Style.EMPTY.dim()))), elapsedArea); } } diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/CamelMonitor.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/CamelMonitor.java index 4a49103dde62..37d15040f9c9 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/CamelMonitor.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/CamelMonitor.java @@ -395,7 +395,8 @@ public class CamelMonitor extends CamelCommand { mcpServer = new TuiMcpServer(mcpPort, mcpFacade); try { mcpServer.start(); - actionsPopup.setMcpEnabled(true, mcpPort, mcpServer::getConnectedClient, mcpServer::getActivityLog); + actionsPopup.setMcpEnabled(true, mcpPort, mcpServer::getConnectedClient, + mcpServer::getActivityLog, mcpServer::getToolCallCount); mcpJsonFile = writeMcpJson(mcpPort); } catch (java.net.BindException e) { System.err.println("MCP server failed to start: port " + mcpPort + " is already in use."); diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/McpLogPopup.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/McpLogPopup.java index 94b69f55cd37..dea6a75d2be9 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/McpLogPopup.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/McpLogPopup.java @@ -48,6 +48,7 @@ class McpLogPopup { private boolean visible; private Supplier<List<TuiMcpServer.LogEntry>> activityLog; + private Supplier<Integer> toolCallCount; private List<TuiMcpServer.LogEntry> entries; private int selected; private int detailScroll; @@ -56,6 +57,10 @@ class McpLogPopup { this.activityLog = activityLog; } + void setToolCallCount(Supplier<Integer> toolCallCount) { + this.toolCallCount = toolCallCount; + } + boolean isVisible() { return visible; } @@ -148,6 +153,16 @@ class McpLogPopup { Span.raw(entry.message())))); } + int count = toolCallCount != null ? toolCallCount.get() : 0; + Line titleLine; + if (count > 0) { + titleLine = Line.from( + Span.styled(" MCP Log ", Style.EMPTY.bold()), + Span.styled("(" + count + " calls) ", Style.EMPTY.dim())); + } else { + titleLine = Line.from(Span.styled(" MCP Log ", Style.EMPTY.bold())); + } + ListState masterState = new ListState(); masterState.select(selected); ListWidget list = ListWidget.builder() @@ -157,7 +172,7 @@ class McpLogPopup { .scrollMode(ScrollMode.AUTO_SCROLL) .block(Block.builder() .borderType(BorderType.ROUNDED).borders(Borders.ALL) - .title(" MCP Log ") + .title(Title.from(titleLine)) .build()) .build(); frame.renderStatefulWidget(list, area, masterState); diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TuiMcpServer.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TuiMcpServer.java index 86e560387641..e7db48ddf0a0 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TuiMcpServer.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/TuiMcpServer.java @@ -72,6 +72,7 @@ class TuiMcpServer { private volatile String clientName; private volatile long lastActivity; private volatile long lastToolCallTime; + private volatile int toolCallCount; private final List<LogEntry> activityLog = new ArrayList<>(); TuiMcpServer(int port, McpFacade facade) { @@ -119,6 +120,10 @@ class TuiMcpServer { return System.currentTimeMillis() - lastToolCallTime < 2000; } + int getToolCallCount() { + return toolCallCount; + } + String getConnectedClient() { if (System.currentTimeMillis() - lastActivity < CLIENT_TIMEOUT_MS) { return clientName != null ? clientName : "unknown"; @@ -580,6 +585,7 @@ class TuiMcpServer { } lastToolCallTime = System.currentTimeMillis(); + toolCallCount++; String text; boolean isError = false;
