This is an automated email from the ASF dual-hosted git repository. luigidemasi pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel.git
commit ccf670e158a95b808554da5e09911bf3b38aea34 Author: Luigi De Masi <[email protected]> AuthorDate: Wed Oct 14 22:31:10 2026 +0200 CAMEL-25669: Support image input and audit previews in JBang Accept explicit image paths, render bounded audit previews, and preserve captured audit text as literal evidence. Keep payloads out of normal output and report actionable export failures. Co-authored-by: OpenAI Codex <[email protected]> Signed-off-by: Luigi De Masi <[email protected]> --- .../apache/camel/catalog/docs/semantic-openai.adoc | 27 ++++ .../src/main/docs/semantic-openai.adoc | 27 ++++ design/semantic-audit.adoc | 5 + .../ROOT/pages/camel-4x-upgrade-guide-4_24.adoc | 4 +- .../ROOT/pages/camel-jbang-tui-observe.adoc | 13 +- .../jbang-commands/camel-jbang-semantic-audit.adoc | 1 + .../jbang-commands/camel-jbang-semantic-eval.adoc | 4 +- .../META-INF/camel-jbang-commands-metadata.json | 2 +- .../jbang/core/commands/action/SemanticAudit.java | 38 +++++- .../core/commands/action/SemanticEvaluate.java | 8 +- .../jbang/core/common/SemanticImagePreview.java | 117 ++++++++++++++++++ .../dsl/jbang/core/common/SemanticMediaInput.java | 134 ++++++++++++++++++++ .../jbang/core/common/SemanticResultFormatter.java | 13 +- .../core/commands/action/SemanticCommandTest.java | 52 ++++++++ .../jbang/core/common/SemanticMediaInputTest.java | 96 +++++++++++++++ .../core/common/SemanticResultFormatterTest.java | 10 ++ .../jbang/core/commands/tui/SemanticAuditView.java | 50 +++++--- .../core/commands/tui/SemanticEvaluationPopup.java | 9 +- .../jbang/core/commands/tui/SemanticImageView.java | 91 ++++++++++++++ .../core/commands/tui/SemanticPlayground.java | 47 +++++-- .../core/commands/tui/SemanticResultView.java | 4 + .../src/main/resources/tui/help/semantic.md | 10 ++ .../core/commands/tui/SemanticImageViewTest.java | 137 +++++++++++++++++++++ 23 files changed, 857 insertions(+), 42 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/semantic-openai.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/semantic-openai.adoc index 7ad4b3546904..40d8325df015 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/semantic-openai.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/semantic-openai.adoc @@ -160,6 +160,33 @@ Refusal raises `OpenAIDecisionRefusalException`. Invalid or inconsistent provide with the route's normal `onException` policy. Unsupported input and invalid declarations fail validation; infrastructure failure is never mapped to false or a score. +== Audit, CLI and TUI + +Audit records classify input as text, image, text plus images, structured or unsupported even +when payload capture is disabled; bounded classification may report unknown. The optional `inputType` audit extension is projected only +to backends that declare support. It contains no image payload and is not added to monitoring events. + +Image payload capture follows the existing per-expert opt-in, redactor and `maxChars` budget. +An oversized image retains a bounded descriptor with `dataOmitted=snapshot_limit` when that +fits the budget. A descriptor is not a replayable image. Captured content remains untrusted data. + +`camel semantic eval` accepts `--body=image:/path/picture.png` for a named definition, or +`--input=image:/path/picture.png` for a direct expert call. Use `json:` for an ordered envelope. +The TUI playground accepts `image:/path/picture.png` and JSON envelopes when the expert advertises +image input. Files are read only on an explicit evaluation request. + +The TUI Audit inspector (`u`) renders bounded inline colour previews from retained image data. +The CLI renders a plain-terminal preview. Decoding uses locally available ImageIO readers; +a missing decoder (for example WebP on a stock JDK), invalid data or preview resource limit +shows an explicit unavailable reason. Previews are limited to 64 pixels per dimension and the +first image frame, with transparency on white; they do not alter the data submitted to the expert. + +Normal command output, including `--json`, replaces image base64 with metadata. Use an explicit +`--output=audit.json` on `camel semantic audit` to write the full returned data to a new file; +case/bulk exports still require `--include-input` to request captured input. Existing files are +never overwritten. Audit drafts retain captured images for an explicit Run without putting +base64 into the editor. Omitted/redacted images cannot be reconstructed by the playground. + == Opt-in API smoke tests `camel-openai` includes `OpenAIDecisionsExternalServiceIT` for native text batches and mixed diff --git a/components/camel-ai/camel-semantic-openai/src/main/docs/semantic-openai.adoc b/components/camel-ai/camel-semantic-openai/src/main/docs/semantic-openai.adoc index 7ad4b3546904..40d8325df015 100644 --- a/components/camel-ai/camel-semantic-openai/src/main/docs/semantic-openai.adoc +++ b/components/camel-ai/camel-semantic-openai/src/main/docs/semantic-openai.adoc @@ -160,6 +160,33 @@ Refusal raises `OpenAIDecisionRefusalException`. Invalid or inconsistent provide with the route's normal `onException` policy. Unsupported input and invalid declarations fail validation; infrastructure failure is never mapped to false or a score. +== Audit, CLI and TUI + +Audit records classify input as text, image, text plus images, structured or unsupported even +when payload capture is disabled; bounded classification may report unknown. The optional `inputType` audit extension is projected only +to backends that declare support. It contains no image payload and is not added to monitoring events. + +Image payload capture follows the existing per-expert opt-in, redactor and `maxChars` budget. +An oversized image retains a bounded descriptor with `dataOmitted=snapshot_limit` when that +fits the budget. A descriptor is not a replayable image. Captured content remains untrusted data. + +`camel semantic eval` accepts `--body=image:/path/picture.png` for a named definition, or +`--input=image:/path/picture.png` for a direct expert call. Use `json:` for an ordered envelope. +The TUI playground accepts `image:/path/picture.png` and JSON envelopes when the expert advertises +image input. Files are read only on an explicit evaluation request. + +The TUI Audit inspector (`u`) renders bounded inline colour previews from retained image data. +The CLI renders a plain-terminal preview. Decoding uses locally available ImageIO readers; +a missing decoder (for example WebP on a stock JDK), invalid data or preview resource limit +shows an explicit unavailable reason. Previews are limited to 64 pixels per dimension and the +first image frame, with transparency on white; they do not alter the data submitted to the expert. + +Normal command output, including `--json`, replaces image base64 with metadata. Use an explicit +`--output=audit.json` on `camel semantic audit` to write the full returned data to a new file; +case/bulk exports still require `--include-input` to request captured input. Existing files are +never overwritten. Audit drafts retain captured images for an explicit Run without putting +base64 into the editor. Omitted/redacted images cannot be reconstructed by the playground. + == Opt-in API smoke tests `camel-openai` includes `OpenAIDecisionsExternalServiceIT` for native text batches and mixed diff --git a/design/semantic-audit.adoc b/design/semantic-audit.adoc index 44ef1d5a7e3e..ee569eac24f0 100644 --- a/design/semantic-audit.adoc +++ b/design/semantic-audit.adoc @@ -88,6 +88,11 @@ capture remains opt-in, bounded and redacted. An image too large for the configu may be replaced by a bounded descriptor (`type=image`, optional mediaType, dataOmitted=snapshot_limit). The redactor still processes that detached snapshot. A descriptor cannot recreate missing data. +Audit displays label input type even when payload capture is disabled. TUI image decoding occurs +off the render thread and uses only retained bytes, with bounded thumbnails and an explicit +unavailable state. Normal CLI human/JSON display omits base64; explicit file exports can retain +requested captured input. No historical URL is fetched and browsing never invokes an expert. + == Read-only access The reader SPI has no evidence-specific limit on getAll. Explicit console callers limit requests to diff --git a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc index e535071478c6..01548ec434a0 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc @@ -744,4 +744,6 @@ for every expert before any provider performs inference. A shared validation fai expert's group as invalid input in the audit. Semantic audit backends can opt into the additive `inputType` extension. Readers that do not -support it keep their existing projection. +support it keep their existing projection. The `camel semantic audit --json` display omits image +base64 and reports metadata instead. Use `--output=<new-file>` to export the full returned JSON, +with `--include-input` for case/bulk requests that need captured input. Existing files are not overwritten. diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc b/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc index d13bb945ff69..9e5ab04496cf 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc @@ -536,10 +536,21 @@ and `Enter` to open one. `Esc` returns to the previous record. Association uses a shared breadcrumb does not establish causality or prove that the case is complete. Missing records and unsupported case retrieval remain explicit. +The Input column and Input type detail distinguish text, images and mixed input, including when +payload capture is disabled. Older records show that the input type was not recorded. +For retained images, `u` renders an inline colour preview without printing base64. Decoding happens +in the background and uses local image readers; unavailable/corrupt/oversized images show a reason. +No remote URL is fetched. The preview uses the first frame and at most 64 pixels per dimension. + When input capture is enabled for an expert, `u` reveals the permitted stored input or its omission reason. Input capture is disabled by default; configure it in the semantic audit DSL. Only the selected state is captured, subject to its configured size limit and optional redactor. `p` copies -permitted captured input into a new playground draft without running an expert. +permitted captured input into a new playground draft without running an expert. Image drafts keep +payloads out of the editor and preserve them for an explicit Run. A capture-limit descriptor is +not a replayable image. Image-capable playgrounds accept `image:/path/picture.png` or a JSON media +envelope; the file is read only when Run is requested. Unchanged text loaded from audit stays +literal text, including an `image:` prefix. Definition samples also accept +`"body":"image:/path/picture.png"`. The latest page refreshes every second; `Space` pauses or resumes polling. Older pages stay fixed until explicitly refreshed. Filters use exact values; `definition` and `operation` apply to all retained history. diff --git a/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-semantic-audit.adoc b/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-semantic-audit.adoc index 631ce46266f9..58188160ae3c 100644 --- a/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-semantic-audit.adoc +++ b/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-semantic-audit.adoc @@ -34,6 +34,7 @@ camel semantic audit [options] | `--limit` | Maximum records per page (1-200) | 50 | int | `--max-records` | Case/export retrieval budget (1-1000) | 200 | int | `--operation` | Filter by recorded expert operation | | String +| `--output` | Write the full JSON response to a new file, including requested captured image data | | Path | `--result-label` | Exact Classification member; requires --result-type=CLASSIFICATION | | String | `--result-type` | Captured result type: BOOLEAN, CHOICE, SCORE or CLASSIFICATION | | String | `--result-value` | Exact Boolean, Choice or numeric Score value; requires --result-type | | String diff --git a/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-semantic-eval.adoc b/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-semantic-eval.adoc index 46a10dfd9c03..4415aef5c4bf 100644 --- a/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-semantic-eval.adoc +++ b/docs/user-manual/modules/ROOT/pages/jbang-commands/camel-jbang-semantic-eval.adoc @@ -19,12 +19,12 @@ camel semantic eval [options] [cols="2,5,1,2",options="header"] |=== | Option | Description | Default | Type -| `--body` | Sample message body (text, file:path or json:value) | | String +| `--body` | Sample body (text, file:path, image:path or json:value) | | String | `--evaluation` | Named semantic definition to evaluate | | String | `--evaluation-timeout` | Provider evaluation timeout in milliseconds (1-50000) | 50000 | long | `--expert` | Expert to call directly | | String | `--header` | Sample header (key=value; use json: for a typed value) | | List -| `--input` | Direct expert input (text, file:path or json:value) | | String +| `--input` | Direct input (text, file:path, image:path or json:value) | | String | `--json` | Output a single JSON document | | boolean | `--operation` | Expert operation to call directly | | String | `--parameter` | Expert parameter (key=value; use json: for a typed value) | | List diff --git a/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json b/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json index ee61341736b4..331d29b32497 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json +++ b/dsl/camel-jbang/camel-jbang-core/src/generated/resources/META-INF/camel-jbang-commands-metadata.json @@ -30,7 +30,7 @@ { "name": "run", "fullName": "run", "description": "Run as local Camel integration", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.Run", "options": [ { "names": "--background", "description": "Run in the background", "defaultValue": "false", "javaType": "boolean", "type": "boolean" }, { "names": "--background-wait", "description": "To wait for run in background to startup successfully, before returning", "defaultValue": "true", "javaType": "boolean", "type": "boolean" }, { [...] { "name": "sbom", "fullName": "sbom", "description": "Generate a CycloneDX or SPDX SBOM for a specific project", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.SBOMGenerator", "options": [ { "names": "--build-property", "description": "Maven build properties, ex. --build-property=prop1=foo", "javaType": "java.util.List", "type": "array" }, { "names": "--camel-spring-boot-version", "description": "Camel version to use with Spring Boot", "javaType": "java.lang.String", "type" [...] { "name": "script", "fullName": "script", "description": "Run Camel integration as shell script for terminal scripting", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.Script", "options": [ { "names": "--logging", "description": "Can be used to turn on logging (logs to file in <user home>\/.camel directory)", "defaultValue": "false", "javaType": "boolean", "type": "boolean" }, { "names": "--logging-level", "description": "Logging level (ERROR, WARN, INFO, DEBUG, TRACE)", "d [...] - { "name": "semantic", "fullName": "semantic", "description": "List semantic definitions and expert contracts", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.action.CamelSemantic", "options": [ { "names": "--expert", "description": "Show the operations and parameter contract of this expert", "javaType": "java.lang.String", "type": "string" }, { "names": "--json", "description": "Output a single JSON document", "javaType": "boolean", "type": "boolean" }, { "names": "--timeou [...] + { "name": "semantic", "fullName": "semantic", "description": "List semantic definitions and expert contracts", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.action.CamelSemantic", "options": [ { "names": "--expert", "description": "Show the operations and parameter contract of this expert", "javaType": "java.lang.String", "type": "string" }, { "names": "--json", "description": "Output a single JSON document", "javaType": "boolean", "type": "boolean" }, { "names": "--timeou [...] { "name": "shell", "fullName": "shell", "description": "Interactive Camel CLI shell.", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.Shell", "options": [ { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ] }, { "name": "stop", "fullName": "stop", "description": "Shuts down running Camel integrations", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.process.StopProcess", "options": [ { "names": "--kill", "description": "To force killing the process (SIGKILL)", "javaType": "boolean", "type": "boolean" }, { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ] }, { "name": "top", "fullName": "top", "description": "Top status of Camel integrations", "sourceClass": "org.apache.camel.dsl.jbang.core.commands.process.CamelTop", "options": [ { "names": "--watch", "description": "Execute periodically and showing output fullscreen", "javaType": "boolean", "type": "boolean" }, { "names": "-h,--help", "description": "Display the help and sub-commands", "javaType": "boolean", "type": "boolean" } ], "subcommands": [ { "name": "context", "fullName": "top [...] diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticAudit.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticAudit.java index e6bf4ac0de66..79465f060a01 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticAudit.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticAudit.java @@ -16,6 +16,13 @@ */ package org.apache.camel.dsl.jbang.core.commands.action; +import java.io.IOException; +import java.nio.file.AccessDeniedException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystemException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.time.Instant; import java.time.format.DateTimeParseException; import java.util.List; @@ -29,6 +36,8 @@ import com.github.freva.asciitable.ColumnData; import com.github.freva.asciitable.HorizontalAlign; import org.apache.camel.dsl.jbang.core.commands.CamelJBangMain; import org.apache.camel.dsl.jbang.core.common.SemanticAuditFilters; +import org.apache.camel.dsl.jbang.core.common.SemanticImagePreview; +import org.apache.camel.dsl.jbang.core.common.SemanticMediaInput; import org.apache.camel.dsl.jbang.core.common.SemanticResultFormatter; import org.apache.camel.util.json.JsonArray; import org.apache.camel.util.json.JsonObject; @@ -112,6 +121,10 @@ public class SemanticAudit extends SemanticActionCommand { @CommandLine.Option(names = "--include-input", description = "Include untrusted captured input in an explicit case/export") boolean includeInput; + @CommandLine.Option(names = "--output", + description = "Write the full JSON response to a new file, including requested captured image data") + Path output; + public SemanticAudit(CamelJBangMain main) { super(main); } @@ -205,12 +218,31 @@ public class SemanticAudit extends SemanticActionCommand { if (Boolean.TRUE.equals(response.get("cursorExpired"))) { return error(1, "Audit cursor expired; start again without --cursor", response); } - return super.printResponse(response); + if (output != null) { + try { + Files.writeString(output, response.toJson(), StandardOpenOption.CREATE_NEW); + printer().println("Audit exported to " + displayText(output)); + return 0; + } catch (IOException e) { + String reason = e instanceof FileAlreadyExistsException ? "file already exists" + : e instanceof AccessDeniedException ? "permission denied" + : e instanceof FileSystemException failure && failure.getReason() != null + ? displayText(failure.getReason()) + : "I/O failure (" + e.getClass().getSimpleName() + ")"; + return error(1, "Cannot create audit export file: " + reason, null); + } + } + return super.printResponse(json ? (JsonObject) SemanticMediaInput.withoutImageData(response) : response); } private void renderAuditField(String key, Object value, String indent) { if ("input".equals(key)) { - renderInput("Input", value, indent); + renderInput("Input", SemanticMediaInput.withoutImageData(value), indent); + for (var preview : SemanticImagePreview.prepare(value)) { + preview.textLines().forEach(line -> printer().println(indent + line)); + } + } else if ("inputType".equals(key)) { + renderField("Input type", SemanticMediaInput.typeLabel(value), indent); } else { renderField(fieldLabel(key), value, indent); } @@ -266,6 +298,8 @@ public class SemanticAudit extends SemanticActionCommand { column("EVENT ID", "eventId"), column("TIMESTAMP", "timestamp"), column("EXECUTING ROUTE", "executingRouteId"), column("EVALUATION", "definition"), column("EXPERT", "expert"), + new Column().header("INPUT").dataAlign(HorizontalAlign.LEFT) + .with((JsonObject row) -> SemanticMediaInput.typeLabel(row.get("inputType"))), new Column().header("RESULT").dataAlign(HorizontalAlign.LEFT) .with((JsonObject row) -> SemanticResultFormatter.audit(row).compact()), column("EXECUTION", "status"), column("BREADCRUMB ID", "breadcrumbId")))); diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticEvaluate.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticEvaluate.java index 481aa5f9eb13..94ddf41c8e21 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticEvaluate.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticEvaluate.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import org.apache.camel.dsl.jbang.core.commands.CamelJBangMain; +import org.apache.camel.dsl.jbang.core.common.SemanticMediaInput; import org.apache.camel.util.json.JsonObject; import picocli.CommandLine; @@ -43,7 +44,7 @@ public class SemanticEvaluate extends SemanticActionCommand { @CommandLine.Option(names = "--evaluation", description = "Named semantic definition to evaluate") String evaluation; - @CommandLine.Option(names = "--body", description = "Sample message body (text, file:path or json:value)") + @CommandLine.Option(names = "--body", description = "Sample body (text, file:path, image:path or json:value)") String body; @CommandLine.Option(names = "--header", description = "Sample header (key=value; use json: for a typed value)") @@ -59,7 +60,7 @@ public class SemanticEvaluate extends SemanticActionCommand { @CommandLine.Option(names = "--operation", description = "Expert operation to call directly") String operation; - @CommandLine.Option(names = "--input", description = "Direct expert input (text, file:path or json:value)") + @CommandLine.Option(names = "--input", description = "Direct input (text, file:path, image:path or json:value)") String input; @CommandLine.Option(names = "--parameter", description = "Expert parameter (key=value; use json: for a typed value)") @@ -146,6 +147,9 @@ public class SemanticEvaluate extends SemanticActionCommand { } private static Object value(String value, boolean allowFile) { + if (allowFile && value.startsWith("image:")) { + return SemanticMediaInput.readImage(value); + } if (value.startsWith("json:")) { try { return JSON_VALUE.readValue(value.substring(5)); diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticImagePreview.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticImagePreview.java new file mode 100644 index 000000000000..1307ba4a33a3 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticImagePreview.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.camel.dsl.jbang.core.common; + +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +import javax.imageio.ImageIO; +import javax.imageio.stream.MemoryCacheImageInputStream; + +/** Bounded local thumbnail decoding. Never fetches a URL, opens a viewer, or writes image data to disk. */ +public record SemanticImagePreview(String description, List<List<Integer>> pixels) { + public SemanticImagePreview { + pixels = pixels.stream().map(List::copyOf).toList(); + } + + public static List<SemanticImagePreview> prepare(Object input) { + List<SemanticImagePreview> previews = new ArrayList<>(); + for (Object item : SemanticMediaInput.parts(input)) { + if (item instanceof Map<?, ?> part && "image".equals(part.get("type"))) { + previews.add(decode(part)); + } + } + return List.copyOf(previews); + } + + private static SemanticImagePreview decode(Map<?, ?> part) { + var metadata = SemanticMediaInput.imageDescription(part); + String label = "Image (" + metadata.getOrDefault("mediaType", "format not recorded") + ")"; + if (!(part.get("data") instanceof String data)) { + return unavailable(label, "payload was not retained"); + } + int start = SemanticMediaInput.imageDataOffset(data); + if (start < 0 || data.length() > 1_000_000) { + return unavailable(label, "invalid image or preview byte limit"); + } + try (var stream = new MemoryCacheImageInputStream( + new ByteArrayInputStream(Base64.getDecoder().decode(data.substring(start))))) { + var readers = ImageIO.getImageReaders(stream); + if (!readers.hasNext()) { + return unavailable(label, "no local decoder for this image"); + } + var reader = readers.next(); + try { + reader.setInput(stream, true, true); + int width = reader.getWidth(0); + int height = reader.getHeight(0); + if (width <= 0 || height <= 0 || (long) width * height > 40_000_000) { + return unavailable(label, "preview pixel limit"); + } + var parameters = reader.getDefaultReadParam(); + int sample = Math.max(1, (Math.max(width, height) + 63) / 64); + parameters.setSourceSubsampling(sample, sample, 0, 0); + BufferedImage image = reader.read(0, parameters); + List<List<Integer>> rows = new ArrayList<>(); + for (int y = 0; y < image.getHeight(); y++) { + List<Integer> row = new ArrayList<>(); + for (int x = 0; x < image.getWidth(); x++) { + row.add(onWhite(image.getRGB(x, y))); + } + rows.add(row); + } + return new SemanticImagePreview(label + " · " + width + " × " + height + " · terminal preview", rows); + } finally { + reader.dispose(); + } + } catch (Exception failure) { + return unavailable(label, "image could not be decoded"); + } + } + + private static SemanticImagePreview unavailable(String label, String reason) { + return new SemanticImagePreview(label + " · preview unavailable: " + reason, List.of()); + } + + private static int onWhite(int argb) { + int alpha = argb >>> 24; + int red = ((argb >> 16 & 255) * alpha + 255 * (255 - alpha)) / 255; + int green = ((argb >> 8 & 255) * alpha + 255 * (255 - alpha)) / 255; + int blue = ((argb & 255) * alpha + 255 * (255 - alpha)) / 255; + return red << 16 | green << 8 | blue; + } + + /** Plain terminal fallback; the TUI renders the same pixels with foreground/background colours. */ + public List<String> textLines() { + List<String> lines = new ArrayList<>(); + lines.add(description); + for (int y = 0; y < pixels.size(); y += 2) { + StringBuilder line = new StringBuilder(); + for (int rgb : pixels.get(y)) { + int luminance = ((rgb >> 16 & 255) * 299 + (rgb >> 8 & 255) * 587 + (rgb & 255) * 114) / 1000; + line.append(" .:-=+*#%@".charAt(luminance * 9 / 255)); + } + lines.add(line.toString()); + } + return List.copyOf(lines); + } +} diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInput.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInput.java new file mode 100644 index 000000000000..8d332eb02188 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInput.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.camel.dsl.jbang.core.common; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.apache.camel.util.json.JsonArray; +import org.apache.camel.util.json.JsonObject; + +/** Media input shared by the CLI and TUI; independent of expert implementations. */ +public final class SemanticMediaInput { + private static final String PARTS = "$semanticInput"; + + private SemanticMediaInput() { + } + + public static boolean isMedia(Object input) { + return input instanceof Map<?, ?> map && map.get(PARTS) instanceof List<?>; + } + + public static List<?> parts(Object input) { + return isMedia(input) ? (List<?>) ((Map<?, ?>) input).get(PARTS) : List.of(); + } + + /** Read only an explicitly requested local image. This must run off the TUI render thread. */ + public static Object readImage(String value) { + if (!value.startsWith("image:")) { + return value; + } + Path path = Path.of(value.substring(6)); + String name = path.getFileName().toString().toLowerCase(Locale.ROOT); + String type = name.endsWith(".png") ? "image/png" + : name.endsWith(".jpg") || name.endsWith(".jpeg") ? "image/jpeg" + : name.endsWith(".gif") ? "image/gif" : name.endsWith(".webp") ? "image/webp" : null; + if (type == null) { + throw new IllegalArgumentException("image: requires a local PNG, JPEG, GIF or WebP file"); + } + try { + String data = "data:" + type + ";base64," + Base64.getEncoder().encodeToString(Files.readAllBytes(path)); + return Map.of(PARTS, List.of(Map.of("type", "image", "data", data))); + } catch (IOException e) { + throw new IllegalArgumentException("Cannot read the selected image file", e); + } + } + + /** Preserve metadata in terminal/JSON displays without printing image payloads. */ + public static Object withoutImageData(Object value) { + if (value instanceof Map<?, ?> map) { + if (isMedia(map)) { + JsonArray parts = new JsonArray(); + for (Object item : parts(map)) { + parts.add(item instanceof Map<?, ?> part && "image".equals(part.get("type")) + ? imageDescription(part) : withoutImageData(item)); + } + JsonObject result = new JsonObject(); + result.put(PARTS, parts); + return result; + } + JsonObject result = new JsonObject(); + map.forEach((key, item) -> result.put(key.toString(), withoutImageData(item))); + return result; + } + if (value instanceof List<?> list) { + JsonArray result = new JsonArray(); + list.forEach(item -> result.add(withoutImageData(item))); + return result; + } + if (value instanceof String text && text.startsWith("data:image/")) { + return imageDescription(Map.of("type", "image", "data", text)); + } + return value; + } + + public static JsonObject imageDescription(Map<?, ?> part) { + JsonObject result = new JsonObject(); + result.put("type", "image"); + if (part.get("data") instanceof String data) { + int start = imageDataOffset(data); + if (start >= 0) { + result.put("mediaType", data.substring(5, start - 8)); + result.put("encodedCharacters", data.length() - start); + } + result.put("capture", "available"); + result.put("dataOmitted", "display"); + } else { + result.put("capture", "omitted"); + result.put("dataOmitted", part.getOrDefault("dataOmitted", null)); + if (part.get("mediaType") instanceof String type && type.matches("image/[a-z0-9.+-]{1,100}")) { + result.put("mediaType", type); + } + } + return result; + } + + /** Offset of the base64 payload, or -1 when the inline image prefix is invalid. */ + static int imageDataOffset(String data) { + int marker = data.indexOf(";base64,"); + return data.startsWith("data:image/") && marker > 5 && marker < 120 + && data.substring(5, marker).matches("image/[a-z0-9.+-]+") ? marker + 8 : -1; + } + + public static String typeLabel(Object type) { + return switch (type instanceof String text ? text : "") { + case "image" -> "Image"; + case "text-and-images" -> "Text + images"; + case "text" -> "Text"; + case "structured" -> "Structured"; + case "unsupported" -> "Unsupported input"; + case "unknown" -> "Not determined"; + default -> "Not recorded"; + }; + } +} diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticResultFormatter.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticResultFormatter.java index db7360ce142b..b971e713e1b9 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticResultFormatter.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticResultFormatter.java @@ -90,8 +90,9 @@ public final class SemanticResultFormatter { int lower = (int) Math.floor(position); int upper = (int) Math.ceil(position); lines.add(lower == upper - ? level(levels, lower) - : "Between " + level(levels, lower) + " and " + level(levels, upper)); + ? level(levels, lower, interpretation) + : "Between " + level(levels, lower, interpretation) + " and " + + level(levels, upper, interpretation)); } else { lines.add("Score outside the captured scale; numeric value preserved"); } @@ -161,7 +162,7 @@ public final class SemanticResultFormatter { } for (int i = 0; i < levels.size(); i++) { if (Integer.toString(i).equals(entry.getKey())) { - modes.add(level(levels, i)); + modes.add(level(levels, i, interpretation)); } } } @@ -178,8 +179,10 @@ public final class SemanticResultFormatter { return value instanceof Map<?, ?> map ? map : Map.of(); } - private static String level(List<?> levels, int index) { - return "[" + index + "] " + text(levels.get(index)); + private static String level(List<?> levels, int index, Map<?, ?> interpretation) { + String label = text(levels.get(index)); + Object description = map(interpretation.get("scoreDescriptions")).get(levels.get(index)); + return "[" + index + "] " + label + (description == null ? "" : " — " + text(description)); } private static String value(Object value) { diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticCommandTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticCommandTest.java index c5b2e473af00..5d01d33b5d1f 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticCommandTest.java +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticCommandTest.java @@ -58,6 +58,58 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; class SemanticCommandTest { + @Test + void imageFileBodyIsSentAsExplicitMedia() throws Exception { + Path image = home.resolve("input.png"); + Files.write(image, new byte[] { 1, 2, 3 }); + assertEquals(0, respond(new SemanticEvaluate(main()), request -> { + assertEquals(Map.of("$semanticInput", List.of(Map.of("type", "image", "data", "data:image/png;base64,AQID"))), + request.get("body")); + return jsonUnchecked("{\"status\":\"success\",\"value\":true}"); + }, "--evaluation=inspect", "--body=image:" + image)); + } + + @ParameterizedTest + @ValueSource(booleans = { true, false }) + void imageAuditDisplaysTypeWithoutPrintingPayload(boolean json) throws Exception { + var response = new JsonObject( + Map.of("record", new JsonObject( + Map.of("eventId", "image-event", "inputType", "image", + "input", Map.of("$semanticInput", + List.of(Map.of("type", "image", "data", "data:image/png;base64,AQID"))))))); + assertEquals(0, respond(new SemanticAudit(main()), request -> response, + "--event-id=image-event", "--json=" + json)); + if (json) { + Map<?, ?> input = (Map<?, ?>) json(printer.getOutput()).getMap("record").get("input"); + List<?> parts = (List<?>) input.get("$semanticInput"); + assertEquals("image/png", ((Map<?, ?>) parts.get(0)).get("mediaType")); + } else { + assertTrue(printer.getOutput().contains("image/png")); + } + assertFalse(printer.getOutput().contains("base64")); + if (!json) { + assertTrue(printer.getOutput().contains("Input type: Image")); + assertTrue(printer.getOutput().contains("preview unavailable")); + } + } + + @Test + void explicitFileExportPreservesCapturedImageAndNeverOverwrites() throws Exception { + Path target = home.resolve("audit.json"); + var response = new JsonObject( + Map.of("record", new JsonObject( + Map.of("eventId", "image-event", "inputType", "image", + "input", Map.of("$semanticInput", + List.of(Map.of("type", "image", "data", "data:image/png;base64,AQID"))))))); + assertEquals(0, respond(new SemanticAudit(main()), request -> response, + "--event-id=image-event", "--output=" + target)); + assertEquals(response, json(Files.readString(target))); + assertFalse(printer.getOutput().contains("base64")); + assertEquals(1, respond(new SemanticAudit(main()), request -> response, + "--event-id=image-event", "--output=" + target)); + assertTrue(errors.toString().contains("file already exists")); + assertEquals(response, json(Files.readString(target))); + } @TempDir Path home; diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInputTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInputTest.java new file mode 100644 index 000000000000..68cbd070f22d --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInputTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.camel.dsl.jbang.core.common; + +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; +import java.nio.file.Path; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +import javax.imageio.ImageIO; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SemanticMediaInputTest { + @TempDir + Path directory; + + @Test + void imageFilesBecomeExplicitMediaAndDisplaysNeverContainBase64() throws Exception { + var image = new BufferedImage(4, 4, BufferedImage.TYPE_INT_RGB); + image.setRGB(0, 0, 0xff0000); + Path path = directory.resolve("sample.png"); + ImageIO.write(image, "png", path.toFile()); + Object input = SemanticMediaInput.readImage("image:" + path); + assertTrue(SemanticMediaInput.isMedia(input)); + var description = SemanticMediaInput.withoutImageData(input).toString(); + assertTrue(description.contains("image/png")); + assertTrue(description.contains("available")); + assertFalse(description.contains("base64")); + var previews = SemanticImagePreview.prepare(input); + assertEquals(1, previews.size()); + assertEquals(0xff0000, previews.get(0).pixels().get(0).get(0)); + assertTrue(previews.get(0).description().contains("4 × 4")); + assertFalse(previews.get(0).textLines().toString().contains("base64")); + } + + @Test + void previewIsBoundedAndCorruptOrOmittedImagesAreExplicit() throws Exception { + var image = new BufferedImage(200, 100, BufferedImage.TYPE_INT_RGB); + var buffer = new ByteArrayOutputStream(); + ImageIO.write(image, "png", buffer); + String data = "data:image/png;base64," + Base64.getEncoder().encodeToString(buffer.toByteArray()); + var input = Map.of("$semanticInput", List.of(Map.of("type", "image", "data", data), + Map.of("type", "image", "data", "data:image/png;base64,broken"), + Map.of("type", "image", "dataOmitted", "snapshot_limit", "mediaType", "image/png"))); + var previews = SemanticImagePreview.prepare(input); + assertTrue(previews.get(0).pixels().get(0).size() <= 64); + assertTrue(previews.get(0).pixels().size() <= 64); + assertTrue(previews.get(1).pixels().isEmpty()); + assertTrue(previews.get(1).description().contains("unavailable")); + assertTrue(previews.get(2).description().contains("not retained")); + assertEquals("Text + images", SemanticMediaInput.typeLabel("text-and-images")); + } + + @Test + void transparentPixelsAreCompositedOnWhite() throws Exception { + var image = new BufferedImage(2, 1, BufferedImage.TYPE_INT_ARGB); + image.setRGB(0, 0, 0x00ff0000); + image.setRGB(1, 0, 0x80000000); + var buffer = new ByteArrayOutputStream(); + ImageIO.write(image, "png", buffer); + String data = "data:image/png;base64," + Base64.getEncoder().encodeToString(buffer.toByteArray()); + var input = Map.of("$semanticInput", List.of(Map.of("type", "image", "data", data))); + var preview = SemanticImagePreview.prepare(input).get(0); + assertEquals(List.of(0xffffff, 0x7f7f7f), preview.pixels().get(0)); + } + + @Test + void ordinaryStateAndTextKeepTheirMeaning() { + Object input = Map.of("type", "image", "data", "application state"); + assertEquals(input, SemanticMediaInput.withoutImageData(input)); + assertEquals("https://example.com/picture.png";, SemanticMediaInput.readImage("https://example.com/picture.png";)); + } +} diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/SemanticResultFormatterTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/SemanticResultFormatterTest.java index 42777ea03fff..aa5ef92e8325 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/SemanticResultFormatterTest.java +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/SemanticResultFormatterTest.java @@ -61,6 +61,16 @@ class SemanticResultFormatterTest { assertFalse(String.join(" ", answer.details()).contains("severity")); } + @Test + void includesCapturedRubricDescriptionsWithoutChangingTheScore() { + var answer = SemanticResultFormatter.audit(Map.of("status", "success", "semantics", Map.of("resultType", "SCORE"), + "result", Map.of("value", 0.96), "interpretation", Map.of( + "scoreLevels", List.of("routine", "urgent"), + "scoreDescriptions", Map.of("routine", "Within a week", "urgent", "Immediate action")))); + assertEquals("0.96 [SCORE]", answer.compact()); + assertTrue(answer.details().contains("Between [0] routine — Within a week and [1] urgent — Immediate action")); + } + @Test void reportsIncompleteCapturedDomainWithoutInventingMissingProbabilities() { var view = SemanticResultFormatter.audit(Map.of("semantics", Map.of("resultType", "CHOICE"), diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticAuditView.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticAuditView.java index c89cb843936b..2120ee94b14b 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticAuditView.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticAuditView.java @@ -59,6 +59,7 @@ import dev.tamboui.widgets.table.Row; import dev.tamboui.widgets.table.Table; import dev.tamboui.widgets.table.TableState; import org.apache.camel.dsl.jbang.core.common.SemanticAuditFilters; +import org.apache.camel.dsl.jbang.core.common.SemanticMediaInput; import org.apache.camel.dsl.jbang.core.common.SemanticResultFormatter; import org.apache.camel.util.json.JsonObject; @@ -71,6 +72,7 @@ final class SemanticAuditView { = Stream.concat(SemanticAuditFilters.QUERY_FIELDS.stream(), Stream.of("page.status", "page.result", "page.definition")).collect(Collectors.toUnmodifiableSet()); private final MonitorContext context; + private final SemanticImageView imageView; private final LongSupplier nanoTime; private final Supplier<Instant> now; private long lastRefresh; @@ -121,11 +123,13 @@ final class SemanticAuditView { SemanticAuditView(MonitorContext context, LongSupplier nanoTime, Supplier<Instant> now) { this.context = context; + this.imageView = new SemanticImageView(context.backgroundExecutor); this.nanoTime = nanoTime; this.now = now; } void reset() { + imageView.reset(); if (pending != null) { pending.cancel(true); } @@ -373,6 +377,7 @@ final class SemanticAuditView { } caseIndex = 0; showInput = false; + imageView.reset(); details = null; scroll = 0; if (pendingDetails != null) { @@ -875,7 +880,8 @@ final class SemanticAuditView { String expert = text(row, "expert") + (wide && row.get("provider") != null ? " / " + text(row, "provider") : ""); content.add(wide ? Row.from(Cell.from(time(row, "MM-dd HH:mm:ss")), Cell.from(executingRoute(row)), - Cell.from(text(row, "definition")), Cell.from(expert), result, execution, + Cell.from(text(row, "definition")), Cell.from(expert), + Cell.from(SemanticMediaInput.typeLabel(row.get("inputType"))), result, execution, Cell.from(duration(row)), Cell.from(text(row, "breadcrumbId"))) : Row.from(Cell.from(time(row, "MM-dd HH:mm:ss")), Cell.from(executingRoute(row)), Cell.from(expert), result, execution)); @@ -895,11 +901,13 @@ final class SemanticAuditView { } Table table = Table.builder().rows(content) .header((wide - ? Row.from("TIMESTAMP", "EXECUTING ROUTE", "EVALUATION", "EXPERT / PROVIDER", "RESULT", "EXECUTION", + ? Row.from("TIMESTAMP", "EXECUTING ROUTE", "EVALUATION", "EXPERT / PROVIDER", "INPUT", "RESULT", + "EXECUTION", "DURATION", "BREADCRUMB ID") : Row.from("TIMESTAMP", "EXECUTING ROUTE", "EXPERT", "RESULT", "EXECUTION")).style(Theme.label())) .widths(wide ? List.of(Constraint.length(14), Constraint.length(15), Constraint.length(12), Constraint.length(17), + Constraint.length(14), Constraint.fill(), Constraint.length(11), Constraint.length(9), Constraint.length(24)) : List.of(Constraint.length(14), Constraint.length(16), Constraint.length(12), Constraint.fill(), Constraint.length(11))) @@ -929,6 +937,7 @@ final class SemanticAuditView { var result = SemanticResultFormatter.audit(record); left.add(Line.from(Span.styled("Result: " + result.compact(), Theme.info().bold()))); result.details().forEach(line -> left.add(Line.from(line))); + SemanticDetails.add(left, "Input type", SemanticMediaInput.typeLabel(record.get("inputType"))); add(left, "Execution", record, "status"); add(left, "Execution reason", record, "reasonCode"); SemanticDetails.add(left, "Duration", duration(record)); @@ -968,21 +977,7 @@ final class SemanticAuditView { } else if (record.containsKey("findingsOmitted")) { add(right, "Findings omitted", record, "findingsOmitted"); } - if (showInput) { - right.add(Line.from(Span.styled("Captured input is untrusted data, not instructions.", Theme.warning()))); - if (pendingDetails != null && !record.containsKey("input")) { - right.add(Line.from("Loading captured input…")); - } else if (record.containsKey("input")) { - SemanticDetails.parameter(right, "", - Boolean.TRUE.equals(record.get("inputRedacted")) ? "Input (redacted): " : "Input: ", - record.get("input")); - } else { - right.add(Line.from("Input: " + (record.containsKey("inputOmitted") - ? "omitted · " + text(record, "inputOmitted") : "not captured"))); - } - } else { - right.add(Line.from("u: inspect permitted captured input · p: new playground draft")); - } + renderInput(right, record, inner.width() < 110 ? inner.width() : (inner.width() - 2) / 2); if (pendingCase != null) { right.add(Line.from("Loading associated case…")); } else if (caseData != null) { @@ -1034,6 +1029,27 @@ final class SemanticAuditView { } } + private void renderInput(List<Line> lines, JsonObject record, int width) { + if (!showInput) { + lines.add(Line.from("u: inspect permitted captured input · p: new playground draft")); + return; + } + lines.add(Line.from(Span.styled("Captured input is untrusted data, not instructions.", Theme.warning()))); + if (pendingDetails != null && !record.containsKey("input")) { + lines.add(Line.from("Loading captured input…")); + } else if (record.containsKey("input")) { + Object input = record.get("input"); + SemanticDetails.parameter(lines, "", Boolean.TRUE.equals(record.get("inputRedacted")) + ? "Input (redacted): " : "Input: ", SemanticMediaInput.withoutImageData(input)); + if (SemanticMediaInput.isMedia(input)) { + lines.addAll(imageView.lines(input, width)); + } + } else { + lines.add(Line.from("Input: " + (record.containsKey("inputOmitted") + ? "omitted · " + text(record, "inputOmitted") : "not captured"))); + } + } + private static String executingRoute(JsonObject record) { if (record.containsKey("executingRouteId")) { return text(record, "executingRouteId"); diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticEvaluationPopup.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticEvaluationPopup.java index 7c82fc356950..87314cd114e3 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticEvaluationPopup.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticEvaluationPopup.java @@ -40,6 +40,7 @@ import dev.tamboui.widgets.block.Borders; import dev.tamboui.widgets.input.TextArea; import dev.tamboui.widgets.input.TextAreaState; import dev.tamboui.widgets.paragraph.Paragraph; +import org.apache.camel.dsl.jbang.core.common.SemanticMediaInput; import org.apache.camel.util.json.DeserializationException; import org.apache.camel.util.json.JsonObject; import org.apache.camel.util.json.Jsoner; @@ -201,7 +202,13 @@ final class SemanticEvaluationPopup { submittedInterpretation = SemanticResultView.interpretation(operation, definition.get("parameters"), submittedScoreLevels); pending = CompletableFuture - .supplyAsync(() -> ctx.executeIndependentAction(pid, request, 60000), ctx.backgroundExecutor) + .supplyAsync(() -> { + JsonObject prepared = new JsonObject(request); + if (prepared.get("body") instanceof String text) { + prepared.put("body", SemanticMediaInput.readImage(text)); + } + return ctx.executeIndependentAction(pid, prepared, 60000); + }, ctx.backgroundExecutor) .exceptionally(SemanticResultView::failure); } catch (Exception invalid) { error = "Input error: " + invalid.getMessage(); diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticImageView.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticImageView.java new file mode 100644 index 000000000000..c20e385d89ac --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticImageView.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.tui; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; + +import dev.tamboui.style.Color; +import dev.tamboui.style.Style; +import dev.tamboui.text.Line; +import dev.tamboui.text.Span; +import org.apache.camel.dsl.jbang.core.common.SemanticImagePreview; + +/** Decode off the render thread; draw only bounded pixels from the selected captured input. */ +final class SemanticImageView { + private final Executor executor; + private Object selected; + private CompletableFuture<List<SemanticImagePreview>> pending; + + SemanticImageView(Executor executor) { + this.executor = executor; + } + + void reset() { + if (pending != null) { + pending.cancel(true); + } + selected = null; + pending = null; + } + + List<Line> lines(Object input, int width) { + if (selected != input) { + reset(); + selected = input; + pending = CompletableFuture.supplyAsync(() -> SemanticImagePreview.prepare(input), executor) + .exceptionally(failure -> List.of(new SemanticImagePreview("Image preview unavailable", List.of()))); + } + if (pending == null || !pending.isDone()) { + return List.of(Line.from("Preparing image preview…")); + } + List<Line> result = new ArrayList<>(); + for (var preview : pending.getNow(List.of())) { + result.add(Line.from(Span.styled(preview.description(), Theme.label()))); + result.addAll(pixels(preview, width)); + } + return result; + } + + private static List<Line> pixels(SemanticImagePreview preview, int availableWidth) { + List<List<Integer>> pixels = preview.pixels(); + if (pixels.isEmpty() || availableWidth <= 0) { + return List.of(); + } + int sourceWidth = pixels.get(0).size(); + int width = Math.min(sourceWidth, availableWidth); + int height = Math.max(1, pixels.size() * width / sourceWidth); + List<Line> lines = new ArrayList<>(); + for (int y = 0; y < height; y += 2) { + List<Span> spans = new ArrayList<>(); + for (int x = 0; x < width; x++) { + int sourceX = x * sourceWidth / width; + int top = pixels.get(y * pixels.size() / height).get(sourceX); + int bottom = pixels.get(Math.min(y + 1, height - 1) * pixels.size() / height).get(sourceX); + spans.add(Span.styled("▀", Style.EMPTY.fg(color(top)).bg(color(bottom)))); + } + lines.add(Line.from(spans)); + } + return lines; + } + + private static Color color(int rgb) { + return Color.rgb(rgb >> 16 & 255, rgb >> 8 & 255, rgb & 255); + } +} diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticPlayground.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticPlayground.java index 0636eba05e9c..54c760bd48dd 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticPlayground.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticPlayground.java @@ -36,6 +36,7 @@ import dev.tamboui.widgets.block.BorderType; import dev.tamboui.widgets.block.Borders; import dev.tamboui.widgets.input.TextArea; import dev.tamboui.widgets.input.TextAreaState; +import org.apache.camel.dsl.jbang.core.common.SemanticMediaInput; import org.apache.camel.util.json.JsonObject; import org.apache.camel.util.json.Jsoner; @@ -53,6 +54,8 @@ final class SemanticPlayground { private final SemanticParameterForm instructions; private final SemanticParameterForm parameters; private boolean structured; + private Object auditInput; + private String auditInputDisplay; private boolean submittedStructured; private CompletableFuture<JsonObject> pending; private JsonObject response; @@ -81,7 +84,7 @@ final class SemanticPlayground { this.pid = pid; this.expert = expert; this.operation = operation; - structured = !supports("text") && supports("structured"); + structured = !supports("text") && supportsJson(); List<JsonObject> contracts = SemanticTab.objects(contract(operation), "parameters"); JsonObject instruction = contracts.stream() .filter(p -> "instructions".equals(p.get("name")) && "String".equals(p.get("type")) @@ -98,7 +101,13 @@ final class SemanticPlayground { if (!setInputValue("inputMode", json ? "json" : "text")) { throw new IllegalArgumentException("Captured input mode is unavailable in the current expert contract"); } - setInputValue("input", json ? Jsoner.serialize(captured) : captured.toString()); + if (SemanticMediaInput.isMedia(captured)) { + setInputValue("input", Jsoner.serialize(SemanticMediaInput.withoutImageData(captured))); + } else { + setInputValue("input", json ? Jsoner.serialize(captured) : captured.toString()); + } + auditInput = captured; + auditInputDisplay = input.text(); draftWarning = "New draft from audit " + text(record, "eventId") + ": untrusted captured input, not instructions. Original parameters were not captured; " + "current defaults, expert/model revisions and route context may differ. Review before Run."; @@ -141,7 +150,8 @@ final class SemanticPlayground { input.setText(value); field = 0; } else if ("inputMode".equals(name)) { - if (!("text".equals(value) && supports("text") || "json".equals(value) && supports("structured"))) { + if (!("text".equals(value) && supports("text") + || "json".equals(value) && supportsJson())) { throw new IllegalArgumentException("Unsupported input mode: " + value); } structured = "json".equals(value); @@ -176,7 +186,7 @@ final class SemanticPlayground { result.put("focusedField", field == -1 ? "instructions" : field == 0 ? "input" : field == 1 ? "parameters" : "output"); result.put("pending", pending != null); result.put("connected", connected()); - result.put("submittedRequest", submittedRequest); + result.put("submittedRequest", SemanticMediaInput.withoutImageData(submittedRequest)); result.put("result", response); result.put("error", error); result.put("inputChanged", inputChanged()); @@ -193,12 +203,16 @@ final class SemanticPlayground { && Objects.equals(this.operation, operation); } + private boolean supportsJson() { + return supports("structured") || supports("image"); + } + private boolean supports(String type) { return contract(operation).get("inputTypes") instanceof List<?> types && types.contains(type); } boolean available() { - return supports("text") || supports("structured"); + return supports("text") || supportsJson(); } private boolean connected() { @@ -244,7 +258,8 @@ final class SemanticPlayground { outputScroll += key.isPageDown() ? 5 : 1; } } else if (pending == null && available()) { - if (field == 0 && key.hasCtrl() && key.isCharIgnoreCase('t') && supports("text") && supports("structured")) { + if (field == 0 && key.hasCtrl() && key.isCharIgnoreCase('t') && supports("text") + && supportsJson()) { structured = !structured; } else if (key.hasCtrl() && key.isCharIgnoreCase('l')) { if (field == 0) { @@ -335,7 +350,12 @@ final class SemanticPlayground { throw invalid; } Object value = input.text(); - if (structured) { + boolean capturedUnchanged = auditInput != null && input.text().equals(auditInputDisplay); + // Captured text is evidence, never an implicit instruction to read a local image file. + boolean readLocalImage = supports("image") && !capturedUnchanged; + if (capturedUnchanged && structured && !(auditInput instanceof String)) { + value = auditInput; + } else if (structured && !(readLocalImage && input.text().startsWith("image:"))) { value = Jsoner.deserialize(input.text()); if (!(value instanceof JsonObject || value instanceof List<?>)) { throw new IllegalArgumentException("Structured input must be a JSON object or array."); @@ -355,7 +375,13 @@ final class SemanticPlayground { submittedInstructions = instructions.revision(); submittedParameters = parameters.revision(); pending = CompletableFuture - .supplyAsync(() -> ctx.executeIndependentAction(pid, request, 60000), ctx.backgroundExecutor) + .supplyAsync(() -> { + JsonObject prepared = new JsonObject(request); + if (readLocalImage && prepared.get("input") instanceof String text) { + prepared.put("input", SemanticMediaInput.readImage(text)); + } + return ctx.executeIndependentAction(pid, prepared, 60000); + }, ctx.backgroundExecutor) .exceptionally(SemanticResultView::failure); } catch (Exception invalid) { error = "Input error: " + invalid.getMessage(); @@ -457,7 +483,8 @@ final class SemanticPlayground { } String label = instructions.isEmpty() ? "Input" : structured ? "State to assess" : "Text to assess"; inputArea = renderEditor(frame, stateBox, input, label + " · " + (structured ? "JSON" : "text"), - focused && field == 0, structured ? "Paste a JSON object or array" : "Type or paste the content to assess"); + focused && field == 0, supports("image") ? "Text, image:/path, or a JSON media envelope" + : structured ? "Paste a JSON object or array" : "Type or paste the content to assess"); int outputY = inputBox.bottom(); if (sideBySide) { Rect paramsBox = new Rect(inputBox.right() + 1, y, inner.right() - inputBox.right() - 2, editorHeight); @@ -534,7 +561,7 @@ final class SemanticPlayground { } hint(spans, "Ctrl+e", expanded ? "restore" : "expand"); if (field == 0) { - if (supports("text") && supports("structured")) { + if (supports("text") && supportsJson()) { hint(spans, "Ctrl+t", "text/JSON"); } hintLast(spans, "Enter", "new line"); diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticResultView.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticResultView.java index e0cd6ccd00c5..3398978bc554 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticResultView.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticResultView.java @@ -97,6 +97,10 @@ final class SemanticResultView { } } if (parameters instanceof Map<?, ?> values) { + Object descriptions = contract.get("scoreDescriptionsParameter"); + if (descriptions instanceof String parameter && values.get(parameter) instanceof Map<?, ?> labels) { + result.put("scoreDescriptions", Map.copyOf(labels)); + } for (String kind : List.of("choice", "boolean")) { Object name = contract.get(kind + "LabelsParameter"); if (name instanceof String parameter && values.get(parameter) instanceof Map<?, ?> labels) { diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/semantic.md b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/semantic.md index da6ee25580d6..b653df459c1a 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/semantic.md +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/resources/tui/help/semantic.md @@ -98,6 +98,11 @@ JSON), `sample.body` (text), or `sample.headers` / `sample.variables` (JSON obje Reading or editing never runs the expert; Ctrl+r explicitly evaluates the draft. +Image-capable experts accept `image:/path/picture.png` or a JSON `$semanticInput` envelope. +The local file is read only on Run. An audit image draft keeps its captured payload out of the +editor and submits it only on an explicit Run; omitted image data cannot be reconstructed. +Unchanged text loaded from audit remains literal text, including an `image:` prefix. + ## Audit history Choose **Audit** to inspect retained evaluations and request outcomes. Result and Execution are separate. @@ -126,6 +131,11 @@ reported probability diagnostics. Historical interpretation is captured once; it were not captured, and current defaults/model revisions/route context may differ. Review the warning and parameters before Run. The inspector retains full exchange and breadcrumb IDs; tables prioritize executing route, evaluation, expert, Result and Execution. +The Input column and inspector identify images even with payload capture disabled. `u` shows +inline colour previews when captured image data and a local decoder are available. Oversized, +missing or undecodable images show a reason. Preview generation runs in the background, never +fetches a URL, and does not print base64. Stock JDKs may lack a WebP decoder. + Health distinguishes capture settings, reader scope, retention, drops, evictions and delivery/query failures. An active OpenTelemetry tracer does not prove an external collector received a trace. diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticImageViewTest.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticImageViewTest.java new file mode 100644 index 000000000000..a293f8df8b44 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticImageViewTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.tui; + +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import javax.imageio.ImageIO; + +import dev.tamboui.tui.event.KeyEvent; +import dev.tamboui.tui.event.KeyModifiers; +import org.apache.camel.util.json.JsonObject; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +class SemanticImageViewTest { + @Test + void imageDecodingIsDeferredAndRenderingUsesBoundedColoredPixels() throws Exception { + var image = new BufferedImage(4, 4, BufferedImage.TYPE_INT_RGB); + image.setRGB(0, 0, 0xff0000); + var bytes = new ByteArrayOutputStream(); + ImageIO.write(image, "png", bytes); + var input = Map.of("$semanticInput", List.of(Map.of("type", "image", "data", + "data:image/png;base64," + Base64.getEncoder().encodeToString(bytes.toByteArray())))); + List<Runnable> work = new ArrayList<>(); + var view = new SemanticImageView(work::add); + assertThat(view.lines(input, 2)).hasSize(1); + assertThat(work).hasSize(1); + work.get(0).run(); + var lines = view.lines(input, 2); + assertThat(lines).hasSize(2); + assertThat(lines.get(1).spans()).hasSize(2); + assertThat(lines.get(1).spans().get(0).content()).isEqualTo("▀"); + assertThat(work).hasSize(1); + view.reset(); + view.lines(input, 2); + assertThat(work).hasSize(2); + } + + @Test + void capturedImageDraftKeepsPayloadOutOfTheEditorAndSendsItOnlyOnRun() { + var runtime = new MonitorContext(new AtomicReference<>(List.of()), new AtomicReference<>(List.of())) { + final List<JsonObject> calls = new CopyOnWriteArrayList<>(); + + @Override + JsonObject executeIndependentAction(String pid, JsonObject request, long timeout) { + calls.add(request); + return new JsonObject(Map.of("status", "success", "value", true)); + } + }; + try { + var integration = new IntegrationInfo(); + integration.pid = "image"; + runtime.data.set(List.of(integration)); + runtime.selectedPid = "image"; + var operation = new JsonObject( + Map.of("name", "boolean", "contract", new JsonObject( + Map.of("inputTypes", List.of("text", "image"), "parameters", List.of())))); + var playground = new SemanticPlayground(runtime, "image", "expert", operation); + var input = Map.of("$semanticInput", List.of(Map.of("type", "image", "data", "data:image/png;base64,AQID"))); + playground.loadAuditInput(new JsonObject(Map.of("eventId", "record", "input", input))); + assertThat(playground.snapshot().toJson()).doesNotContain("base64", "AQID"); + assertThat(runtime.calls).isEmpty(); + playground.handleKeyEvent(KeyEvent.ofChar('r', KeyModifiers.CTRL)); + await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> assertThat(runtime.calls).hasSize(1)); + assertThat(runtime.calls.get(0).get("input")).isEqualTo(input); + assertThat(playground.snapshot().toJson()).doesNotContain("base64", "AQID"); + } finally { + runtime.backgroundExecutor.shutdownNow(); + } + } + + @Test + void capturedTextIsNeverInterpretedAsALocalImagePath(@TempDir Path directory) throws Exception { + Path file = directory.resolve("captured.png"); + Files.write(file, new byte[] { 1, 2, 3 }); + var runtime = new MonitorContext(new AtomicReference<>(List.of()), new AtomicReference<>(List.of())) { + final List<JsonObject> calls = new CopyOnWriteArrayList<>(); + + @Override + JsonObject executeIndependentAction(String pid, JsonObject request, long timeout) { + calls.add(request); + return new JsonObject(Map.of("status", "success", "value", true)); + } + }; + try { + var integration = new IntegrationInfo(); + integration.pid = "image"; + runtime.data.set(List.of(integration)); + runtime.selectedPid = "image"; + var operation = new JsonObject( + Map.of("name", "boolean", "contract", new JsonObject( + Map.of("inputTypes", List.of("text", "image"), "parameters", List.of())))); + var playground = new SemanticPlayground(runtime, "image", "expert", operation); + String input = "image:" + file; + playground.loadAuditInput(new JsonObject(Map.of("eventId", "record", "input", input))); + assertThat(runtime.calls).isEmpty(); + playground.handleKeyEvent(KeyEvent.ofChar('r', KeyModifiers.CTRL)); + await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> assertThat(runtime.calls).hasSize(1)); + assertThat(runtime.calls.get(0).get("input")).isEqualTo(input); + var typed = new SemanticPlayground(runtime, "image", "expert", operation); + assertThat(typed.setInputValue("input", input)).isTrue(); + typed.handleKeyEvent(KeyEvent.ofChar('r', KeyModifiers.CTRL)); + await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> assertThat(runtime.calls).hasSize(2)); + assertThat(runtime.calls.get(1).get("input")).isEqualTo( + Map.of("$semanticInput", List.of(Map.of("type", "image", "data", "data:image/png;base64,AQID")))); + } finally { + runtime.backgroundExecutor.shutdownNow(); + } + } +}
![https://example.com/picture.png"](https://example.com/picture.png"){kind=link}