This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch CAMEL-26202-langchain4j-agent-content in repository https://gitbox.apache.org/repos/asf/camel.git
commit eef67323a031201f7caa0deb7d706de519f1af29 Author: Claus Ibsen <[email protected]> AuthorDate: Tue Nov 3 00:16:50 2026 +0100 CAMEL-26202: camel-langchain4j-agent - send a file, bytes or stream content once without a user message, accept JSON/XML/YAML and File/Path bodies A file, bytes or stream body without the CamelLangChain4jAgentUserMessage header got an empty user message text, and LangChain4j then put the content twice in the user message. An AiAgentBody with a blank user message and a content failed with "template cannot be null or blank". The agents now call new content-only service methods when the user message is empty or blank. application/json, application/xml, application/yaml and +json/+xml types are sent as a TextContent instead of failing with Unsupported MIME type, and MIME types match case-insensitively. A java.io.File or java.nio.file.Path body is converted like a WrappedFile. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../catalog/components/langchain4j-agent.json | 2 +- .../catalog/docs/langchain4j-agent-multimodal.adoc | 15 ++ .../langchain4j/agent/api/AgentWithMemory.java | 8 +- .../langchain4j/agent/api/AgentWithoutMemory.java | 8 +- .../agent/api/AiAgentWithMemoryService.java | 21 ++ .../agent/api/AiAgentWithoutMemoryService.java | 18 ++ .../agent/LangChain4jAgentConverterLoader.java | 16 ++ .../langchain4j/agent/langchain4j-agent.json | 2 +- .../main/docs/langchain4j-agent-multimodal.adoc | 15 ++ .../agent/LangChain4jAgentConverter.java | 77 ++++++- .../agent/LangChain4jAgentEndpoint.java | 11 +- .../agent/LangChain4jAgentContentBodyTest.java | 252 +++++++++++++++++++++ 12 files changed, 429 insertions(+), 16 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/langchain4j-agent.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/langchain4j-agent.json index 725bac3f9159..d95ae61ea863 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/langchain4j-agent.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/langchain4j-agent.json @@ -69,7 +69,7 @@ "CamelLangChain4jAgentModerationFlagged": { "index": 14, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "Boolean", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Set to true when user input is rejected by the configured moderation model; not set on success.", "constantName": "org.apache.camel.component.langchain4j.agent.api.Headers#MODERATION_FLAGGED" } }, "bodyTypes": { - "producerIn": { "carriers": [ "text", "pojo", "bytes", "stream", "file" ], "description": "The user message as text, or an AiAgentBody with the user message, the system message, the memory id and a content. Bytes, a stream or a file of the file consumers is sent as an image, audio, video, PDF or text content after the text of the CamelLangChain4jAgentUserMessage header; bytes and a stream need the CamelLangChain4jAgentMediaType or Content-Type header. A Map, a java.io.File or another [...] + "producerIn": { "carriers": [ "text", "pojo", "bytes", "stream", "file" ], "description": "The user message as text, or an AiAgentBody with the user message, the system message, the memory id and a content. Bytes, a stream, a file of the file consumers, a java.io.File or a java.nio.file.Path is sent as an image, audio, video, PDF or text content (JSON, XML and YAML as text), after the text of the CamelLangChain4jAgentUserMessage header or alone without it; bytes and a stream need the [...] "producerOut": { "carriers": [ "text" ], "description": "The answer of the agent as text, also with outputClass or jsonSchema: unmarshal the JSON to read its fields. The token counts, the finish reason, the sources and the tool executions are in headers" } }, "properties": { diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-agent-multimodal.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-agent-multimodal.adoc index efe1792b0744..3c4473b38898 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-agent-multimodal.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-agent-multimodal.adoc @@ -52,8 +52,23 @@ The agent component automatically converts files from various Camel components t |`InputStream` |Various streaming components |From content type headers (required) + +|`java.io.File`, `java.nio.file.Path` +|A local file set as the message body +|From file extension or headers |=== +The MIME type selects the content sent to the model: + +* `image/*`, `audio/*` and `video/*` are sent as an image, audio or video content. +* `application/pdf` is sent as a PDF content. +* `text/*`, `application/json`, `application/xml`, `application/yaml` and types with a `+json` or `+xml` suffix are sent as a text content (read as UTF-8). + +Any other MIME type fails. + +The text of the `CamelLangChain4jAgentUserMessage` header is sent before the content in the same user message. +Without that header (or when it is blank), the user message holds only the content. + ==== Example: Processing Images from File Component [tabs] diff --git a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithMemory.java b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithMemory.java index a14a13d0a98e..647a56541616 100644 --- a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithMemory.java +++ b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithMemory.java @@ -20,6 +20,7 @@ import dev.langchain4j.data.message.Content; import dev.langchain4j.service.AiServices; import dev.langchain4j.service.Result; import dev.langchain4j.service.tool.ToolProvider; +import org.apache.camel.util.ObjectHelper; /** * Implementation of Agent for AI agents with memory support. This agent handles chat interactions while maintaining @@ -45,7 +46,12 @@ public class AgentWithMemory extends AbstractAgent<AiAgentWithMemoryService> { String systemMessage = aiAgentBody.getSystemMessage(); Content content = aiAgentBody.getContent(); - if (content != null) { + if (content != null && ObjectHelper.isEmpty(userMessage)) { + // Only the content (such as a file) without a text: an empty or blank text must not be sent + return systemMessage != null + ? agentService.chat(memoryId, content, systemMessage) + : agentService.chat(memoryId, content); + } else if (content != null) { // Multi-modal message with content return systemMessage != null ? agentService.chat(memoryId, userMessage, content, systemMessage) diff --git a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithoutMemory.java b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithoutMemory.java index 7970237d3276..5c74380aa05c 100644 --- a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithoutMemory.java +++ b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithoutMemory.java @@ -20,6 +20,7 @@ import dev.langchain4j.data.message.Content; import dev.langchain4j.service.AiServices; import dev.langchain4j.service.Result; import dev.langchain4j.service.tool.ToolProvider; +import org.apache.camel.util.ObjectHelper; /** * Implementation of Agent for AI agents without memory support. This agent handles chat interactions without @@ -44,7 +45,12 @@ public class AgentWithoutMemory extends AbstractAgent<AiAgentWithoutMemoryServic String systemMessage = aiAgentBody.getSystemMessage(); Content content = aiAgentBody.getContent(); - if (content != null) { + if (content != null && ObjectHelper.isEmpty(userMessage)) { + // Only the content (such as a file) without a text: an empty or blank text must not be sent + return systemMessage != null + ? agentService.chat(content, systemMessage) + : agentService.chat(content); + } else if (content != null) { // Multi-modal message with content return systemMessage != null ? agentService.chat(userMessage, content, systemMessage) diff --git a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithMemoryService.java b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithMemoryService.java index 48be3234c8bd..01c9e160a152 100644 --- a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithMemoryService.java +++ b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithMemoryService.java @@ -48,6 +48,16 @@ public interface AiAgentWithMemoryService { */ Result<String> chat(@MemoryId Object memoryId, @UserMessage String message, @UserMessage Content content); + /** + * Chat with a user message that holds only a content (e.g., an image or the text of a file), without a text, with + * memory support. + * + * @param memoryId the memory identifier for conversation history + * @param content the content such as ImageContent, TextContent, etc. + * @return the AI response with token usage metadata + */ + Result<String> chat(@MemoryId Object memoryId, @UserMessage Content content); + /** * Simple chat with a user message, system message and memory window * @@ -75,4 +85,15 @@ public interface AiAgentWithMemoryService { @MemoryId Object memoryId, @UserMessage String message, @UserMessage Content content, @V("prompt") String prompt); + /** + * Chat with a user message that holds only a content, without a text, with system message and memory support. + * + * @param memoryId the memory identifier for conversation history + * @param content the content such as ImageContent, TextContent, etc. + * @param prompt the system message template + * @return the AI response with token usage metadata + */ + @SystemMessage("{{prompt}}") + Result<String> chat(@MemoryId Object memoryId, @UserMessage Content content, @V("prompt") String prompt); + } diff --git a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithoutMemoryService.java b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithoutMemoryService.java index 57519a53e81a..5e49d897c8ab 100644 --- a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithoutMemoryService.java +++ b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithoutMemoryService.java @@ -44,6 +44,14 @@ public interface AiAgentWithoutMemoryService { */ Result<String> chat(@UserMessage String message, @UserMessage Content content); + /** + * Chat with a user message that holds only a content (e.g., an image or the text of a file), without a text. + * + * @param content the content such as ImageContent, TextContent, etc. + * @return the AI response with token usage metadata + */ + Result<String> chat(@UserMessage Content content); + /** * Simple chat with a single user message and system message * @@ -65,4 +73,14 @@ public interface AiAgentWithoutMemoryService { @SystemMessage("{{prompt}}") Result<String> chat(@UserMessage String message, @UserMessage Content content, @V("prompt") String prompt); + /** + * Chat with a user message that holds only a content, without a text, with system message. + * + * @param content the content such as ImageContent, TextContent, etc. + * @param prompt the system message template + * @return the AI response with token usage metadata + */ + @SystemMessage("{{prompt}}") + Result<String> chat(@UserMessage Content content, @V("prompt") String prompt); + } diff --git a/components/camel-ai/camel-langchain4j-agent/src/generated/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverterLoader.java b/components/camel-ai/camel-langchain4j-agent/src/generated/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverterLoader.java index 240733da94ce..a9fb40da5667 100644 --- a/components/camel-ai/camel-langchain4j-agent/src/generated/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverterLoader.java +++ b/components/camel-ai/camel-langchain4j-agent/src/generated/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverterLoader.java @@ -52,6 +52,14 @@ public final class LangChain4jAgentConverterLoader implements TypeConverterLoade } return answer; }); + addTypeConverter(registry, org.apache.camel.component.langchain4j.agent.api.AiAgentBody.class, java.io.File.class, false, + (type, exchange, value) -> { + Object answer = org.apache.camel.component.langchain4j.agent.LangChain4jAgentConverter.fileToAiAgentBody((java.io.File) value, exchange); + if (false && answer == null) { + answer = Void.class; + } + return answer; + }); addTypeConverter(registry, org.apache.camel.component.langchain4j.agent.api.AiAgentBody.class, java.io.InputStream.class, false, (type, exchange, value) -> { Object answer = org.apache.camel.component.langchain4j.agent.LangChain4jAgentConverter.inputStreamToAiAgentBody((java.io.InputStream) value, exchange); @@ -68,6 +76,14 @@ public final class LangChain4jAgentConverterLoader implements TypeConverterLoade } return answer; }); + addTypeConverter(registry, org.apache.camel.component.langchain4j.agent.api.AiAgentBody.class, java.nio.file.Path.class, false, + (type, exchange, value) -> { + Object answer = org.apache.camel.component.langchain4j.agent.LangChain4jAgentConverter.pathToAiAgentBody((java.nio.file.Path) value, exchange); + if (false && answer == null) { + answer = Void.class; + } + return answer; + }); addTypeConverter(registry, org.apache.camel.component.langchain4j.agent.api.AiAgentBody.class, org.apache.camel.WrappedFile.class, false, (type, exchange, value) -> { Object answer = org.apache.camel.component.langchain4j.agent.LangChain4jAgentConverter.toAiAgentBody((org.apache.camel.WrappedFile) value, exchange); diff --git a/components/camel-ai/camel-langchain4j-agent/src/generated/resources/META-INF/org/apache/camel/component/langchain4j/agent/langchain4j-agent.json b/components/camel-ai/camel-langchain4j-agent/src/generated/resources/META-INF/org/apache/camel/component/langchain4j/agent/langchain4j-agent.json index 725bac3f9159..d95ae61ea863 100644 --- a/components/camel-ai/camel-langchain4j-agent/src/generated/resources/META-INF/org/apache/camel/component/langchain4j/agent/langchain4j-agent.json +++ b/components/camel-ai/camel-langchain4j-agent/src/generated/resources/META-INF/org/apache/camel/component/langchain4j/agent/langchain4j-agent.json @@ -69,7 +69,7 @@ "CamelLangChain4jAgentModerationFlagged": { "index": 14, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "Boolean", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Set to true when user input is rejected by the configured moderation model; not set on success.", "constantName": "org.apache.camel.component.langchain4j.agent.api.Headers#MODERATION_FLAGGED" } }, "bodyTypes": { - "producerIn": { "carriers": [ "text", "pojo", "bytes", "stream", "file" ], "description": "The user message as text, or an AiAgentBody with the user message, the system message, the memory id and a content. Bytes, a stream or a file of the file consumers is sent as an image, audio, video, PDF or text content after the text of the CamelLangChain4jAgentUserMessage header; bytes and a stream need the CamelLangChain4jAgentMediaType or Content-Type header. A Map, a java.io.File or another [...] + "producerIn": { "carriers": [ "text", "pojo", "bytes", "stream", "file" ], "description": "The user message as text, or an AiAgentBody with the user message, the system message, the memory id and a content. Bytes, a stream, a file of the file consumers, a java.io.File or a java.nio.file.Path is sent as an image, audio, video, PDF or text content (JSON, XML and YAML as text), after the text of the CamelLangChain4jAgentUserMessage header or alone without it; bytes and a stream need the [...] "producerOut": { "carriers": [ "text" ], "description": "The answer of the agent as text, also with outputClass or jsonSchema: unmarshal the JSON to read its fields. The token counts, the finish reason, the sources and the tool executions are in headers" } }, "properties": { diff --git a/components/camel-ai/camel-langchain4j-agent/src/main/docs/langchain4j-agent-multimodal.adoc b/components/camel-ai/camel-langchain4j-agent/src/main/docs/langchain4j-agent-multimodal.adoc index efe1792b0744..3c4473b38898 100644 --- a/components/camel-ai/camel-langchain4j-agent/src/main/docs/langchain4j-agent-multimodal.adoc +++ b/components/camel-ai/camel-langchain4j-agent/src/main/docs/langchain4j-agent-multimodal.adoc @@ -52,8 +52,23 @@ The agent component automatically converts files from various Camel components t |`InputStream` |Various streaming components |From content type headers (required) + +|`java.io.File`, `java.nio.file.Path` +|A local file set as the message body +|From file extension or headers |=== +The MIME type selects the content sent to the model: + +* `image/*`, `audio/*` and `video/*` are sent as an image, audio or video content. +* `application/pdf` is sent as a PDF content. +* `text/*`, `application/json`, `application/xml`, `application/yaml` and types with a `+json` or `+xml` suffix are sent as a text content (read as UTF-8). + +Any other MIME type fails. + +The text of the `CamelLangChain4jAgentUserMessage` header is sent before the content in the same user message. +Without that header (or when it is blank), the user message holds only the content. + ==== Example: Processing Images from File Component [tabs] diff --git a/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverter.java b/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverter.java index 3b890a1ade04..57b8f1fc5b19 100644 --- a/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverter.java +++ b/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverter.java @@ -22,7 +22,9 @@ import java.io.InputStream; import java.net.URLConnection; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.util.Base64; +import java.util.Locale; import dev.langchain4j.data.audio.Audio; import dev.langchain4j.data.image.Image; @@ -59,6 +61,7 @@ import static org.apache.camel.component.langchain4j.agent.api.Headers.USER_MESS * </p> * <ul> * <li>{@link WrappedFile} - from file, ftp, sftp components</li> + * <li>{@link File} and {@link Path} - a local file</li> * <li>{@code byte[]} - from aws2-s3, azure-storage-blob, and other cloud components</li> * <li>{@link InputStream} - from various streaming components</li> * </ul> @@ -105,6 +108,50 @@ public final class LangChain4jAgentConverter { return buildAiAgentBody(exchange, content, ""); } + /** + * Converts a {@link File} to an {@link AiAgentBody} with the appropriate {@link Content} type. + * <p> + * Works like the {@link WrappedFile} converter: the MIME type comes from the {@code CamelLangChain4jAgentMediaType} + * or {@code CamelFileContentType} header, else from the file extension. + * </p> + * + * @param file the file to send + * @param exchange the Camel exchange containing headers + * @return an AiAgentBody with the appropriate Content type + * @throws IllegalArgumentException if the file cannot be read or the MIME type is unsupported + */ + @Converter + public static AiAgentBody<?> fileToAiAgentBody(File file, Exchange exchange) { + String mimeType = detectMimeType(file.getName(), exchange); + Content content = createContent(readFileBytes(file), mimeType); + return buildAiAgentBody(exchange, content, ""); + } + + /** + * Converts a {@link Path} to an {@link AiAgentBody} with the appropriate {@link Content} type. + * <p> + * Works like the {@link File} converter. + * </p> + * + * @param path the path of the file to send + * @param exchange the Camel exchange containing headers + * @return an AiAgentBody with the appropriate Content type + * @throws IllegalArgumentException if the file cannot be read or the MIME type is unsupported + */ + @Converter + public static AiAgentBody<?> pathToAiAgentBody(Path path, Exchange exchange) { + Path fileName = path.getFileName(); + String mimeType = detectMimeType(fileName != null ? fileName.toString() : null, exchange); + byte[] data; + try { + data = Files.readAllBytes(path); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to read file: " + path.toAbsolutePath(), e); + } + Content content = createContent(data, mimeType); + return buildAiAgentBody(exchange, content, ""); + } + private static String resolveFileName(WrappedFile<?> wrappedFile, Exchange exchange) { String headerName = exchange.getMessage().getHeader(Exchange.FILE_NAME, String.class); if (headerName != null) { @@ -221,42 +268,58 @@ public final class LangChain4jAgentConverter { /** * Creates the appropriate LangChain4j Content object based on the MIME type. + * <p> + * Textual types ({@code text/*}, JSON, XML and YAML, also with a {@code +json} or {@code +xml} suffix) become a + * {@link TextContent}. + * </p> */ static Content createContent(byte[] data, String mimeType) { String base64Data = Base64.getEncoder().encodeToString(data); + String type = mimeType.toLowerCase(Locale.ROOT); - if (mimeType.startsWith("image/")) { + if (type.startsWith("image/")) { Image image = Image.builder() .base64Data(base64Data) .mimeType(mimeType) .build(); return ImageContent.from(image); - } else if (mimeType.startsWith("audio/")) { + } else if (type.startsWith("audio/")) { Audio audio = Audio.builder() .base64Data(base64Data) .mimeType(mimeType) .build(); return AudioContent.from(audio); - } else if (mimeType.startsWith("video/")) { + } else if (type.startsWith("video/")) { Video video = Video.builder() .base64Data(base64Data) .mimeType(mimeType) .build(); return VideoContent.from(video); - } else if ("application/pdf".equals(mimeType)) { + } else if ("application/pdf".equals(type)) { PdfFile pdfFile = PdfFile.builder() .base64Data(base64Data) .build(); return PdfFileContent.from(pdfFile); - } else if (mimeType.startsWith("text/")) { + } else if (isTextual(type)) { return TextContent.from(new String(data, StandardCharsets.UTF_8)); } else { throw new IllegalArgumentException( "Unsupported MIME type: " + mimeType - + ". Supported types: image/*, audio/*, video/*, application/pdf, text/*"); + + ". Supported types: image/*, audio/*, video/*, application/pdf, text/*," + + " application/json, application/xml, application/yaml, */*+json, */*+xml"); } } + private static boolean isTextual(String type) { + return type.startsWith("text/") + || "application/json".equals(type) + || "application/xml".equals(type) + || "application/yaml".equals(type) + || "application/x-yaml".equals(type) + || type.endsWith("+json") + || type.endsWith("+xml"); + } + /** * Detects the MIME type from headers or file extension. * <p> @@ -361,7 +424,7 @@ public final class LangChain4jAgentConverter { */ private static String normalizeContentType(String contentType) { int semicolon = contentType.indexOf(';'); - return semicolon > 0 ? contentType.substring(0, semicolon).trim() : contentType; + return semicolon > 0 ? contentType.substring(0, semicolon).trim() : contentType.trim(); } /** diff --git a/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentEndpoint.java b/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentEndpoint.java index ca22f0e9c6c9..f065ca8de17a 100644 --- a/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentEndpoint.java +++ b/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentEndpoint.java @@ -40,11 +40,12 @@ import static org.apache.camel.component.langchain4j.agent.LangChain4jAgent.SCHE category = { Category.AI }, headersClass = Headers.class) @BodyTypes(producerIn = @BodyType(value = { Carrier.TEXT, Carrier.POJO, Carrier.BYTES, Carrier.STREAM, Carrier.FILE }, description = "The user message as text, or an AiAgentBody with the user message, the system" - + " message, the memory id and a content. Bytes, a stream or a file of the file" - + " consumers is sent as an image, audio, video, PDF or text content after the" - + " text of the CamelLangChain4jAgentUserMessage header; bytes and a stream need" - + " the CamelLangChain4jAgentMediaType or Content-Type header. A Map, a" - + " java.io.File or another object fails, and no body fails"), + + " message, the memory id and a content. Bytes, a stream, a file of the file" + + " consumers, a java.io.File or a java.nio.file.Path is sent as an image," + + " audio, video, PDF or text content (JSON, XML and YAML as text), after the" + + " text of the CamelLangChain4jAgentUserMessage header or alone without it;" + + " bytes and a stream need the CamelLangChain4jAgentMediaType or Content-Type" + + " header. A Map or another object fails, and no body fails"), producerOut = @BodyType(value = Carrier.TEXT, description = "The answer of the agent as text, also with outputClass or jsonSchema:" + " unmarshal the JSON to read its fields. The token counts, the finish" diff --git a/components/camel-ai/camel-langchain4j-agent/src/test/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentContentBodyTest.java b/components/camel-ai/camel-langchain4j-agent/src/test/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentContentBodyTest.java new file mode 100644 index 000000000000..797f290d99de --- /dev/null +++ b/components/camel-ai/camel-langchain4j-agent/src/test/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentContentBodyTest.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.langchain4j.agent; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.Content; +import dev.langchain4j.data.message.ImageContent; +import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.data.message.TextContent; +import dev.langchain4j.data.message.UserMessage; +import dev.langchain4j.memory.chat.MessageWindowChatMemory; +import dev.langchain4j.model.chat.ChatModel; +import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.response.ChatResponse; +import org.apache.camel.Exchange; +import org.apache.camel.RoutesBuilder; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.file.GenericFile; +import org.apache.camel.component.langchain4j.agent.api.AgentConfiguration; +import org.apache.camel.component.langchain4j.agent.api.AgentWithMemory; +import org.apache.camel.component.langchain4j.agent.api.AgentWithoutMemory; +import org.apache.camel.component.langchain4j.agent.api.AiAgentBody; +import org.apache.camel.component.langchain4j.agent.api.Headers; +import org.apache.camel.spi.Registry; +import org.apache.camel.test.junit6.CamelTestSupport; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Checks the user message the chat model receives for a file, bytes, stream or {@link AiAgentBody} body with a content. + */ +class LangChain4jAgentContentBodyTest extends CamelTestSupport { + + @TempDir + Path tempDir; + + private final CapturingChatModel chatModel = new CapturingChatModel(); + + @Override + protected void bindToRegistry(Registry registry) { + registry.bind("agent", new AgentWithoutMemory(new AgentConfiguration().withChatModel(chatModel))); + registry.bind("memoryAgent", new AgentWithMemory( + new AgentConfiguration() + .withChatModel(chatModel) + .withChatMemoryProvider(memoryId -> MessageWindowChatMemory.withMaxMessages(10)))); + } + + @Override + protected RoutesBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + from("direct:agent").to("langchain4j-agent:test?agent=#agent"); + from("direct:memory").to("langchain4j-agent:memory?agent=#memoryAgent"); + } + }; + } + + @BeforeEach + void reset() { + chatModel.requests.clear(); + } + + @Test + void textFileWithoutUserMessageSendsTheContentOnce() throws Exception { + File file = Files.writeString(tempDir.resolve("notes.txt"), "Hello file").toFile(); + GenericFile<File> genericFile = new GenericFile<>(); + genericFile.setFile(file); + genericFile.setFileName(file.getName()); + + String reply = template.requestBody("direct:agent", genericFile, String.class); + + assertThat(reply).isEqualTo("ok"); + assertThat(lastUserMessageContents()).singleElement() + .isInstanceOfSatisfying(TextContent.class, c -> assertThat(c.text()).isEqualTo("Hello file")); + } + + @Test + void textBytesWithoutUserMessageSendsTheContentOnce() { + template.requestBodyAndHeader("direct:agent", "Hello bytes".getBytes(StandardCharsets.UTF_8), + Headers.MEDIA_TYPE, "text/plain", String.class); + + assertThat(lastUserMessageContents()).singleElement() + .isInstanceOfSatisfying(TextContent.class, c -> assertThat(c.text()).isEqualTo("Hello bytes")); + } + + @Test + void textStreamWithoutUserMessageSendsTheContentOnceWithMemory() { + template.requestBodyAndHeaders("direct:memory", + new ByteArrayInputStream("Hello stream".getBytes(StandardCharsets.UTF_8)), + Map.of(Headers.MEDIA_TYPE, "text/plain", Headers.MEMORY_ID, "m1"), String.class); + + assertThat(lastUserMessageContents()).singleElement() + .isInstanceOfSatisfying(TextContent.class, c -> assertThat(c.text()).isEqualTo("Hello stream")); + } + + @Test + void imageBytesWithoutUserMessageSendsOnlyTheImage() { + template.requestBodyAndHeader("direct:agent", new byte[] { 1, 2, 3 }, Headers.MEDIA_TYPE, "image/png", + String.class); + + assertThat(lastUserMessageContents()).singleElement().isInstanceOf(ImageContent.class); + } + + @Test + void textBytesWithSystemMessageAndWithoutUserMessageSendsTheContentOnce() { + template.requestBodyAndHeaders("direct:agent", "Hello bytes".getBytes(StandardCharsets.UTF_8), + Map.of(Headers.MEDIA_TYPE, "text/plain", Headers.SYSTEM_MESSAGE, "Be brief"), String.class); + + List<ChatMessage> messages = chatModel.requests.get(chatModel.requests.size() - 1).messages(); + assertThat(messages.get(0)).isInstanceOfSatisfying(SystemMessage.class, + m -> assertThat(m.text()).isEqualTo("Be brief")); + assertThat(lastUserMessageContents()).singleElement() + .isInstanceOfSatisfying(TextContent.class, c -> assertThat(c.text()).isEqualTo("Hello bytes")); + } + + @Test + void textFileWithUserMessageSendsTextAndContent() throws Exception { + File file = Files.writeString(tempDir.resolve("notes.txt"), "Hello file").toFile(); + GenericFile<File> genericFile = new GenericFile<>(); + genericFile.setFile(file); + genericFile.setFileName(file.getName()); + + template.requestBodyAndHeader("direct:agent", genericFile, Headers.USER_MESSAGE, "Summarize", String.class); + + assertThat(lastUserMessageContents()).hasSize(2); + assertThat(((TextContent) lastUserMessageContents().get(0)).text()).isEqualTo("Summarize"); + assertThat(((TextContent) lastUserMessageContents().get(1)).text()).isEqualTo("Hello file"); + } + + @Test + void blankUserMessageWithContentSendsOnlyTheContent() { + AiAgentBody<TextContent> body = new AiAgentBody<TextContent>(" ") + .withContent(TextContent.from("Hello content")); + + String reply = template.requestBody("direct:agent", body, String.class); + + assertThat(reply).isEqualTo("ok"); + assertThat(lastUserMessageContents()).singleElement() + .isInstanceOfSatisfying(TextContent.class, c -> assertThat(c.text()).isEqualTo("Hello content")); + } + + @Test + void blankUserMessageWithContentSendsOnlyTheContentWithMemory() { + AiAgentBody<TextContent> body = new AiAgentBody<TextContent>(" ") + .withContent(TextContent.from("Hello content")) + .withMemoryId("m2"); + + String reply = template.requestBody("direct:memory", body, String.class); + + assertThat(reply).isEqualTo("ok"); + assertThat(lastUserMessageContents()).singleElement() + .isInstanceOfSatisfying(TextContent.class, c -> assertThat(c.text()).isEqualTo("Hello content")); + } + + @Test + void jsonFileIsSentAsText() throws Exception { + File file = Files.writeString(tempDir.resolve("order.json"), "{\"id\":1}").toFile(); + GenericFile<File> genericFile = new GenericFile<>(); + genericFile.setFile(file); + genericFile.setFileName(file.getName()); + + template.requestBodyAndHeader("direct:agent", genericFile, Headers.USER_MESSAGE, "Check", String.class); + + assertThat(lastUserMessageContents()).hasSize(2); + assertThat(((TextContent) lastUserMessageContents().get(1)).text()).isEqualTo("{\"id\":1}"); + } + + @Test + void jsonAndXmlBytesAreSentAsText() { + template.requestBodyAndHeader("direct:agent", "{\"id\":2}".getBytes(StandardCharsets.UTF_8), + Exchange.CONTENT_TYPE, "application/json; charset=UTF-8", String.class); + assertThat(lastUserMessageContents()).singleElement() + .isInstanceOfSatisfying(TextContent.class, c -> assertThat(c.text()).isEqualTo("{\"id\":2}")); + + template.requestBodyAndHeader("direct:agent", "<id>3</id>".getBytes(StandardCharsets.UTF_8), + Exchange.CONTENT_TYPE, "Application/XML", String.class); + assertThat(lastUserMessageContents()).singleElement() + .isInstanceOfSatisfying(TextContent.class, c -> assertThat(c.text()).isEqualTo("<id>3</id>")); + + template.requestBodyAndHeader("direct:agent", "{\"id\":4}".getBytes(StandardCharsets.UTF_8), + Exchange.CONTENT_TYPE, "application/vnd.api+json", String.class); + assertThat(lastUserMessageContents()).singleElement() + .isInstanceOfSatisfying(TextContent.class, c -> assertThat(c.text()).isEqualTo("{\"id\":4}")); + } + + @Test + void fileBodyIsConverted() throws Exception { + File file = Files.writeString(tempDir.resolve("plain.txt"), "Hello java.io.File").toFile(); + + template.requestBodyAndHeader("direct:agent", file, Headers.USER_MESSAGE, "Read", String.class); + + assertThat(lastUserMessageContents()).hasSize(2); + assertThat(((TextContent) lastUserMessageContents().get(1)).text()).isEqualTo("Hello java.io.File"); + } + + @Test + void pathBodyIsConverted() throws Exception { + Path path = Files.writeString(tempDir.resolve("data.json"), "{\"path\":true}"); + + template.requestBody("direct:agent", path, String.class); + + assertThat(lastUserMessageContents()).singleElement() + .isInstanceOfSatisfying(TextContent.class, c -> assertThat(c.text()).isEqualTo("{\"path\":true}")); + } + + private List<Content> lastUserMessageContents() { + assertThat(chatModel.requests).isNotEmpty(); + List<ChatMessage> messages = chatModel.requests.get(chatModel.requests.size() - 1).messages(); + ChatMessage last = messages.get(messages.size() - 1); + assertThat(last).isInstanceOf(UserMessage.class); + return ((UserMessage) last).contents(); + } + + private static final class CapturingChatModel implements ChatModel { + + private final List<ChatRequest> requests = new ArrayList<>(); + + @Override + public ChatResponse doChat(ChatRequest request) { + requests.add(request); + return ChatResponse.builder().aiMessage(AiMessage.from("ok")).build(); + } + } +}
