This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch CAMEL-26207-spring-ai-chat-bodies in repository https://gitbox.apache.org/repos/asf/camel.git
commit 01fed545017317a81940227b4750cf329c0d499b Author: Claus Ibsen <[email protected]> AuthorDate: Tue Nov 3 00:29:35 2026 +0100 CAMEL-26207: camel-spring-ai-chat - keep the question with outputFormat, send File, Path and stream bodies as media, and fail media without a user message clearly outputFormat BEAN, MAP or LIST replaced the user text with the format instructions, so the model never got the question. The instructions are now passed as the OUTPUT_FORMAT advisor context, and Spring AI appends them to the last user message, as entity(...) does. A java.io.File, Path or InputStream body of an image, audio, video or document was sent as its text. It is now sent as media; a text one is still sent as text. A media body without userMessage or the CamelSpringAiChatUserMessage header now fails early with a message naming them instead of "text cannot be null or empty" from Spring AI. CHAT_MULTIPLE_MESSAGES takes String elements as UserMessage, and fails other elements with a clear message instead of a ClassCastException. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../camel/catalog/components/spring-ai-chat.json | 2 +- .../catalog/docs/spring-ai-chat-component.adoc | 12 + .../component/springai/chat/spring-ai-chat.json | 2 +- .../src/main/docs/spring-ai-chat-component.adoc | 12 + .../springai/chat/SpringAiChatEndpoint.java | 16 +- .../springai/chat/SpringAiChatProducer.java | 163 ++++++++-- .../chat/SpringAiChatBodyHandlingTest.java | 330 +++++++++++++++++++++ .../ROOT/pages/camel-4x-upgrade-guide-4_24.adoc | 20 ++ 8 files changed, 519 insertions(+), 38 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/spring-ai-chat.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/spring-ai-chat.json index 604b4df4b1c9..dd6a99e1a592 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/spring-ai-chat.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/spring-ai-chat.json @@ -63,7 +63,7 @@ "CamelSpringAiChatToolContext": { "index": 31, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "java.util.Map<String, Object>", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Context map to pass to tools during execution", "constantName": "org.apache.camel.component.springai.chat.SpringAiChatConstants#TOOL_CONTEXT" } }, "bodyTypes": { - "producerIn": { "carriers": [ "any" ], "byOption": [ { "option": "chatOperation", "value": "CHAT_SINGLE_MESSAGE_WITH_PROMPT", "carriers": [ "map" ] }, { "option": "chatOperation", "value": "CHAT_MULTIPLE_MESSAGES", "carriers": [ "list" ] } ], "description": "The prompt as text, or a UserMessage or another Spring AI Message. A byte[] is sent as media (image\/png unless the CamelSpringAiChatMediaType header says otherwise), and a WrappedFile of a local file, or a List of them, as media [...] + "producerIn": { "carriers": [ "any" ], "byOption": [ { "option": "chatOperation", "value": "CHAT_SINGLE_MESSAGE_WITH_PROMPT", "carriers": [ "map" ] }, { "option": "chatOperation", "value": "CHAT_MULTIPLE_MESSAGES", "carriers": [ "list" ] } ], "description": "The prompt as text, or a UserMessage or another Spring AI Message. A byte[] is sent as media (image\/png unless the CamelSpringAiChatMediaType header says otherwise), and a WrappedFile of a local file, or a List of them, as media [...] "producerOut": { "carriers": [ "text" ], "dependsOn": [ "structuredOutputConverter" ], "byOption": [ { "option": "outputFormat", "value": "BEAN", "carriers": [ "pojo" ], "javaTypeOption": "outputClass" }, { "option": "outputFormat", "value": "MAP", "carriers": [ "map" ] }, { "option": "outputFormat", "value": "LIST", "carriers": [ "list" ] } ], "javaTypeOption": "entityClass", "description": "The answer of the model as text, also in the CamelSpringAiChatResponse header. entityClass m [...] }, "properties": { diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/spring-ai-chat-component.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/spring-ai-chat-component.adoc index 7f2309bcb07f..a00434c853df 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/spring-ai-chat-component.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/spring-ai-chat-component.adoc @@ -155,6 +155,9 @@ conversation.add(new UserMessage("Can I make it vegetarian?")); String response = template.requestBody("direct:chat", conversation, String.class); ---- +The list can also hold plain `String` elements: each one is sent as a `UserMessage`. Any other element fails the +exchange with an `IllegalArgumentException`. + === Retrieval Augmented Generation (RAG) RAG allows you to augment prompts with relevant context from your data sources, enabling the LLM to answer questions based on your specific information. @@ -704,10 +707,16 @@ template.requestBody("direct:scan-analyzer", message, String.class); * `byte[]` - Single media file (image, audio, etc.) * `WrappedFile` - Single file from file components * `List<WrappedFile>` - Multiple files for comparison or analysis +* `java.io.File` or `java.nio.file.Path` - Single media file, when its media type (from the headers or the file extension) is an image, audio, video or document (PDF, Word, Excel); a text file, or one with an unknown extension, is sent as its text +* `InputStream` - Single media file, when the `CamelSpringAiChatMediaType`, `CamelFileContentType` or `Content-Type` header, or the extension in the `CamelFileName` header, gives an image, audio, video or document media type; otherwise the stream is read as text. A media stream is read up to `maxFileSize` * `UserMessage` - Spring AI UserMessage with text and/or media * `Message` - Any Spring AI Message type * `List<Message>` - Full conversation history (CHAT_MULTIPLE_MESSAGES operation) +Spring AI needs a text with the media, so a media body (`byte[]`, `java.io.File`, `Path`, `InputStream`, `WrappedFile` +or `List<WrappedFile>`) needs the `userMessage` option or the `CamelSpringAiChatUserMessage` header; without it the +exchange fails with an `IllegalArgumentException` before the media is read. + **MIME Type Detection Priority:** 1. `CamelSpringAiChatMediaType` header (highest priority) @@ -879,6 +888,9 @@ The component supports structured output conversion, allowing you to convert LLM Three output formats are supported: **BEAN** (Java records/POJOs), **MAP** (key-value structures), and **LIST** (arrays). Each can be configured via converter bean, endpoint parameters, or headers. +The format instructions of the converter are appended to the user message, or to the last user message of a +conversation, so the model gets both the question and the format to answer in. + ._Java-only: uses Java record definition and ProducerTemplate_ [source,java] ---- diff --git a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/generated/resources/META-INF/org/apache/camel/component/springai/chat/spring-ai-chat.json b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/generated/resources/META-INF/org/apache/camel/component/springai/chat/spring-ai-chat.json index 604b4df4b1c9..dd6a99e1a592 100644 --- a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/generated/resources/META-INF/org/apache/camel/component/springai/chat/spring-ai-chat.json +++ b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/generated/resources/META-INF/org/apache/camel/component/springai/chat/spring-ai-chat.json @@ -63,7 +63,7 @@ "CamelSpringAiChatToolContext": { "index": 31, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "java.util.Map<String, Object>", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Context map to pass to tools during execution", "constantName": "org.apache.camel.component.springai.chat.SpringAiChatConstants#TOOL_CONTEXT" } }, "bodyTypes": { - "producerIn": { "carriers": [ "any" ], "byOption": [ { "option": "chatOperation", "value": "CHAT_SINGLE_MESSAGE_WITH_PROMPT", "carriers": [ "map" ] }, { "option": "chatOperation", "value": "CHAT_MULTIPLE_MESSAGES", "carriers": [ "list" ] } ], "description": "The prompt as text, or a UserMessage or another Spring AI Message. A byte[] is sent as media (image\/png unless the CamelSpringAiChatMediaType header says otherwise), and a WrappedFile of a local file, or a List of them, as media [...] + "producerIn": { "carriers": [ "any" ], "byOption": [ { "option": "chatOperation", "value": "CHAT_SINGLE_MESSAGE_WITH_PROMPT", "carriers": [ "map" ] }, { "option": "chatOperation", "value": "CHAT_MULTIPLE_MESSAGES", "carriers": [ "list" ] } ], "description": "The prompt as text, or a UserMessage or another Spring AI Message. A byte[] is sent as media (image\/png unless the CamelSpringAiChatMediaType header says otherwise), and a WrappedFile of a local file, or a List of them, as media [...] "producerOut": { "carriers": [ "text" ], "dependsOn": [ "structuredOutputConverter" ], "byOption": [ { "option": "outputFormat", "value": "BEAN", "carriers": [ "pojo" ], "javaTypeOption": "outputClass" }, { "option": "outputFormat", "value": "MAP", "carriers": [ "map" ] }, { "option": "outputFormat", "value": "LIST", "carriers": [ "list" ] } ], "javaTypeOption": "entityClass", "description": "The answer of the model as text, also in the CamelSpringAiChatResponse header. entityClass m [...] }, "properties": { diff --git a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/main/docs/spring-ai-chat-component.adoc b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/main/docs/spring-ai-chat-component.adoc index 7f2309bcb07f..a00434c853df 100644 --- a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/main/docs/spring-ai-chat-component.adoc +++ b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/main/docs/spring-ai-chat-component.adoc @@ -155,6 +155,9 @@ conversation.add(new UserMessage("Can I make it vegetarian?")); String response = template.requestBody("direct:chat", conversation, String.class); ---- +The list can also hold plain `String` elements: each one is sent as a `UserMessage`. Any other element fails the +exchange with an `IllegalArgumentException`. + === Retrieval Augmented Generation (RAG) RAG allows you to augment prompts with relevant context from your data sources, enabling the LLM to answer questions based on your specific information. @@ -704,10 +707,16 @@ template.requestBody("direct:scan-analyzer", message, String.class); * `byte[]` - Single media file (image, audio, etc.) * `WrappedFile` - Single file from file components * `List<WrappedFile>` - Multiple files for comparison or analysis +* `java.io.File` or `java.nio.file.Path` - Single media file, when its media type (from the headers or the file extension) is an image, audio, video or document (PDF, Word, Excel); a text file, or one with an unknown extension, is sent as its text +* `InputStream` - Single media file, when the `CamelSpringAiChatMediaType`, `CamelFileContentType` or `Content-Type` header, or the extension in the `CamelFileName` header, gives an image, audio, video or document media type; otherwise the stream is read as text. A media stream is read up to `maxFileSize` * `UserMessage` - Spring AI UserMessage with text and/or media * `Message` - Any Spring AI Message type * `List<Message>` - Full conversation history (CHAT_MULTIPLE_MESSAGES operation) +Spring AI needs a text with the media, so a media body (`byte[]`, `java.io.File`, `Path`, `InputStream`, `WrappedFile` +or `List<WrappedFile>`) needs the `userMessage` option or the `CamelSpringAiChatUserMessage` header; without it the +exchange fails with an `IllegalArgumentException` before the media is read. + **MIME Type Detection Priority:** 1. `CamelSpringAiChatMediaType` header (highest priority) @@ -879,6 +888,9 @@ The component supports structured output conversion, allowing you to convert LLM Three output formats are supported: **BEAN** (Java records/POJOs), **MAP** (key-value structures), and **LIST** (arrays). Each can be configured via converter bean, endpoint parameters, or headers. +The format instructions of the converter are appended to the user message, or to the last user message of a +conversation, so the model gets both the question and the format to answer in. + ._Java-only: uses Java record definition and ProducerTemplate_ [source,java] ---- diff --git a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/main/java/org/apache/camel/component/springai/chat/SpringAiChatEndpoint.java b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/main/java/org/apache/camel/component/springai/chat/SpringAiChatEndpoint.java index 511c2e2b57e7..5bb1c4658ac2 100644 --- a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/main/java/org/apache/camel/component/springai/chat/SpringAiChatEndpoint.java +++ b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/main/java/org/apache/camel/component/springai/chat/SpringAiChatEndpoint.java @@ -45,12 +45,16 @@ import org.apache.camel.support.DefaultEndpoint; description = "The prompt as text, or a UserMessage or another Spring AI Message. A" + " byte[] is sent as media (image/png unless the" + " CamelSpringAiChatMediaType header says otherwise), and a WrappedFile" - + " of a local file, or a List of them, as media files: these need" - + " userMessage or the CamelSpringAiChatUserMessage header for the text." - + " A stream, a java.io.File or another body is sent as its text, and no" - + " body fails. CHAT_SINGLE_MESSAGE_WITH_PROMPT takes the variables of" - + " the template in the CamelSpringAiChatPromptTemplate header as a Map," - + " and CHAT_MULTIPLE_MESSAGES a List of Message"), + + " of a local file, or a List of them, as media files. A java.io.File, a" + + " Path or a stream is sent as media when its media type (from the" + + " headers or the file name) is an image, audio, video or document," + + " and as its text otherwise." + + " Media needs userMessage or the CamelSpringAiChatUserMessage header" + + " for the text, else it fails. Another body is sent as its text, and" + + " no body fails. CHAT_SINGLE_MESSAGE_WITH_PROMPT takes the variables" + + " of the template in the CamelSpringAiChatPromptTemplate header as a" + + " Map, and CHAT_MULTIPLE_MESSAGES a List of Message or String (a" + + " String is a UserMessage)"), producerOut = @BodyType(value = Carrier.TEXT, dependsOn = "structuredOutputConverter", javaTypeOption = "entityClass", byOption = { diff --git a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/main/java/org/apache/camel/component/springai/chat/SpringAiChatProducer.java b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/main/java/org/apache/camel/component/springai/chat/SpringAiChatProducer.java index 597e59fc3f10..2e67642217f4 100644 --- a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/main/java/org/apache/camel/component/springai/chat/SpringAiChatProducer.java +++ b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/main/java/org/apache/camel/component/springai/chat/SpringAiChatProducer.java @@ -18,13 +18,16 @@ package org.apache.camel.component.springai.chat; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -50,6 +53,7 @@ import org.apache.camel.support.DefaultProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.chat.client.ChatClientAttributes; import org.springframework.ai.chat.client.ResponseEntity; import org.springframework.ai.chat.client.advisor.MessageChatMemoryAdvisor; import org.springframework.ai.chat.client.advisor.SafeGuardAdvisor; @@ -222,6 +226,9 @@ public class SpringAiChatProducer extends DefaultProducer { "Consider using only one RAG method: either automatic (QuestionAnswerAdvisor) or manual (AUGMENTED_DATA header)."); } + // A File, Path or InputStream of an image, audio, video or document is media, otherwise it is read as text + String bodyMediaType = bodyMediaType(exchange, messageBody); + // Handle different body types if (messageBody instanceof String stringBody) { userMessageText = stringBody; @@ -240,18 +247,18 @@ public class SpringAiChatProducer extends DefaultProducer { } else if (messageBody instanceof WrappedFile) { // Handle single WrappedFile for multimodal content WrappedFile<?> wrappedFile = (WrappedFile<?>) messageBody; + String userText = getUserMessageText(exchange); List<Media> mediaList = convertWrappedFilesToMedia(exchange, List.of(wrappedFile)); - UserMessage multimodalMessage = createMultimodalMessageWithMedia(exchange, mediaList); - applyUserMessageWithMedia(request, exchange, multimodalMessage.getText(), multimodalMessage.getMedia()); + applyUserMessageWithMedia(request, exchange, userText, mediaList); } else if (messageBody instanceof List) { // Handle List<WrappedFile> for multimodal content with multiple files List<?> list = (List<?>) messageBody; if (!list.isEmpty() && list.get(0) instanceof WrappedFile) { @SuppressWarnings("unchecked") List<WrappedFile<?>> wrappedFiles = (List<WrappedFile<?>>) list; + String userText = getUserMessageText(exchange); List<Media> mediaList = convertWrappedFilesToMedia(exchange, wrappedFiles); - UserMessage multimodalMessage = createMultimodalMessageWithMedia(exchange, mediaList); - applyUserMessageWithMedia(request, exchange, multimodalMessage.getText(), multimodalMessage.getMedia()); + applyUserMessageWithMedia(request, exchange, userText, mediaList); } else { throw new IllegalArgumentException( "List body must contain WrappedFile objects. Got: " @@ -261,6 +268,16 @@ public class SpringAiChatProducer extends DefaultProducer { // Handle multimodal content (image or audio) UserMessage multimodalMessage = createMultimodalMessage(exchange, bytes); applyUserMessageWithMedia(request, exchange, multimodalMessage.getText(), multimodalMessage.getMedia()); + } else if (bodyMediaType != null) { + // A java.io.File, Path or InputStream of an image, audio, video or document is sent as media + String userText = getUserMessageText(exchange); + byte[] data = messageBody instanceof InputStream stream + ? streamToByteArray(stream, exchange) : fileToByteArray(toFile(messageBody), exchange); + Media media = Media.builder() + .mimeType(MimeType.valueOf(bodyMediaType)) + .data(data) + .build(); + applyUserMessageWithMedia(request, exchange, userText, List.of(media)); } else { String text = exchange.getIn().getBody(String.class); if (text != null) { @@ -268,7 +285,7 @@ public class SpringAiChatProducer extends DefaultProducer { } else { throw new IllegalArgumentException( "Unsupported message type: " + messageBody.getClass().getName() - + ". Expected String, byte[], WrappedFile, List<WrappedFile>, or org.springframework.ai.chat.messages.Message"); + + ". Expected String, byte[], File, Path, InputStream, WrappedFile, List<WrappedFile>, or org.springframework.ai.chat.messages.Message"); } } @@ -329,8 +346,21 @@ public class SpringAiChatProducer extends DefaultProducer { } private void processMultipleMessages(Exchange exchange) throws InvalidPayloadException { - // Get messages from body - List<Message> messages = exchange.getIn().getMandatoryBody(List.class); + // Get messages from body: a Spring AI Message is sent as is, and a text as a UserMessage + List<?> body = exchange.getIn().getMandatoryBody(List.class); + List<Message> messages = new ArrayList<>(body.size()); + for (Object element : body) { + if (element instanceof Message message) { + messages.add(message); + } else if (element instanceof CharSequence text) { + messages.add(new UserMessage(text.toString())); + } else { + throw new IllegalArgumentException( + "CHAT_MULTIPLE_MESSAGES needs a List of org.springframework.ai.chat.messages.Message or String." + + " Got an element of type: " + + (element == null ? "null" : element.getClass().getName())); + } + } ChatClient.ChatClientRequestSpec request = chatClient.prompt().messages(messages); @@ -364,31 +394,92 @@ public class SpringAiChatProducer extends DefaultProducer { } /** - * Create a multimodal message with multiple media items - */ - private UserMessage createMultimodalMessageWithMedia(Exchange exchange, List<Media> mediaList) { - String userMessageText = getUserMessageText(exchange); - - return UserMessage.builder() - .text(userMessageText) - .media(mediaList) - .build(); - } - - /** - * Get user message text from header or configuration, with empty string as fallback + * Get the text that goes with a media body from the header or the configuration. Spring AI needs a text in the user + * message, so a media body without one fails here, before the media is read. */ private String getUserMessageText(Exchange exchange) { String userMessageText = exchange.getIn().getHeader(SpringAiChatConstants.USER_MESSAGE, String.class); - if (userMessageText == null) { + if (userMessageText == null || userMessageText.isBlank()) { userMessageText = getEndpoint().getConfiguration().getUserMessage(); } - if (userMessageText == null) { - userMessageText = ""; // Empty text for media-only messages + if (userMessageText == null || userMessageText.isBlank()) { + throw new IllegalArgumentException( + "A media body (byte[], File, Path, InputStream or WrappedFile) needs the text of the user message:" + + " set the userMessage option or the " + + SpringAiChatConstants.USER_MESSAGE + " header"); } return userMessageText; } + /** + * The media type of a File, Path or InputStream body when it is an image, audio, video or document, or null when it + * is text, another or an unknown type, or another body. + */ + private String bodyMediaType(Exchange exchange, Object body) { + String mediaType; + if (body instanceof File || body instanceof Path) { + mediaType = exchange.getIn().getHeader(SpringAiChatConstants.MEDIA_TYPE, String.class); + if (mediaType == null) { + mediaType = exchange.getIn().getHeader(Exchange.FILE_CONTENT_TYPE, String.class); + } + if (mediaType == null) { + mediaType = mimeTypeFromFileName( + body instanceof Path path ? String.valueOf(path.getFileName()) : ((File) body).getName()); + } + } else if (body instanceof InputStream) { + mediaType = exchange.getIn().getHeader(SpringAiChatConstants.MEDIA_TYPE, String.class); + if (mediaType == null) { + mediaType = exchange.getIn().getHeader(Exchange.FILE_CONTENT_TYPE, String.class); + } + if (mediaType == null) { + mediaType = exchange.getIn().getHeader(Exchange.CONTENT_TYPE, String.class); + } + if (mediaType == null) { + String fileName = exchange.getIn().getHeader(Exchange.FILE_NAME, String.class); + mediaType = fileName != null ? mimeTypeFromFileName(fileName) : null; + } + } else { + return null; + } + return mediaType != null && isMediaType(mediaType) ? mediaType : null; + } + + /** + * Whether the media type is an image, audio, video or document (PDF, Word or Excel) to send as media + */ + private static boolean isMediaType(String mediaType) { + String type = mediaType.trim().toLowerCase(Locale.ROOT); + return type.startsWith("image/") || type.startsWith("audio/") || type.startsWith("video/") + || type.startsWith("application/pdf") + || type.startsWith("application/msword") + || type.startsWith("application/vnd.ms-excel") + || type.startsWith("application/vnd.openxmlformats-officedocument."); + } + + private static File toFile(Object body) { + return body instanceof Path path ? path.toFile() : (File) body; + } + + /** + * Read a stream body with the same size limit as a file + */ + private byte[] streamToByteArray(InputStream stream, Exchange exchange) { + long maxSize = getMaxFileSize(exchange); + try { + if (maxSize <= 0) { + return stream.readAllBytes(); + } + byte[] data = stream.readNBytes((int) Math.min(maxSize + 1, Integer.MAX_VALUE)); + if (data.length > maxSize) { + throw new IllegalArgumentException( + String.format("Stream size exceeds maximum allowed size (%d bytes)", maxSize)); + } + return data; + } catch (IOException e) { + throw new IllegalArgumentException("Failed to read the stream of the message body", e); + } + } + /** * Convert a list of WrappedFile objects to Media objects */ @@ -445,7 +536,21 @@ public class SpringAiChatProducer extends DefaultProducer { } // Auto-detect from file extension - String fileName = file.getName().toLowerCase(); + String mimeType = mimeTypeFromFileName(file.getName()); + if (mimeType != null) { + return mimeType; + } + + // Default to image/png if extension not recognized + LOG.debug("Could not detect MIME type from file extension: {}, defaulting to image/png", file.getName()); + return "image/png"; + } + + /** + * The MIME type of a file name by its extension, or null when the extension is not known + */ + private static String mimeTypeFromFileName(String name) { + String fileName = name.toLowerCase(Locale.ROOT); // Image formats (matching Spring AI Media.Format constants) if (fileName.endsWith(".png")) { @@ -513,10 +618,7 @@ public class SpringAiChatProducer extends DefaultProducer { } else if (fileName.endsWith(".md")) { return "text/markdown"; } - - // Default to image/png if extension not recognized - LOG.debug("Could not detect MIME type from file extension: {}, defaulting to image/png", fileName); - return "image/png"; + return null; } /** @@ -1015,8 +1117,9 @@ public class SpringAiChatProducer extends DefaultProducer { // Get format instructions from the converter String format = converter.getFormat(); - // Append format instructions to the request - request.user(u -> u.text(format)); + // Let Spring AI append the format instructions to the last user message, as entity(...) does, so the question + // is kept (request.user(...) would replace the text of the user message) + request.advisors(a -> a.param(ChatClientAttributes.OUTPUT_FORMAT.getKey(), format)); ChatResponse response = callWithObservability(request, exchange); populateTokenUsage(response, exchange); diff --git a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/test/java/org/apache/camel/component/springai/chat/SpringAiChatBodyHandlingTest.java b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/test/java/org/apache/camel/component/springai/chat/SpringAiChatBodyHandlingTest.java new file mode 100644 index 000000000000..d7846f475725 --- /dev/null +++ b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-chat/src/test/java/org/apache/camel/component/springai/chat/SpringAiChatBodyHandlingTest.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.springai.chat; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.camel.CamelContext; +import org.apache.camel.CamelExecutionException; +import org.apache.camel.Exchange; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.ai.observability.GenAiObservabilityProperties; +import org.apache.camel.test.junit6.CamelTestSupport; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Checks what the producer sends to the model for the different message bodies, with a fake {@link ChatModel} that + * records the prompt it gets. + */ +class SpringAiChatBodyHandlingTest extends CamelTestSupport { + + private static final byte[] PNG_BYTES = { (byte) 0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 0, 0, 0, 0x0d }; + + private final RecordingChatModel chatModel = new RecordingChatModel(); + + @TempDir + Path tempDir; + + @Override + protected CamelContext createCamelContext() throws Exception { + CamelContext context = super.createCamelContext(); + Properties properties = new Properties(); + properties.setProperty(GenAiObservabilityProperties.ENABLED, "false"); + context.getPropertiesComponent().setOverrideProperties(properties); + context.getRegistry().bind("chatModel", chatModel); + return context; + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + from("direct:chat") + .to("spring-ai-chat:chat?chatModel=#chatModel"); + + from("direct:list") + .to("spring-ai-chat:list?chatModel=#chatModel&outputFormat=LIST"); + + from("direct:map") + .to("spring-ai-chat:map?chatModel=#chatModel&outputFormat=MAP"); + + from("direct:multiple") + .to("spring-ai-chat:multiple?chatModel=#chatModel&chatOperation=CHAT_MULTIPLE_MESSAGES"); + + from("direct:multipleList") + .to("spring-ai-chat:multipleList?chatModel=#chatModel&chatOperation=CHAT_MULTIPLE_MESSAGES" + + "&outputFormat=LIST"); + } + }; + } + + @Test + void outputFormatListKeepsTheQuestion() { + chatModel.answer("red, green, blue"); + + Object body = template.requestBody("direct:list", "Name three colors", Object.class); + + assertThat(body).isEqualTo(List.of("red", "green", "blue")); + UserMessage user = chatModel.lastUserMessage(); + assertThat(user.getText()).startsWith("Name three colors").contains("comma-separated"); + assertThat(chatModel.lastPrompt().getInstructions()).hasSize(1); + } + + @Test + void outputFormatMapKeepsTheQuestion() { + chatModel.answer("{\"capital\": \"Paris\"}"); + + Object body = template.requestBody("direct:map", "What is the capital of France?", Object.class); + + assertThat(body).isEqualTo(Map.of("capital", "Paris")); + UserMessage user = chatModel.lastUserMessage(); + assertThat(user.getText()).startsWith("What is the capital of France?").contains("JSON"); + } + + @Test + void outputFormatListAppendsTheFormatToTheLastUserMessageOfAConversation() { + chatModel.answer("one, two"); + + List<Message> conversation = List.of( + new UserMessage("Hello"), + new AssistantMessage("Hi"), + new UserMessage("Count to two")); + Object body = template.requestBody("direct:multipleList", conversation, Object.class); + + assertThat(body).isEqualTo(List.of("one", "two")); + List<Message> sent = chatModel.lastPrompt().getInstructions(); + assertThat(sent).hasSize(3); + assertThat(sent.get(0).getText()).isEqualTo("Hello"); + assertThat(sent.get(2).getText()).startsWith("Count to two").contains("comma-separated"); + } + + @Test + void mediaBodyWithoutUserMessageFailsWithAClearMessage() { + assertThatThrownBy(() -> template.requestBody("direct:chat", PNG_BYTES, String.class)) + .isInstanceOf(CamelExecutionException.class) + .cause() + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("userMessage") + .hasMessageContaining(SpringAiChatConstants.USER_MESSAGE); + assertThat(chatModel.lastPrompt()).isNull(); + } + + @Test + void mediaBodyWithUserMessageHeader() { + String answer = template.requestBodyAndHeader("direct:chat", PNG_BYTES, + SpringAiChatConstants.USER_MESSAGE, "What is in this image?", String.class); + + assertThat(answer).isEqualTo("ok"); + UserMessage user = chatModel.lastUserMessage(); + assertThat(user.getText()).isEqualTo("What is in this image?"); + assertThat(user.getMedia()).singleElement() + .satisfies(media -> assertThat(media.getMimeType().toString()).isEqualTo("image/png")); + } + + @Test + void fileBodyIsSentAsMedia() throws Exception { + File image = tempDir.resolve("photo.png").toFile(); + Files.write(image.toPath(), PNG_BYTES); + + String answer = template.requestBodyAndHeader("direct:chat", image, + SpringAiChatConstants.USER_MESSAGE, "What is in this image?", String.class); + + assertThat(answer).isEqualTo("ok"); + UserMessage user = chatModel.lastUserMessage(); + assertThat(user.getText()).isEqualTo("What is in this image?"); + assertThat(user.getMedia()).singleElement().satisfies(media -> { + assertThat(media.getMimeType().toString()).isEqualTo("image/png"); + assertThat(media.getDataAsByteArray()).isEqualTo(PNG_BYTES); + }); + } + + @Test + void pathBodyIsSentAsMedia() throws Exception { + Path image = tempDir.resolve("photo.jpg"); + Files.write(image, PNG_BYTES); + + template.requestBodyAndHeader("direct:chat", image, + SpringAiChatConstants.USER_MESSAGE, "What is in this image?", String.class); + + assertThat(chatModel.lastUserMessage().getMedia()).singleElement() + .satisfies(media -> assertThat(media.getMimeType().toString()).isEqualTo("image/jpeg")); + } + + @Test + void textFileBodyIsStillSentAsText() throws Exception { + File prompt = tempDir.resolve("prompt.txt").toFile(); + Files.writeString(prompt.toPath(), "What is Apache Camel?"); + + template.requestBody("direct:chat", prompt, String.class); + + UserMessage user = chatModel.lastUserMessage(); + assertThat(user.getText()).isEqualTo("What is Apache Camel?"); + assertThat(user.getMedia()).isEmpty(); + } + + @Test + void streamBodyWithMediaTypeIsSentAsMedia() { + InputStream stream = new ByteArrayInputStream(PNG_BYTES); + + template.requestBodyAndHeaders("direct:chat", stream, + Map.of(SpringAiChatConstants.USER_MESSAGE, "What is in this image?", + SpringAiChatConstants.MEDIA_TYPE, "image/png"), + String.class); + + UserMessage user = chatModel.lastUserMessage(); + assertThat(user.getText()).isEqualTo("What is in this image?"); + assertThat(user.getMedia()).singleElement().satisfies(media -> { + assertThat(media.getMimeType().toString()).isEqualTo("image/png"); + assertThat(media.getDataAsByteArray()).isEqualTo(PNG_BYTES); + }); + } + + @Test + void streamBodyWithImageFileNameIsSentAsMedia() { + InputStream stream = new ByteArrayInputStream(PNG_BYTES); + + template.requestBodyAndHeaders("direct:chat", stream, + Map.of(SpringAiChatConstants.USER_MESSAGE, "What is in this image?", + Exchange.FILE_NAME, "inbox/photo.png"), + String.class); + + assertThat(chatModel.lastUserMessage().getMedia()).singleElement() + .satisfies(media -> assertThat(media.getMimeType().toString()).isEqualTo("image/png")); + } + + @Test + void streamBodyLargerThanMaxFileSizeFails() { + InputStream stream = new ByteArrayInputStream(PNG_BYTES); + + assertThatThrownBy(() -> template.requestBodyAndHeaders("direct:chat", stream, + Map.of(SpringAiChatConstants.USER_MESSAGE, "What is in this image?", + SpringAiChatConstants.MEDIA_TYPE, "image/png", + SpringAiChatConstants.MAX_FILE_SIZE, 4L), + String.class)) + .isInstanceOf(CamelExecutionException.class) + .cause() + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("exceeds maximum allowed size (4 bytes)"); + assertThat(chatModel.lastPrompt()).isNull(); + } + + @Test + void textStreamBodyIsStillSentAsText() { + InputStream stream = new ByteArrayInputStream("What is Apache Camel?".getBytes(StandardCharsets.UTF_8)); + + template.requestBodyAndHeader("direct:chat", stream, Exchange.CONTENT_TYPE, "text/plain; charset=UTF-8", + String.class); + + UserMessage user = chatModel.lastUserMessage(); + assertThat(user.getText()).isEqualTo("What is Apache Camel?"); + assertThat(user.getMedia()).isEmpty(); + } + + @Test + void streamBodyWithOctetStreamContentTypeIsStillSentAsText() { + InputStream stream = new ByteArrayInputStream("What is Apache Camel?".getBytes(StandardCharsets.UTF_8)); + + template.requestBodyAndHeader("direct:chat", stream, Exchange.CONTENT_TYPE, "application/octet-stream", + String.class); + + UserMessage user = chatModel.lastUserMessage(); + assertThat(user.getText()).isEqualTo("What is Apache Camel?"); + assertThat(user.getMedia()).isEmpty(); + } + + @Test + void multipleMessagesTakesStrings() { + template.requestBody("direct:multiple", List.of("Hello", "What is Apache Camel?"), String.class); + + List<Message> sent = chatModel.lastPrompt().getInstructions(); + assertThat(sent).hasSize(2).allSatisfy(message -> assertThat(message).isInstanceOf(UserMessage.class)); + assertThat(sent.get(0).getText()).isEqualTo("Hello"); + assertThat(sent.get(1).getText()).isEqualTo("What is Apache Camel?"); + } + + @Test + void multipleMessagesTakesMessagesAndStrings() { + template.requestBody("direct:multiple", + List.of(new SystemMessage("Be brief"), "What is Apache Camel?"), String.class); + + List<Message> sent = chatModel.lastPrompt().getInstructions(); + assertThat(sent).hasSize(2); + assertThat(sent.get(0)).isInstanceOf(SystemMessage.class); + assertThat(sent.get(1)).isInstanceOf(UserMessage.class); + assertThat(sent.get(1).getText()).isEqualTo("What is Apache Camel?"); + } + + @Test + void multipleMessagesWithOtherElementFailsWithAClearMessage() { + assertThatThrownBy(() -> template.requestBody("direct:multiple", List.of(42), String.class)) + .isInstanceOf(CamelExecutionException.class) + .cause() + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("java.lang.Integer") + .hasMessageContaining("org.springframework.ai.chat.messages.Message"); + assertThat(chatModel.lastPrompt()).isNull(); + } + + /** + * A fake model that records the prompt and answers with a preset text. + */ + private static final class RecordingChatModel implements ChatModel { + + private final AtomicReference<Prompt> lastPrompt = new AtomicReference<>(); + private volatile String answer = "ok"; + + void answer(String answer) { + this.answer = answer; + } + + Prompt lastPrompt() { + return lastPrompt.get(); + } + + UserMessage lastUserMessage() { + return lastPrompt.get().getUserMessage(); + } + + @Override + public ChatResponse call(Prompt prompt) { + lastPrompt.set(prompt); + return new ChatResponse(List.of(new Generation(new AssistantMessage(answer)))); + } + } +} diff --git a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc index 10f5a47c1b89..7d20376df53d 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc @@ -4357,6 +4357,26 @@ The `paho` and `paho-mqtt5` producers now fail with an `InvalidPayloadException` body that does not convert to a `byte[]`, such as a `Map`, a POJO or no body. Previously they failed with a `NullPointerException` with no message. +=== camel-spring-ai-chat - File, Path and stream bodies, media without a user message, and outputFormat + +A `java.io.File` or `java.nio.file.Path` body of an image, audio, video or document is now sent as media, like a +`WrappedFile`. An `InputStream` body is sent as media when the `CamelSpringAiChatMediaType`, `CamelFileContentType` or +`Content-Type` header, or the extension in the `CamelFileName` header, names an image, audio, video or document (PDF, +Word, Excel) media type. Previously both were sent as their text. A text file, a file with an unknown extension, and a +stream with another or no media type are still sent as text. + +A media body (`byte[]`, `File`, `Path`, `InputStream`, `WrappedFile` or `List<WrappedFile>`) without the `userMessage` +option or the `CamelSpringAiChatUserMessage` header now fails with an `IllegalArgumentException` that names them, before +the media is read. Previously it failed inside Spring AI with "text cannot be null or empty". + +With `outputFormat` set to `BEAN`, `MAP` or `LIST`, the format instructions are now appended to the user message, or to +the last user message of `CHAT_MULTIPLE_MESSAGES` and `CHAT_SINGLE_MESSAGE_WITH_PROMPT`. Previously they replaced the +text of the user message, so the model did not get the question, and they were an extra user message after a +conversation or a prompt template. + +`CHAT_MULTIPLE_MESSAGES` now takes `String` elements in the list and sends each one as a `UserMessage`. Another element +fails with an `IllegalArgumentException`. Previously both failed with a `ClassCastException`. + === camel-spring-rabbitmq - the content type and the correlation id of a consumer reply The reply of a `spring-rabbitmq` consumer used to take the content type of the request whenever the route left the
