This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new b875a54603d4 CAMEL-26202: camel-langchain4j-agent - send a file, bytes
or stream content once without a user message, accept JSON/XML/YAML and
File/Path bodies (#29026)
b875a54603d4 is described below
commit b875a54603d4da864add3352b7e3944ca54287c3
Author: Claus Ibsen <[email protected]>
AuthorDate: Tue Nov 3 11:44:28 2026 +0100
CAMEL-26202: camel-langchain4j-agent - send a file, bytes or stream content
once without a user message, accept JSON/XML/YAML and File/Path bodies (#29026)
* CAMEL-26202: camel-langchain4j-agent - send a file, bytes or stream c…
…ontent once without a user message, accept JSON/XML/YAML and File/Path
bodies
A file, bytes or stream body without the CamelLangChain4jAgentUserMessage
header
got an empty user message text, and LangChain4j then put the content twice
in the
user message. An AiAgentBody with a blank user message and a content failed
with
"template cannot be null or blank". The agents now call new content-only
service
methods when the user message is empty or blank.
application/json, application/xml, application/yaml and +json/+xml types are
sent as a TextContent instead of failing with Unsupported MIME type, and
MIME
types match case-insensitively. A java.io.File or java.nio.file.Path body is
converted like a WrappedFile.
* CAMEL-26202: camel-langchain4j-agent - detect application/yaml from a…
… .yaml or .yml file name
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Signed-off-by: Claus Ibsen <[email protected]>
---
.../catalog/components/langchain4j-agent.json | 2 +-
.../catalog/docs/langchain4j-agent-multimodal.adoc | 15 ++
.../langchain4j/agent/api/AgentWithMemory.java | 8 +-
.../langchain4j/agent/api/AgentWithoutMemory.java | 8 +-
.../agent/api/AiAgentWithMemoryService.java | 21 ++
.../agent/api/AiAgentWithoutMemoryService.java | 18 ++
.../agent/LangChain4jAgentConverterLoader.java | 16 ++
.../langchain4j/agent/langchain4j-agent.json | 2 +-
.../main/docs/langchain4j-agent-multimodal.adoc | 15 ++
.../agent/LangChain4jAgentConverter.java | 110 +++++++-
.../agent/LangChain4jAgentEndpoint.java | 11 +-
.../agent/LangChain4jAgentContentBodyTest.java | 286 +++++++++++++++++++++
12 files changed, 489 insertions(+), 23 deletions(-)
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/langchain4j-agent.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/langchain4j-agent.json
index 725bac3f9159..d95ae61ea863 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/langchain4j-agent.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/langchain4j-agent.json
@@ -69,7 +69,7 @@
"CamelLangChain4jAgentModerationFlagged": { "index": 14, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "Boolean", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Set to true when user input is rejected
by the configured moderation model; not set on success.", "constantName":
"org.apache.camel.component.langchain4j.agent.api.Headers#MODERATION_FLAGGED" }
},
"bodyTypes": {
- "producerIn": { "carriers": [ "text", "pojo", "bytes", "stream", "file" ],
"description": "The user message as text, or an AiAgentBody with the user
message, the system message, the memory id and a content. Bytes, a stream or a
file of the file consumers is sent as an image, audio, video, PDF or text
content after the text of the CamelLangChain4jAgentUserMessage header; bytes
and a stream need the CamelLangChain4jAgentMediaType or Content-Type header. A
Map, a java.io.File or another [...]
+ "producerIn": { "carriers": [ "text", "pojo", "bytes", "stream", "file" ],
"description": "The user message as text, or an AiAgentBody with the user
message, the system message, the memory id and a content. Bytes, a stream, a
file of the file consumers, a java.io.File or a java.nio.file.Path is sent as
an image, audio, video, PDF or text content (JSON, XML and YAML as text), after
the text of the CamelLangChain4jAgentUserMessage header or alone without it;
bytes and a stream need the [...]
"producerOut": { "carriers": [ "text" ], "description": "The answer of the
agent as text, also with outputClass or jsonSchema: unmarshal the JSON to read
its fields. The token counts, the finish reason, the sources and the tool
executions are in headers" }
},
"properties": {
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-agent-multimodal.adoc
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-agent-multimodal.adoc
index efe1792b0744..3c4473b38898 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-agent-multimodal.adoc
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-agent-multimodal.adoc
@@ -52,8 +52,23 @@ The agent component automatically converts files from
various Camel components t
|`InputStream`
|Various streaming components
|From content type headers (required)
+
+|`java.io.File`, `java.nio.file.Path`
+|A local file set as the message body
+|From file extension or headers
|===
+The MIME type selects the content sent to the model:
+
+* `image/*`, `audio/*` and `video/*` are sent as an image, audio or video
content.
+* `application/pdf` is sent as a PDF content.
+* `text/*`, `application/json`, `application/xml`, `application/yaml` and
types with a `+json` or `+xml` suffix are sent as a text content (read as
UTF-8).
+
+Any other MIME type fails.
+
+The text of the `CamelLangChain4jAgentUserMessage` header is sent before the
content in the same user message.
+Without that header (or when it is blank), the user message holds only the
content.
+
==== Example: Processing Images from File Component
[tabs]
diff --git
a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithMemory.java
b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithMemory.java
index a14a13d0a98e..647a56541616 100644
---
a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithMemory.java
+++
b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithMemory.java
@@ -20,6 +20,7 @@ import dev.langchain4j.data.message.Content;
import dev.langchain4j.service.AiServices;
import dev.langchain4j.service.Result;
import dev.langchain4j.service.tool.ToolProvider;
+import org.apache.camel.util.ObjectHelper;
/**
* Implementation of Agent for AI agents with memory support. This agent
handles chat interactions while maintaining
@@ -45,7 +46,12 @@ public class AgentWithMemory extends
AbstractAgent<AiAgentWithMemoryService> {
String systemMessage = aiAgentBody.getSystemMessage();
Content content = aiAgentBody.getContent();
- if (content != null) {
+ if (content != null && ObjectHelper.isEmpty(userMessage)) {
+ // Only the content (such as a file) without a text: an empty or
blank text must not be sent
+ return systemMessage != null
+ ? agentService.chat(memoryId, content, systemMessage)
+ : agentService.chat(memoryId, content);
+ } else if (content != null) {
// Multi-modal message with content
return systemMessage != null
? agentService.chat(memoryId, userMessage, content,
systemMessage)
diff --git
a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithoutMemory.java
b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithoutMemory.java
index 7970237d3276..5c74380aa05c 100644
---
a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithoutMemory.java
+++
b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AgentWithoutMemory.java
@@ -20,6 +20,7 @@ import dev.langchain4j.data.message.Content;
import dev.langchain4j.service.AiServices;
import dev.langchain4j.service.Result;
import dev.langchain4j.service.tool.ToolProvider;
+import org.apache.camel.util.ObjectHelper;
/**
* Implementation of Agent for AI agents without memory support. This agent
handles chat interactions without
@@ -44,7 +45,12 @@ public class AgentWithoutMemory extends
AbstractAgent<AiAgentWithoutMemoryServic
String systemMessage = aiAgentBody.getSystemMessage();
Content content = aiAgentBody.getContent();
- if (content != null) {
+ if (content != null && ObjectHelper.isEmpty(userMessage)) {
+ // Only the content (such as a file) without a text: an empty or
blank text must not be sent
+ return systemMessage != null
+ ? agentService.chat(content, systemMessage)
+ : agentService.chat(content);
+ } else if (content != null) {
// Multi-modal message with content
return systemMessage != null
? agentService.chat(userMessage, content, systemMessage)
diff --git
a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithMemoryService.java
b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithMemoryService.java
index 48be3234c8bd..01c9e160a152 100644
---
a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithMemoryService.java
+++
b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithMemoryService.java
@@ -48,6 +48,16 @@ public interface AiAgentWithMemoryService {
*/
Result<String> chat(@MemoryId Object memoryId, @UserMessage String
message, @UserMessage Content content);
+ /**
+ * Chat with a user message that holds only a content (e.g., an image or
the text of a file), without a text, with
+ * memory support.
+ *
+ * @param memoryId the memory identifier for conversation history
+ * @param content the content such as ImageContent, TextContent, etc.
+ * @return the AI response with token usage metadata
+ */
+ Result<String> chat(@MemoryId Object memoryId, @UserMessage Content
content);
+
/**
* Simple chat with a user message, system message and memory window
*
@@ -75,4 +85,15 @@ public interface AiAgentWithMemoryService {
@MemoryId Object memoryId, @UserMessage String message,
@UserMessage Content content, @V("prompt") String prompt);
+ /**
+ * Chat with a user message that holds only a content, without a text,
with system message and memory support.
+ *
+ * @param memoryId the memory identifier for conversation history
+ * @param content the content such as ImageContent, TextContent, etc.
+ * @param prompt the system message template
+ * @return the AI response with token usage metadata
+ */
+ @SystemMessage("{{prompt}}")
+ Result<String> chat(@MemoryId Object memoryId, @UserMessage Content
content, @V("prompt") String prompt);
+
}
diff --git
a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithoutMemoryService.java
b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithoutMemoryService.java
index 57519a53e81a..5e49d897c8ab 100644
---
a/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithoutMemoryService.java
+++
b/components/camel-ai/camel-langchain4j-agent-api/src/main/java/org/apache/camel/component/langchain4j/agent/api/AiAgentWithoutMemoryService.java
@@ -44,6 +44,14 @@ public interface AiAgentWithoutMemoryService {
*/
Result<String> chat(@UserMessage String message, @UserMessage Content
content);
+ /**
+ * Chat with a user message that holds only a content (e.g., an image or
the text of a file), without a text.
+ *
+ * @param content the content such as ImageContent, TextContent, etc.
+ * @return the AI response with token usage metadata
+ */
+ Result<String> chat(@UserMessage Content content);
+
/**
* Simple chat with a single user message and system message
*
@@ -65,4 +73,14 @@ public interface AiAgentWithoutMemoryService {
@SystemMessage("{{prompt}}")
Result<String> chat(@UserMessage String message, @UserMessage Content
content, @V("prompt") String prompt);
+ /**
+ * Chat with a user message that holds only a content, without a text,
with system message.
+ *
+ * @param content the content such as ImageContent, TextContent, etc.
+ * @param prompt the system message template
+ * @return the AI response with token usage metadata
+ */
+ @SystemMessage("{{prompt}}")
+ Result<String> chat(@UserMessage Content content, @V("prompt") String
prompt);
+
}
diff --git
a/components/camel-ai/camel-langchain4j-agent/src/generated/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverterLoader.java
b/components/camel-ai/camel-langchain4j-agent/src/generated/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverterLoader.java
index 240733da94ce..a9fb40da5667 100644
---
a/components/camel-ai/camel-langchain4j-agent/src/generated/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverterLoader.java
+++
b/components/camel-ai/camel-langchain4j-agent/src/generated/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverterLoader.java
@@ -52,6 +52,14 @@ public final class LangChain4jAgentConverterLoader
implements TypeConverterLoade
}
return answer;
});
+ addTypeConverter(registry,
org.apache.camel.component.langchain4j.agent.api.AiAgentBody.class,
java.io.File.class, false,
+ (type, exchange, value) -> {
+ Object answer =
org.apache.camel.component.langchain4j.agent.LangChain4jAgentConverter.fileToAiAgentBody((java.io.File)
value, exchange);
+ if (false && answer == null) {
+ answer = Void.class;
+ }
+ return answer;
+ });
addTypeConverter(registry,
org.apache.camel.component.langchain4j.agent.api.AiAgentBody.class,
java.io.InputStream.class, false,
(type, exchange, value) -> {
Object answer =
org.apache.camel.component.langchain4j.agent.LangChain4jAgentConverter.inputStreamToAiAgentBody((java.io.InputStream)
value, exchange);
@@ -68,6 +76,14 @@ public final class LangChain4jAgentConverterLoader
implements TypeConverterLoade
}
return answer;
});
+ addTypeConverter(registry,
org.apache.camel.component.langchain4j.agent.api.AiAgentBody.class,
java.nio.file.Path.class, false,
+ (type, exchange, value) -> {
+ Object answer =
org.apache.camel.component.langchain4j.agent.LangChain4jAgentConverter.pathToAiAgentBody((java.nio.file.Path)
value, exchange);
+ if (false && answer == null) {
+ answer = Void.class;
+ }
+ return answer;
+ });
addTypeConverter(registry,
org.apache.camel.component.langchain4j.agent.api.AiAgentBody.class,
org.apache.camel.WrappedFile.class, false,
(type, exchange, value) -> {
Object answer =
org.apache.camel.component.langchain4j.agent.LangChain4jAgentConverter.toAiAgentBody((org.apache.camel.WrappedFile)
value, exchange);
diff --git
a/components/camel-ai/camel-langchain4j-agent/src/generated/resources/META-INF/org/apache/camel/component/langchain4j/agent/langchain4j-agent.json
b/components/camel-ai/camel-langchain4j-agent/src/generated/resources/META-INF/org/apache/camel/component/langchain4j/agent/langchain4j-agent.json
index 725bac3f9159..d95ae61ea863 100644
---
a/components/camel-ai/camel-langchain4j-agent/src/generated/resources/META-INF/org/apache/camel/component/langchain4j/agent/langchain4j-agent.json
+++
b/components/camel-ai/camel-langchain4j-agent/src/generated/resources/META-INF/org/apache/camel/component/langchain4j/agent/langchain4j-agent.json
@@ -69,7 +69,7 @@
"CamelLangChain4jAgentModerationFlagged": { "index": 14, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "Boolean", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Set to true when user input is rejected
by the configured moderation model; not set on success.", "constantName":
"org.apache.camel.component.langchain4j.agent.api.Headers#MODERATION_FLAGGED" }
},
"bodyTypes": {
- "producerIn": { "carriers": [ "text", "pojo", "bytes", "stream", "file" ],
"description": "The user message as text, or an AiAgentBody with the user
message, the system message, the memory id and a content. Bytes, a stream or a
file of the file consumers is sent as an image, audio, video, PDF or text
content after the text of the CamelLangChain4jAgentUserMessage header; bytes
and a stream need the CamelLangChain4jAgentMediaType or Content-Type header. A
Map, a java.io.File or another [...]
+ "producerIn": { "carriers": [ "text", "pojo", "bytes", "stream", "file" ],
"description": "The user message as text, or an AiAgentBody with the user
message, the system message, the memory id and a content. Bytes, a stream, a
file of the file consumers, a java.io.File or a java.nio.file.Path is sent as
an image, audio, video, PDF or text content (JSON, XML and YAML as text), after
the text of the CamelLangChain4jAgentUserMessage header or alone without it;
bytes and a stream need the [...]
"producerOut": { "carriers": [ "text" ], "description": "The answer of the
agent as text, also with outputClass or jsonSchema: unmarshal the JSON to read
its fields. The token counts, the finish reason, the sources and the tool
executions are in headers" }
},
"properties": {
diff --git
a/components/camel-ai/camel-langchain4j-agent/src/main/docs/langchain4j-agent-multimodal.adoc
b/components/camel-ai/camel-langchain4j-agent/src/main/docs/langchain4j-agent-multimodal.adoc
index efe1792b0744..3c4473b38898 100644
---
a/components/camel-ai/camel-langchain4j-agent/src/main/docs/langchain4j-agent-multimodal.adoc
+++
b/components/camel-ai/camel-langchain4j-agent/src/main/docs/langchain4j-agent-multimodal.adoc
@@ -52,8 +52,23 @@ The agent component automatically converts files from
various Camel components t
|`InputStream`
|Various streaming components
|From content type headers (required)
+
+|`java.io.File`, `java.nio.file.Path`
+|A local file set as the message body
+|From file extension or headers
|===
+The MIME type selects the content sent to the model:
+
+* `image/*`, `audio/*` and `video/*` are sent as an image, audio or video
content.
+* `application/pdf` is sent as a PDF content.
+* `text/*`, `application/json`, `application/xml`, `application/yaml` and
types with a `+json` or `+xml` suffix are sent as a text content (read as
UTF-8).
+
+Any other MIME type fails.
+
+The text of the `CamelLangChain4jAgentUserMessage` header is sent before the
content in the same user message.
+Without that header (or when it is blank), the user message holds only the
content.
+
==== Example: Processing Images from File Component
[tabs]
diff --git
a/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverter.java
b/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverter.java
index 3b890a1ade04..1c0478ab4f0f 100644
---
a/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverter.java
+++
b/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentConverter.java
@@ -22,7 +22,9 @@ import java.io.InputStream;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.Base64;
+import java.util.Locale;
import dev.langchain4j.data.audio.Audio;
import dev.langchain4j.data.image.Image;
@@ -59,6 +61,7 @@ import static
org.apache.camel.component.langchain4j.agent.api.Headers.USER_MESS
* </p>
* <ul>
* <li>{@link WrappedFile} - from file, ftp, sftp components</li>
+ * <li>{@link File} and {@link Path} - a local file</li>
* <li>{@code byte[]} - from aws2-s3, azure-storage-blob, and other cloud
components</li>
* <li>{@link InputStream} - from various streaming components</li>
* </ul>
@@ -105,6 +108,50 @@ public final class LangChain4jAgentConverter {
return buildAiAgentBody(exchange, content, "");
}
+ /**
+ * Converts a {@link File} to an {@link AiAgentBody} with the appropriate
{@link Content} type.
+ * <p>
+ * Works like the {@link WrappedFile} converter: the MIME type comes from
the {@code CamelLangChain4jAgentMediaType}
+ * or {@code CamelFileContentType} header, else from the file extension.
+ * </p>
+ *
+ * @param file the file to send
+ * @param exchange the Camel exchange containing headers
+ * @return an AiAgentBody with the appropriate
Content type
+ * @throws IllegalArgumentException if the file cannot be read or the MIME
type is unsupported
+ */
+ @Converter
+ public static AiAgentBody<?> fileToAiAgentBody(File file, Exchange
exchange) {
+ String mimeType = detectMimeType(file.getName(), exchange);
+ Content content = createContent(readFileBytes(file), mimeType);
+ return buildAiAgentBody(exchange, content, "");
+ }
+
+ /**
+ * Converts a {@link Path} to an {@link AiAgentBody} with the appropriate
{@link Content} type.
+ * <p>
+ * Works like the {@link File} converter.
+ * </p>
+ *
+ * @param path the path of the file to send
+ * @param exchange the Camel exchange containing headers
+ * @return an AiAgentBody with the appropriate
Content type
+ * @throws IllegalArgumentException if the file cannot be read or the MIME
type is unsupported
+ */
+ @Converter
+ public static AiAgentBody<?> pathToAiAgentBody(Path path, Exchange
exchange) {
+ Path fileName = path.getFileName();
+ String mimeType = detectMimeType(fileName != null ?
fileName.toString() : null, exchange);
+ byte[] data;
+ try {
+ data = Files.readAllBytes(path);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Failed to read file: " +
path.toAbsolutePath(), e);
+ }
+ Content content = createContent(data, mimeType);
+ return buildAiAgentBody(exchange, content, "");
+ }
+
private static String resolveFileName(WrappedFile<?> wrappedFile, Exchange
exchange) {
String headerName =
exchange.getMessage().getHeader(Exchange.FILE_NAME, String.class);
if (headerName != null) {
@@ -221,42 +268,58 @@ public final class LangChain4jAgentConverter {
/**
* Creates the appropriate LangChain4j Content object based on the MIME
type.
+ * <p>
+ * Textual types ({@code text/*}, JSON, XML and YAML, also with a {@code
+json} or {@code +xml} suffix) become a
+ * {@link TextContent}.
+ * </p>
*/
static Content createContent(byte[] data, String mimeType) {
String base64Data = Base64.getEncoder().encodeToString(data);
+ String type = mimeType.toLowerCase(Locale.ROOT);
- if (mimeType.startsWith("image/")) {
+ if (type.startsWith("image/")) {
Image image = Image.builder()
.base64Data(base64Data)
.mimeType(mimeType)
.build();
return ImageContent.from(image);
- } else if (mimeType.startsWith("audio/")) {
+ } else if (type.startsWith("audio/")) {
Audio audio = Audio.builder()
.base64Data(base64Data)
.mimeType(mimeType)
.build();
return AudioContent.from(audio);
- } else if (mimeType.startsWith("video/")) {
+ } else if (type.startsWith("video/")) {
Video video = Video.builder()
.base64Data(base64Data)
.mimeType(mimeType)
.build();
return VideoContent.from(video);
- } else if ("application/pdf".equals(mimeType)) {
+ } else if ("application/pdf".equals(type)) {
PdfFile pdfFile = PdfFile.builder()
.base64Data(base64Data)
.build();
return PdfFileContent.from(pdfFile);
- } else if (mimeType.startsWith("text/")) {
+ } else if (isTextual(type)) {
return TextContent.from(new String(data, StandardCharsets.UTF_8));
} else {
throw new IllegalArgumentException(
"Unsupported MIME type: " + mimeType
- + ". Supported types: image/*,
audio/*, video/*, application/pdf, text/*");
+ + ". Supported types: image/*,
audio/*, video/*, application/pdf, text/*,"
+ + " application/json,
application/xml, application/yaml, */*+json, */*+xml");
}
}
+ private static boolean isTextual(String type) {
+ return type.startsWith("text/")
+ || "application/json".equals(type)
+ || "application/xml".equals(type)
+ || "application/yaml".equals(type)
+ || "application/x-yaml".equals(type)
+ || type.endsWith("+json")
+ || type.endsWith("+xml");
+ }
+
/**
* Detects the MIME type from headers or file extension.
* <p>
@@ -343,6 +406,10 @@ public final class LangChain4jAgentConverter {
String fileName = message.getHeader(Exchange.FILE_NAME, String.class);
if (fileName != null) {
String mime = URLConnection.guessContentTypeFromName(fileName);
+ if (mime == null) {
+ // the JDK does not know some extensions such as .yaml
+ mime = mimeTypeFromExtension(fileName);
+ }
if (mime != null) {
return normalizeContentType(mime);
}
@@ -361,14 +428,32 @@ public final class LangChain4jAgentConverter {
*/
private static String normalizeContentType(String contentType) {
int semicolon = contentType.indexOf(';');
- return semicolon > 0 ? contentType.substring(0, semicolon).trim() :
contentType;
+ return semicolon > 0 ? contentType.substring(0, semicolon).trim() :
contentType.trim();
}
/**
* Detects the MIME type from the file extension.
+ *
+ * @throws IllegalArgumentException if the file extension is not known
*/
private static String detectMimeTypeFromExtension(String fileName) {
- String lowerName = fileName.toLowerCase();
+ String mime = mimeTypeFromExtension(fileName);
+ if (mime == null) {
+ LOG.warn(
+ "Could not detect MIME type from file extension: {}.
Please set the CamelLangChain4jAgentMediaType header.",
+ fileName);
+ throw new IllegalArgumentException(
+ "Cannot determine MIME type for file: " + fileName
+ + ". Please set the
CamelLangChain4jAgentMediaType header.");
+ }
+ return mime;
+ }
+
+ /**
+ * The MIME type of a known file extension, or {@code null} if the
extension is not known.
+ */
+ private static String mimeTypeFromExtension(String fileName) {
+ String lowerName = fileName.toLowerCase(Locale.ROOT);
// Image formats
if (lowerName.endsWith(".png")) {
@@ -427,13 +512,10 @@ public final class LangChain4jAgentConverter {
return "text/xml";
} else if (lowerName.endsWith(".json")) {
return "application/json";
+ } else if (lowerName.endsWith(".yaml") || lowerName.endsWith(".yml")) {
+ return "application/yaml";
}
-
- LOG.warn("Could not detect MIME type from file extension: {}. Please
set the CamelLangChain4jAgentMediaType header.",
- fileName);
- throw new IllegalArgumentException(
- "Cannot determine MIME type for file: " + fileName
- + ". Please set the
CamelLangChain4jAgentMediaType header.");
+ return null;
}
/**
diff --git
a/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentEndpoint.java
b/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentEndpoint.java
index ca22f0e9c6c9..f065ca8de17a 100644
---
a/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentEndpoint.java
+++
b/components/camel-ai/camel-langchain4j-agent/src/main/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentEndpoint.java
@@ -40,11 +40,12 @@ import static
org.apache.camel.component.langchain4j.agent.LangChain4jAgent.SCHE
category = { Category.AI }, headersClass = Headers.class)
@BodyTypes(producerIn = @BodyType(value = { Carrier.TEXT, Carrier.POJO,
Carrier.BYTES, Carrier.STREAM, Carrier.FILE },
description = "The user message as text, or
an AiAgentBody with the user message, the system"
- + " message, the memory id and
a content. Bytes, a stream or a file of the file"
- + " consumers is sent as an
image, audio, video, PDF or text content after the"
- + " text of the
CamelLangChain4jAgentUserMessage header; bytes and a stream need"
- + " the
CamelLangChain4jAgentMediaType or Content-Type header. A Map, a"
- + " java.io.File or another
object fails, and no body fails"),
+ + " message, the memory id and
a content. Bytes, a stream, a file of the file"
+ + " consumers, a java.io.File
or a java.nio.file.Path is sent as an image,"
+ + " audio, video, PDF or text
content (JSON, XML and YAML as text), after the"
+ + " text of the
CamelLangChain4jAgentUserMessage header or alone without it;"
+ + " bytes and a stream need
the CamelLangChain4jAgentMediaType or Content-Type"
+ + " header. A Map or another
object fails, and no body fails"),
producerOut = @BodyType(value = Carrier.TEXT,
description = "The answer of the agent as
text, also with outputClass or jsonSchema:"
+ " unmarshal the JSON to
read its fields. The token counts, the finish"
diff --git
a/components/camel-ai/camel-langchain4j-agent/src/test/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentContentBodyTest.java
b/components/camel-ai/camel-langchain4j-agent/src/test/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentContentBodyTest.java
new file mode 100644
index 000000000000..426cd4770917
--- /dev/null
+++
b/components/camel-ai/camel-langchain4j-agent/src/test/java/org/apache/camel/component/langchain4j/agent/LangChain4jAgentContentBodyTest.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.langchain4j.agent;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import dev.langchain4j.data.message.AiMessage;
+import dev.langchain4j.data.message.ChatMessage;
+import dev.langchain4j.data.message.Content;
+import dev.langchain4j.data.message.ImageContent;
+import dev.langchain4j.data.message.SystemMessage;
+import dev.langchain4j.data.message.TextContent;
+import dev.langchain4j.data.message.UserMessage;
+import dev.langchain4j.memory.chat.MessageWindowChatMemory;
+import dev.langchain4j.model.chat.ChatModel;
+import dev.langchain4j.model.chat.request.ChatRequest;
+import dev.langchain4j.model.chat.response.ChatResponse;
+import org.apache.camel.Exchange;
+import org.apache.camel.RoutesBuilder;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.file.GenericFile;
+import org.apache.camel.component.langchain4j.agent.api.AgentConfiguration;
+import org.apache.camel.component.langchain4j.agent.api.AgentWithMemory;
+import org.apache.camel.component.langchain4j.agent.api.AgentWithoutMemory;
+import org.apache.camel.component.langchain4j.agent.api.AiAgentBody;
+import org.apache.camel.component.langchain4j.agent.api.Headers;
+import org.apache.camel.spi.Registry;
+import org.apache.camel.test.junit6.CamelTestSupport;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Checks the user message the chat model receives for a file, bytes, stream
or {@link AiAgentBody} body with a content.
+ */
+class LangChain4jAgentContentBodyTest extends CamelTestSupport {
+
+ @TempDir
+ Path tempDir;
+
+ private final CapturingChatModel chatModel = new CapturingChatModel();
+
+ @Override
+ protected void bindToRegistry(Registry registry) {
+ registry.bind("agent", new AgentWithoutMemory(new
AgentConfiguration().withChatModel(chatModel)));
+ registry.bind("memoryAgent", new AgentWithMemory(
+ new AgentConfiguration()
+ .withChatModel(chatModel)
+ .withChatMemoryProvider(memoryId ->
MessageWindowChatMemory.withMaxMessages(10))));
+ }
+
+ @Override
+ protected RoutesBuilder createRouteBuilder() {
+ return new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("direct:agent").to("langchain4j-agent:test?agent=#agent");
+
from("direct:memory").to("langchain4j-agent:memory?agent=#memoryAgent");
+ }
+ };
+ }
+
+ @BeforeEach
+ void reset() {
+ chatModel.requests.clear();
+ }
+
+ @Test
+ void textFileWithoutUserMessageSendsTheContentOnce() throws Exception {
+ File file = Files.writeString(tempDir.resolve("notes.txt"), "Hello
file").toFile();
+ GenericFile<File> genericFile = new GenericFile<>();
+ genericFile.setFile(file);
+ genericFile.setFileName(file.getName());
+
+ String reply = template.requestBody("direct:agent", genericFile,
String.class);
+
+ assertThat(reply).isEqualTo("ok");
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("Hello file"));
+ }
+
+ @Test
+ void textBytesWithoutUserMessageSendsTheContentOnce() {
+ template.requestBodyAndHeader("direct:agent", "Hello
bytes".getBytes(StandardCharsets.UTF_8),
+ Headers.MEDIA_TYPE, "text/plain", String.class);
+
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("Hello bytes"));
+ }
+
+ @Test
+ void textStreamWithoutUserMessageSendsTheContentOnceWithMemory() {
+ template.requestBodyAndHeaders("direct:memory",
+ new ByteArrayInputStream("Hello
stream".getBytes(StandardCharsets.UTF_8)),
+ Map.of(Headers.MEDIA_TYPE, "text/plain", Headers.MEMORY_ID,
"m1"), String.class);
+
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("Hello stream"));
+ }
+
+ @Test
+ void imageBytesWithoutUserMessageSendsOnlyTheImage() {
+ template.requestBodyAndHeader("direct:agent", new byte[] { 1, 2, 3 },
Headers.MEDIA_TYPE, "image/png",
+ String.class);
+
+
assertThat(lastUserMessageContents()).singleElement().isInstanceOf(ImageContent.class);
+ }
+
+ @Test
+ void textBytesWithSystemMessageAndWithoutUserMessageSendsTheContentOnce() {
+ template.requestBodyAndHeaders("direct:agent", "Hello
bytes".getBytes(StandardCharsets.UTF_8),
+ Map.of(Headers.MEDIA_TYPE, "text/plain",
Headers.SYSTEM_MESSAGE, "Be brief"), String.class);
+
+ List<ChatMessage> messages =
chatModel.requests.get(chatModel.requests.size() - 1).messages();
+ assertThat(messages.get(0)).isInstanceOfSatisfying(SystemMessage.class,
+ m -> assertThat(m.text()).isEqualTo("Be brief"));
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("Hello bytes"));
+ }
+
+ @Test
+ void textFileWithUserMessageSendsTextAndContent() throws Exception {
+ File file = Files.writeString(tempDir.resolve("notes.txt"), "Hello
file").toFile();
+ GenericFile<File> genericFile = new GenericFile<>();
+ genericFile.setFile(file);
+ genericFile.setFileName(file.getName());
+
+ template.requestBodyAndHeader("direct:agent", genericFile,
Headers.USER_MESSAGE, "Summarize", String.class);
+
+ assertThat(lastUserMessageContents()).hasSize(2);
+ assertThat(((TextContent)
lastUserMessageContents().get(0)).text()).isEqualTo("Summarize");
+ assertThat(((TextContent)
lastUserMessageContents().get(1)).text()).isEqualTo("Hello file");
+ }
+
+ @Test
+ void blankUserMessageWithContentSendsOnlyTheContent() {
+ AiAgentBody<TextContent> body = new AiAgentBody<TextContent>(" ")
+ .withContent(TextContent.from("Hello content"));
+
+ String reply = template.requestBody("direct:agent", body,
String.class);
+
+ assertThat(reply).isEqualTo("ok");
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("Hello content"));
+ }
+
+ @Test
+ void blankUserMessageWithContentSendsOnlyTheContentWithMemory() {
+ AiAgentBody<TextContent> body = new AiAgentBody<TextContent>(" ")
+ .withContent(TextContent.from("Hello content"))
+ .withMemoryId("m2");
+
+ String reply = template.requestBody("direct:memory", body,
String.class);
+
+ assertThat(reply).isEqualTo("ok");
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("Hello content"));
+ }
+
+ @Test
+ void jsonFileIsSentAsText() throws Exception {
+ File file = Files.writeString(tempDir.resolve("order.json"),
"{\"id\":1}").toFile();
+ GenericFile<File> genericFile = new GenericFile<>();
+ genericFile.setFile(file);
+ genericFile.setFileName(file.getName());
+
+ template.requestBodyAndHeader("direct:agent", genericFile,
Headers.USER_MESSAGE, "Check", String.class);
+
+ assertThat(lastUserMessageContents()).hasSize(2);
+ assertThat(((TextContent)
lastUserMessageContents().get(1)).text()).isEqualTo("{\"id\":1}");
+ }
+
+ @Test
+ void jsonAndXmlBytesAreSentAsText() {
+ template.requestBodyAndHeader("direct:agent",
"{\"id\":2}".getBytes(StandardCharsets.UTF_8),
+ Exchange.CONTENT_TYPE, "application/json; charset=UTF-8",
String.class);
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("{\"id\":2}"));
+
+ template.requestBodyAndHeader("direct:agent",
"<id>3</id>".getBytes(StandardCharsets.UTF_8),
+ Exchange.CONTENT_TYPE, "Application/XML", String.class);
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("<id>3</id>"));
+
+ template.requestBodyAndHeader("direct:agent",
"{\"id\":4}".getBytes(StandardCharsets.UTF_8),
+ Exchange.CONTENT_TYPE, "application/vnd.api+json",
String.class);
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("{\"id\":4}"));
+ }
+
+ @Test
+ void fileBodyIsConverted() throws Exception {
+ File file = Files.writeString(tempDir.resolve("plain.txt"), "Hello
java.io.File").toFile();
+
+ template.requestBodyAndHeader("direct:agent", file,
Headers.USER_MESSAGE, "Read", String.class);
+
+ assertThat(lastUserMessageContents()).hasSize(2);
+ assertThat(((TextContent)
lastUserMessageContents().get(1)).text()).isEqualTo("Hello java.io.File");
+ }
+
+ @Test
+ void pathBodyIsConverted() throws Exception {
+ Path path = Files.writeString(tempDir.resolve("data.json"),
"{\"path\":true}");
+
+ template.requestBody("direct:agent", path, String.class);
+
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("{\"path\":true}"));
+ }
+
+ @Test
+ void yamlFileBodyIsSentAsText() throws Exception {
+ File yaml = Files.writeString(tempDir.resolve("config.yaml"), "name:
yaml").toFile();
+ template.requestBody("direct:agent", yaml, String.class);
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("name: yaml"));
+
+ File yml = Files.writeString(tempDir.resolve("config.yml"), "name:
yml").toFile();
+ template.requestBody("direct:agent", yml, String.class);
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("name: yml"));
+ }
+
+ @Test
+ void yamlPathBodyIsSentAsText() throws Exception {
+ Path yaml = Files.writeString(tempDir.resolve("route.YAML"), "kind:
yaml");
+ template.requestBody("direct:agent", yaml, String.class);
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("kind: yaml"));
+
+ Path yml = Files.writeString(tempDir.resolve("route.yml"), "kind:
yml");
+ template.requestBody("direct:agent", yml, String.class);
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("kind: yml"));
+ }
+
+ @Test
+ void yamlBytesWithFileNameAreSentAsText() {
+ template.requestBodyAndHeader("direct:agent", "name:
bytes".getBytes(StandardCharsets.UTF_8),
+ Exchange.FILE_NAME, "config.yaml", String.class);
+ assertThat(lastUserMessageContents()).singleElement()
+ .isInstanceOfSatisfying(TextContent.class, c ->
assertThat(c.text()).isEqualTo("name: bytes"));
+ }
+
+ private List<Content> lastUserMessageContents() {
+ assertThat(chatModel.requests).isNotEmpty();
+ List<ChatMessage> messages =
chatModel.requests.get(chatModel.requests.size() - 1).messages();
+ ChatMessage last = messages.get(messages.size() - 1);
+ assertThat(last).isInstanceOf(UserMessage.class);
+ return ((UserMessage) last).contents();
+ }
+
+ private static final class CapturingChatModel implements ChatModel {
+
+ private final List<ChatRequest> requests = new ArrayList<>();
+
+ @Override
+ public ChatResponse doChat(ChatRequest request) {
+ requests.add(request);
+ return
ChatResponse.builder().aiMessage(AiMessage.from("ok")).build();
+ }
+ }
+}
