This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch CAMEL-26210-spring-ai-vector-store-delete-ids in repository https://gitbox.apache.org/repos/asf/camel.git
commit 3b3ee1dee9831aad2cb8576ddefd27f32a874b6b Author: Claus Ibsen <[email protected]> AuthorDate: Tue Nov 3 00:30:03 2026 +0100 CAMEL-26210: camel-spring-ai-vector-store - DELETE takes the ids from the body first, and the embeddings headers only go with a vector body DELETE read the CamelSpringAiVectorStoreDocumentIds header before the body, and ADD and SIMILARITY_SEARCH set that header to their output ids, so a later DELETE deleted those documents. DELETE now takes an id, a Document, or a List of ids or Documents from the body, and the header only without a body. ADD used a leftover CamelSpringAiEmbeddingInputText(s) header instead of a Document or text body, stored a float[] without its text as "[F@...", dropped mismatched vectors/texts, and skipped List elements it did not know. The headers now go only with a float[] or List of float[] body, and the other cases fail with a message. SIMILARITY_SEARCH with a float[] body searches for the text of the CamelSpringAiEmbeddingInputText header. DocumentsDeleted is documented as the number of ids given to the vector store. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../catalog/components/spring-ai-vector-store.json | 6 +- .../docs/spring-ai-vector-store-component.adoc | 31 ++-- .../vectorstore/spring-ai-vector-store.json | 6 +- .../docs/spring-ai-vector-store-component.adoc | 31 ++-- .../vectorstore/SpringAiVectorStoreEndpoint.java | 20 ++- .../vectorstore/SpringAiVectorStoreHeaders.java | 7 +- .../vectorstore/SpringAiVectorStoreProducer.java | 170 ++++++++++++--------- .../SpringAiVectorStoreComponentTest.java | 152 ++++++++++++++++++ .../ROOT/pages/camel-4x-upgrade-guide-4_24.adoc | 21 +++ .../SpringAiVectorStoreEndpointBuilderFactory.java | 7 +- 10 files changed, 345 insertions(+), 106 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/spring-ai-vector-store.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/spring-ai-vector-store.json index c4e957c49f71..cd877bad6562 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/spring-ai-vector-store.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/spring-ai-vector-store.json @@ -38,13 +38,13 @@ "CamelSpringAiVectorStoreTopK": { "index": 1, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "Integer", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The maximum number of similar documents to return (topK)", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#TOP_K" }, "CamelSpringAiVectorStoreSimilarityThreshold": { "index": 2, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "Double", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The similarity threshold (0-1)", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#SIMILARITY_THRESHOLD" }, "CamelSpringAiVectorStoreFilterExpression": { "index": 3, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Filter expression for metadata-based filtering", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#FILTER_EXPRESSION" }, - "CamelSpringAiVectorStoreDocumentIds": { "index": 4, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "java.util.List<String>", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "List of document IDs (input for DELETE, output for ADD and SIMILARITY_SEARCH)", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#DOCUMENT_IDS" }, + "CamelSpringAiVectorStoreDocumentIds": { "index": 4, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "java.util.List<String>", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "List of document IDs (output of ADD and SIMILARITY_SEARCH, and input of DELETE when the message has no body)", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#DOCUMEN [...] "CamelSpringAiVectorStoreSimilarDocuments": { "index": 5, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "java.util.List<org.springframework.ai.document.Document>", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "List of similar documents found", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#SIMILAR_DOCUMENTS" }, "CamelSpringAiVectorStoreDocumentsAdded": { "index": 6, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "Integer", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Number of documents added", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#DOCUMENTS_ADDED" }, - "CamelSpringAiVectorStoreDocumentsDeleted": { "index": 7, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "Integer", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Number of documents deleted", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#DOCUMENTS_DELETED" } + "CamelSpringAiVectorStoreDocumentsDeleted": { "index": 7, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "Integer", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Number of document IDs that DELETE gave to the vector store, as the vector store does not tell how many of them existed", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#DOCUME [...] }, "bodyTypes": { - "producerIn": { "carriers": [ "any" ], "byOption": [ { "option": "operation", "value": "DELETE", "carriers": [ "text", "list" ] }, { "option": "operation", "value": "SIMILARITY_SEARCH", "carriers": [ "text" ] } ], "description": "ADD takes a Document, a List of Document or of texts, or text (another body is added as its text). After spring-ai-embeddings it adds the texts in the CamelSpringAiEmbeddingInputText(s) headers instead of the body. DELETE takes the id or a List of ids, unles [...] + "producerIn": { "carriers": [ "any" ], "byOption": [ { "option": "operation", "value": "DELETE", "carriers": [ "text", "list", "pojo", "null" ] }, { "option": "operation", "value": "SIMILARITY_SEARCH", "carriers": [ "text", "list" ] } ], "description": "ADD takes a Document, a List of Document or of texts, or text (another body is added as its text); a List with another element fails. After spring-ai-embeddings it adds the float[] (or List of float[]) body with the text of the CamelS [...] "producerOut": { "carriers": [ "unchanged" ], "byOption": [ { "option": "operation", "value": "SIMILARITY_SEARCH", "carriers": [ "list" ] } ], "description": "SIMILARITY_SEARCH gives the List of Document found. ADD and DELETE keep the body, with the ids and the count in headers" } }, "properties": { diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/spring-ai-vector-store-component.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/spring-ai-vector-store-component.adoc index a2ed6ba7fb5f..484246bb8aee 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/spring-ai-vector-store-component.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/spring-ai-vector-store-component.adoc @@ -48,8 +48,13 @@ Stores documents in the vector store. Can accept: * String text (embeddings will be generated automatically) * Document objects -* List of Documents -* Output from the spring-ai-embeddings component (semi-automatic integration, this depends on the underlying Spring AI Vectore Store implementation) +* List of Documents or of String texts (an element of another type fails) +* Output from the spring-ai-embeddings component (semi-automatic integration, this depends on the underlying Spring AI Vectore Store implementation): +a `float[]` body with the `CamelSpringAiEmbeddingInputText` header, or a List of `float[]` body with the +`CamelSpringAiEmbeddingInputTexts` header with one text per vector. A vector without its text fails. + +The `CamelSpringAiEmbeddingInputText(s)` headers are only used together with a vector body, so a header left over from +an earlier spring-ai-embeddings call does not replace a Document or text body. After documents are added, the component sets the `CamelSpringAiVectorStoreDocumentIds` header containing a list of the IDs of the added documents. This allows you to reference or track the stored documents in subsequent processing steps. @@ -57,11 +62,19 @@ NOTE: *Best Practice:* Use the ADD operation directly with the vector store comp === DELETE Operation -Removes documents from the vector store by their IDs. +Removes documents from the vector store by their IDs. The body has the documents to delete: an ID, a Document, or a +List of IDs or of Documents (such as the result of SIMILARITY_SEARCH). Only when there is no body, the IDs are taken +from the `CamelSpringAiVectorStoreDocumentIds` header. The body comes first because ADD and SIMILARITY_SEARCH set this +header to their own output IDs. + +The `CamelSpringAiVectorStoreDocumentsDeleted` header is the number of IDs given to the vector store. The Spring AI +`VectorStore` API does not tell how many of them existed, so IDs that are not in the store are counted too. === SIMILARITY_SEARCH Operation -Searches for documents similar to a query text. +Searches for documents similar to a query text. After the spring-ai-embeddings component, where the body is the +`float[]` vector of the text, the query is the text in the `CamelSpringAiEmbeddingInputText` header (the vector store +embeds the query itself). The component returns matching documents in the message body and sets the following headers: @@ -217,17 +230,17 @@ YAML:: === Delete Documents -Delete documents by specifying their IDs: +Delete documents by specifying their IDs in the body: -._Java-only: Java constants and Arrays.asList()_ +._Java-only: Arrays.asList()_ [source,java] ---- from("direct:delete") - .setHeader(SpringAiVectorStoreHeaders.DOCUMENT_IDS, constant(Arrays.asList("doc1", "doc2"))) + .setBody(constant(Arrays.asList("doc1", "doc2"))) .to("spring-ai-vector-store:myStore?operation=DELETE"); ---- -You can also chain operations to search for documents and then delete them using the returned IDs: +You can also chain operations to search for documents and then delete the documents found: ._Java-only: Java constants and enum values_ [source,java] @@ -236,7 +249,7 @@ from("direct:searchAndDelete") .setBody(constant("outdated content")) .to("spring-ai-vector-store:myStore?operation=SIMILARITY_SEARCH&topK=10") .log("Found ${header.CamelSpringAiVectorStoreDocumentIds.size()} documents to delete") - // The DOCUMENT_IDS header from SIMILARITY_SEARCH is automatically used by DELETE + // DELETE deletes the List of Document that SIMILARITY_SEARCH gives in the body .setHeader(SpringAiVectorStoreHeaders.OPERATION, constant(SpringAiVectorStoreOperation.DELETE)) .to("spring-ai-vector-store:myStore"); ---- diff --git a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/generated/resources/META-INF/org/apache/camel/component/springai/vectorstore/spring-ai-vector-store.json b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/generated/resources/META-INF/org/apache/camel/component/springai/vectorstore/spring-ai-vector-store.json index c4e957c49f71..cd877bad6562 100644 --- a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/generated/resources/META-INF/org/apache/camel/component/springai/vectorstore/spring-ai-vector-store.json +++ b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/generated/resources/META-INF/org/apache/camel/component/springai/vectorstore/spring-ai-vector-store.json @@ -38,13 +38,13 @@ "CamelSpringAiVectorStoreTopK": { "index": 1, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "Integer", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The maximum number of similar documents to return (topK)", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#TOP_K" }, "CamelSpringAiVectorStoreSimilarityThreshold": { "index": 2, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "Double", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The similarity threshold (0-1)", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#SIMILARITY_THRESHOLD" }, "CamelSpringAiVectorStoreFilterExpression": { "index": 3, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Filter expression for metadata-based filtering", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#FILTER_EXPRESSION" }, - "CamelSpringAiVectorStoreDocumentIds": { "index": 4, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "java.util.List<String>", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "List of document IDs (input for DELETE, output for ADD and SIMILARITY_SEARCH)", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#DOCUMENT_IDS" }, + "CamelSpringAiVectorStoreDocumentIds": { "index": 4, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "java.util.List<String>", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "List of document IDs (output of ADD and SIMILARITY_SEARCH, and input of DELETE when the message has no body)", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#DOCUMEN [...] "CamelSpringAiVectorStoreSimilarDocuments": { "index": 5, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "java.util.List<org.springframework.ai.document.Document>", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "List of similar documents found", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#SIMILAR_DOCUMENTS" }, "CamelSpringAiVectorStoreDocumentsAdded": { "index": 6, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "Integer", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Number of documents added", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#DOCUMENTS_ADDED" }, - "CamelSpringAiVectorStoreDocumentsDeleted": { "index": 7, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "Integer", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Number of documents deleted", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#DOCUMENTS_DELETED" } + "CamelSpringAiVectorStoreDocumentsDeleted": { "index": 7, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "Integer", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Number of document IDs that DELETE gave to the vector store, as the vector store does not tell how many of them existed", "constantName": "org.apache.camel.component.springai.vectorstore.SpringAiVectorStoreHeaders#DOCUME [...] }, "bodyTypes": { - "producerIn": { "carriers": [ "any" ], "byOption": [ { "option": "operation", "value": "DELETE", "carriers": [ "text", "list" ] }, { "option": "operation", "value": "SIMILARITY_SEARCH", "carriers": [ "text" ] } ], "description": "ADD takes a Document, a List of Document or of texts, or text (another body is added as its text). After spring-ai-embeddings it adds the texts in the CamelSpringAiEmbeddingInputText(s) headers instead of the body. DELETE takes the id or a List of ids, unles [...] + "producerIn": { "carriers": [ "any" ], "byOption": [ { "option": "operation", "value": "DELETE", "carriers": [ "text", "list", "pojo", "null" ] }, { "option": "operation", "value": "SIMILARITY_SEARCH", "carriers": [ "text", "list" ] } ], "description": "ADD takes a Document, a List of Document or of texts, or text (another body is added as its text); a List with another element fails. After spring-ai-embeddings it adds the float[] (or List of float[]) body with the text of the CamelS [...] "producerOut": { "carriers": [ "unchanged" ], "byOption": [ { "option": "operation", "value": "SIMILARITY_SEARCH", "carriers": [ "list" ] } ], "description": "SIMILARITY_SEARCH gives the List of Document found. ADD and DELETE keep the body, with the ids and the count in headers" } }, "properties": { diff --git a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/docs/spring-ai-vector-store-component.adoc b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/docs/spring-ai-vector-store-component.adoc index a2ed6ba7fb5f..484246bb8aee 100644 --- a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/docs/spring-ai-vector-store-component.adoc +++ b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/docs/spring-ai-vector-store-component.adoc @@ -48,8 +48,13 @@ Stores documents in the vector store. Can accept: * String text (embeddings will be generated automatically) * Document objects -* List of Documents -* Output from the spring-ai-embeddings component (semi-automatic integration, this depends on the underlying Spring AI Vectore Store implementation) +* List of Documents or of String texts (an element of another type fails) +* Output from the spring-ai-embeddings component (semi-automatic integration, this depends on the underlying Spring AI Vectore Store implementation): +a `float[]` body with the `CamelSpringAiEmbeddingInputText` header, or a List of `float[]` body with the +`CamelSpringAiEmbeddingInputTexts` header with one text per vector. A vector without its text fails. + +The `CamelSpringAiEmbeddingInputText(s)` headers are only used together with a vector body, so a header left over from +an earlier spring-ai-embeddings call does not replace a Document or text body. After documents are added, the component sets the `CamelSpringAiVectorStoreDocumentIds` header containing a list of the IDs of the added documents. This allows you to reference or track the stored documents in subsequent processing steps. @@ -57,11 +62,19 @@ NOTE: *Best Practice:* Use the ADD operation directly with the vector store comp === DELETE Operation -Removes documents from the vector store by their IDs. +Removes documents from the vector store by their IDs. The body has the documents to delete: an ID, a Document, or a +List of IDs or of Documents (such as the result of SIMILARITY_SEARCH). Only when there is no body, the IDs are taken +from the `CamelSpringAiVectorStoreDocumentIds` header. The body comes first because ADD and SIMILARITY_SEARCH set this +header to their own output IDs. + +The `CamelSpringAiVectorStoreDocumentsDeleted` header is the number of IDs given to the vector store. The Spring AI +`VectorStore` API does not tell how many of them existed, so IDs that are not in the store are counted too. === SIMILARITY_SEARCH Operation -Searches for documents similar to a query text. +Searches for documents similar to a query text. After the spring-ai-embeddings component, where the body is the +`float[]` vector of the text, the query is the text in the `CamelSpringAiEmbeddingInputText` header (the vector store +embeds the query itself). The component returns matching documents in the message body and sets the following headers: @@ -217,17 +230,17 @@ YAML:: === Delete Documents -Delete documents by specifying their IDs: +Delete documents by specifying their IDs in the body: -._Java-only: Java constants and Arrays.asList()_ +._Java-only: Arrays.asList()_ [source,java] ---- from("direct:delete") - .setHeader(SpringAiVectorStoreHeaders.DOCUMENT_IDS, constant(Arrays.asList("doc1", "doc2"))) + .setBody(constant(Arrays.asList("doc1", "doc2"))) .to("spring-ai-vector-store:myStore?operation=DELETE"); ---- -You can also chain operations to search for documents and then delete them using the returned IDs: +You can also chain operations to search for documents and then delete the documents found: ._Java-only: Java constants and enum values_ [source,java] @@ -236,7 +249,7 @@ from("direct:searchAndDelete") .setBody(constant("outdated content")) .to("spring-ai-vector-store:myStore?operation=SIMILARITY_SEARCH&topK=10") .log("Found ${header.CamelSpringAiVectorStoreDocumentIds.size()} documents to delete") - // The DOCUMENT_IDS header from SIMILARITY_SEARCH is automatically used by DELETE + // DELETE deletes the List of Document that SIMILARITY_SEARCH gives in the body .setHeader(SpringAiVectorStoreHeaders.OPERATION, constant(SpringAiVectorStoreOperation.DELETE)) .to("spring-ai-vector-store:myStore"); ---- diff --git a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreEndpoint.java b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreEndpoint.java index 582433eb1d0a..9ae250ea4e5a 100644 --- a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreEndpoint.java +++ b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreEndpoint.java @@ -40,15 +40,21 @@ import org.apache.camel.support.DefaultEndpoint; @BodyTypes(producerIn = @BodyType(value = Carrier.ANY, byOption = { @BodyType.When(option = "operation", value = "DELETE", - carriers = { Carrier.TEXT, Carrier.LIST }), + carriers = { + Carrier.TEXT, Carrier.LIST, Carrier.POJO, + Carrier.NULL }), @BodyType.When(option = "operation", value = "SIMILARITY_SEARCH", - carriers = Carrier.TEXT) }, + carriers = { Carrier.TEXT, Carrier.LIST }) }, description = "ADD takes a Document, a List of Document or of texts, or text (another" - + " body is added as its text). After spring-ai-embeddings it adds the" - + " texts in the CamelSpringAiEmbeddingInputText(s) headers instead of the" - + " body. DELETE takes the id or a List of ids, unless the" - + " CamelSpringAiVectorStoreDocumentIds header has them, and" - + " SIMILARITY_SEARCH the query text. No body fails. The" + + " body is added as its text); a List with another element fails." + + " After spring-ai-embeddings it adds the float[] (or List of float[])" + + " body with the text of the CamelSpringAiEmbeddingInputText(s)" + + " header, and a vector without its text fails. DELETE takes the id, a" + + " Document, or a List of ids or Documents (such as the result of" + + " SIMILARITY_SEARCH); the CamelSpringAiVectorStoreDocumentIds header is" + + " used only without a body. SIMILARITY_SEARCH takes the query text, or" + + " after spring-ai-embeddings the float[] body with the" + + " CamelSpringAiEmbeddingInputText header. Otherwise no body fails. The" + " CamelSpringAiVectorStoreOperation header overrides operation"), producerOut = @BodyType(value = Carrier.UNCHANGED, byOption = @BodyType.When(option = "operation", value = "SIMILARITY_SEARCH", diff --git a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreHeaders.java b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreHeaders.java index 6ed5372738db..f29368874733 100644 --- a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreHeaders.java +++ b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreHeaders.java @@ -35,7 +35,8 @@ public class SpringAiVectorStoreHeaders { public static final String FILTER_EXPRESSION = "CamelSpringAiVectorStoreFilterExpression"; // Delete parameters / Results - @Metadata(description = "List of document IDs (input for DELETE, output for ADD and SIMILARITY_SEARCH)", + @Metadata(description = "List of document IDs (output of ADD and SIMILARITY_SEARCH, and input of DELETE when the message has no" + + " body)", javaType = "java.util.List<String>") public static final String DOCUMENT_IDS = "CamelSpringAiVectorStoreDocumentIds"; @@ -47,6 +48,8 @@ public class SpringAiVectorStoreHeaders { @Metadata(description = "Number of documents added", javaType = "Integer") public static final String DOCUMENTS_ADDED = "CamelSpringAiVectorStoreDocumentsAdded"; - @Metadata(description = "Number of documents deleted", javaType = "Integer") + @Metadata(description = "Number of document IDs that DELETE gave to the vector store, as the vector store does not tell how many" + + " of them existed", + javaType = "Integer") public static final String DOCUMENTS_DELETED = "CamelSpringAiVectorStoreDocumentsDeleted"; } diff --git a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreProducer.java b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreProducer.java index f62b023662d2..efea5df027e3 100644 --- a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreProducer.java +++ b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/main/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreProducer.java @@ -29,6 +29,11 @@ import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.VectorStore; public class SpringAiVectorStoreProducer extends DefaultProducer { + + // The headers of the spring-ai-embeddings component with the text of the embedding(s) in the body + private static final String INPUT_TEXT = "CamelSpringAiEmbeddingInputText"; + private static final String INPUT_TEXTS = "CamelSpringAiEmbeddingInputTexts"; + public SpringAiVectorStoreProducer(SpringAiVectorStoreEndpoint endpoint) { super(endpoint); } @@ -67,69 +72,59 @@ public class SpringAiVectorStoreProducer extends DefaultProducer { private void processAdd(Exchange exchange, VectorStore vectorStore) { final Message message = exchange.getMessage(); List<Document> documents = new ArrayList<>(); - - // Check if the message contains text and embedding from the embeddings component - String inputText = message.getHeader("CamelSpringAiEmbeddingInputText", String.class); Object body = message.getBody(); - if (inputText != null && body instanceof float[] floats) { + // The input text headers of the embeddings component belong to the vector(s) in the body: a leftover + // header must not replace a Document or text body + if (body instanceof float[] floats) { + String inputText = message.getHeader(INPUT_TEXT, String.class); + if (inputText == null) { + throw new IllegalArgumentException( + "ADD of a float[] body needs the " + INPUT_TEXT + " header with the text of the embedding"); + } // Create document from embedding component output with pre-computed embedding - Document doc = createDocumentWithEmbedding(inputText, floats); - documents.add(doc); - } else if (inputText != null) { - // Create document from embedding component output (text only, vector store will embed) - Document doc = new Document(inputText); - documents.add(doc); - } else { - // Check for batch texts and embeddings - List<String> inputTexts = message.getHeader("CamelSpringAiEmbeddingInputTexts", List.class); - - if (inputTexts != null && !inputTexts.isEmpty() && body instanceof List<?> bodyList) { - // Check if body contains float arrays (pre-computed embeddings) - if (!bodyList.isEmpty() && bodyList.get(0) instanceof float[]) { - // Create documents with pre-computed embeddings - List<float[]> embeddings = (List<float[]>) bodyList; - for (int i = 0; i < inputTexts.size() && i < embeddings.size(); i++) { - Document doc = createDocumentWithEmbedding(inputTexts.get(i), embeddings.get(i)); - documents.add(doc); - } - } else { - // Create documents without embeddings - for (String text : inputTexts) { - Document doc = new Document(text); - documents.add(doc); - } - } - } else if (inputTexts != null && !inputTexts.isEmpty()) { - // Create documents from texts only - for (String text : inputTexts) { - Document doc = new Document(text); - documents.add(doc); + documents.add(createDocumentWithEmbedding(inputText, floats)); + } else if (body instanceof List<?> list && !list.isEmpty() && list.get(0) instanceof float[]) { + List<String> inputTexts = message.getHeader(INPUT_TEXTS, List.class); + if (inputTexts == null || inputTexts.size() != list.size()) { + throw new IllegalArgumentException( + "ADD of a List of float[] body needs the " + INPUT_TEXTS + + " header with one text per embedding, got " + + (inputTexts == null ? "no header" : inputTexts.size() + " texts") + + " for " + list.size() + " embeddings"); + } + // Create documents with pre-computed embeddings + for (int i = 0; i < list.size(); i++) { + if (!(list.get(i) instanceof float[] floats)) { + throw new IllegalArgumentException( + "ADD of a List of float[] body has an element of type " + typeName(list.get(i)) + + " at index " + i); } - } else { - // Try to get documents directly from body - if (body instanceof Document doc) { + documents.add(createDocumentWithEmbedding(inputTexts.get(i), floats)); + } + } else if (body instanceof Document doc) { + documents.add(doc); + } else if (body instanceof List<?> list) { + for (Object item : list) { + if (item instanceof Document doc) { documents.add(doc); - } else if (body instanceof List<?> list) { - for (Object item : list) { - if (item instanceof Document doc) { - documents.add(doc); - } else if (item instanceof String str) { - // Create document from text (vector store will generate embeddings) - documents.add(new Document(str)); - } - } + } else if (item instanceof String str) { + // Create document from text (vector store will generate embeddings) + documents.add(new Document(str)); } else { - String str = message.getBody(String.class); - if (str != null) { - documents.add(new Document(str)); - } else { - throw new IllegalArgumentException( - "Message body must be a Document, List<Document>, String, List<String>, " + - "float[], or List<float[]>, or embeddings must be present in headers"); - } + throw new IllegalArgumentException( + "ADD of a List body takes Document or String elements, not " + typeName(item)); } } + } else { + String str = message.getBody(String.class); + if (str != null) { + documents.add(new Document(str)); + } else { + throw new IllegalArgumentException( + "Message body must be a Document, List<Document>, String, List<String>, " + + "float[], or List<float[]>, or embeddings must be present in headers"); + } } // Add documents to vector store @@ -143,6 +138,10 @@ public class SpringAiVectorStoreProducer extends DefaultProducer { message.setHeader(SpringAiVectorStoreHeaders.DOCUMENT_IDS, documentIds); } + private static String typeName(Object value) { + return value == null ? "null" : value.getClass().getName(); + } + /** * Creates a Document with pre-computed embedding stored in metadata. The embedding is stored in metadata with a * special key that can be used by vector stores that support pre-computed embeddings. @@ -160,19 +159,11 @@ public class SpringAiVectorStoreProducer extends DefaultProducer { private void processDelete(Exchange exchange, VectorStore vectorStore) { final Message message = exchange.getMessage(); - // Get document IDs from header or body - List<String> documentIds = message.getHeader(SpringAiVectorStoreHeaders.DOCUMENT_IDS, List.class); - - if (documentIds == null) { - Object body = message.getBody(); - if (body instanceof List) { - documentIds = (List<String>) body; - } else { - String str = message.getBody(String.class); - if (str != null) { - documentIds = List.of(str); - } - } + // Get document IDs from the body, else from the header. The body comes first, as ADD and SIMILARITY_SEARCH + // set the header to their own output ids + List<String> documentIds = documentIdsFromBody(message); + if (documentIds.isEmpty()) { + documentIds = message.getHeader(SpringAiVectorStoreHeaders.DOCUMENT_IDS, List.class); } if (documentIds != null && !documentIds.isEmpty()) { @@ -198,11 +189,50 @@ public class SpringAiVectorStoreProducer extends DefaultProducer { } } + /** + * The ids of the documents to delete in the body: an id, a Document, or a List of ids or Documents (such as the + * result of SIMILARITY_SEARCH). + */ + private static List<String> documentIdsFromBody(Message message) { + Object body = message.getBody(); + if (body == null) { + return List.of(); + } + if (body instanceof Document doc) { + return List.of(doc.getId()); + } + if (body instanceof List<?> list) { + List<String> ids = new ArrayList<>(list.size()); + for (Object item : list) { + if (item instanceof Document doc) { + ids.add(doc.getId()); + } else if (item instanceof String str) { + ids.add(str); + } else { + throw new IllegalArgumentException( + "DELETE of a List body takes String ids or Documents, not " + typeName(item)); + } + } + return ids; + } + String str = message.getBody(String.class); + return str == null || str.isEmpty() ? List.of() : List.of(str); + } + private void processSimilaritySearch(Exchange exchange, VectorStore vectorStore) { final Message message = exchange.getMessage(); - // Get search parameters - String query = message.getBody(String.class); + // Get search parameters. After spring-ai-embeddings the body is the vector of the text, so search for its text + String query; + if (message.getBody() instanceof float[]) { + query = message.getHeader(INPUT_TEXT, String.class); + if (query == null) { + throw new IllegalArgumentException( + "SIMILARITY_SEARCH of a float[] body needs the " + INPUT_TEXT + " header with the query text"); + } + } else { + query = message.getBody(String.class); + } if (query == null || query.isEmpty()) { throw new IllegalArgumentException("Query text is required for SIMILARITY_SEARCH operation"); } diff --git a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/test/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreComponentTest.java b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/test/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreComponentTest.java index 19de85225150..fc2069afd082 100644 --- a/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/test/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreComponentTest.java +++ b/components/camel-spring-parent/camel-spring-ai/camel-spring-ai-vector-store/src/test/java/org/apache/camel/component/springai/vectorstore/SpringAiVectorStoreComponentTest.java @@ -21,11 +21,13 @@ import java.util.List; import java.util.Map; import org.apache.camel.CamelContext; +import org.apache.camel.CamelExecutionException; import org.apache.camel.Message; import org.apache.camel.builder.RouteBuilder; import org.apache.camel.test.junit6.CamelTestSupport; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.springframework.ai.document.Document; import org.springframework.ai.embedding.Embedding; import org.springframework.ai.embedding.EmbeddingModel; @@ -34,8 +36,12 @@ import org.springframework.ai.embedding.EmbeddingResponse; import org.springframework.ai.embedding.EmbeddingResponseMetadata; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.SimpleVectorStore; +import org.springframework.ai.vectorstore.VectorStore; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; public class SpringAiVectorStoreComponentTest extends CamelTestSupport { @@ -345,6 +351,152 @@ public class SpringAiVectorStoreComponentTest extends CamelTestSupport { assertThat(documentsDeleted).isEqualTo(1); } + @Test + public void testAddDocumentIgnoresLeftoverInputTextHeader() { + // The header of an earlier spring-ai-embeddings call must not replace a Document body + Document doc = new Document("doc-new", "The new document", Map.of()); + + Message result = fluentTemplate.to("spring-ai-vector-store:test?operation=ADD") + .withBody(doc) + .withHeader("CamelSpringAiEmbeddingInputText", "Old embedded text") + .request(Message.class); + + assertThat(result.getHeader(SpringAiVectorStoreHeaders.DOCUMENT_IDS, List.class)).containsExactly("doc-new"); + assertThat(allTexts()).containsExactly("The new document"); + } + + @Test + public void testAddListOfTextsIgnoresLeftoverInputTextsHeader() { + Message result = fluentTemplate.to("spring-ai-vector-store:test?operation=ADD") + .withBody(List.of("First text", "Second text")) + .withHeader("CamelSpringAiEmbeddingInputTexts", List.of("Old text")) + .request(Message.class); + + assertThat(result.getHeader(SpringAiVectorStoreHeaders.DOCUMENTS_ADDED, Integer.class)).isEqualTo(2); + assertThat(allTexts()).containsExactlyInAnyOrder("First text", "Second text"); + } + + @Test + public void testAddVectorWithoutInputTextFails() { + assertThatThrownBy(() -> fluentTemplate.to("spring-ai-vector-store:test?operation=ADD") + .withBody(new float[] { 0.1f, 0.2f, 0.3f }) + .request()) + .isInstanceOf(CamelExecutionException.class) + .cause() + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("CamelSpringAiEmbeddingInputText"); + + assertThat(allTexts()).isEmpty(); + } + + @Test + public void testAddVectorsWithMismatchedInputTextsFails() { + assertThatThrownBy(() -> fluentTemplate.to("spring-ai-vector-store:test?operation=ADD") + .withBody(List.of(new float[] { 0.1f }, new float[] { 0.2f })) + .withHeader("CamelSpringAiEmbeddingInputTexts", List.of("Only one text")) + .request()) + .isInstanceOf(CamelExecutionException.class) + .cause() + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("CamelSpringAiEmbeddingInputTexts") + .hasMessageContaining("1 texts for 2 embeddings"); + + assertThat(allTexts()).isEmpty(); + } + + @Test + public void testAddListOfMapFails() { + assertThatThrownBy(() -> fluentTemplate.to("spring-ai-vector-store:test?operation=ADD") + .withBody(List.of(Map.of("text", "A map"))) + .request()) + .isInstanceOf(CamelExecutionException.class) + .cause() + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Document or String elements") + .hasMessageContaining("Map"); + } + + @Test + public void testSimilaritySearchWithVectorBodySearchesInputText() { + VectorStore recordingStore = mock(VectorStore.class); + context.getRegistry().bind("recordingStore", recordingStore); + + fluentTemplate.to("spring-ai-vector-store:test?operation=SIMILARITY_SEARCH&vectorStore=#recordingStore") + .withBody(new float[] { 0.1f, 0.2f, 0.3f }) + .withHeader("CamelSpringAiEmbeddingInputText", "What is AI?") + .request(Message.class); + + ArgumentCaptor<SearchRequest> request = ArgumentCaptor.forClass(SearchRequest.class); + verify(recordingStore).similaritySearch(request.capture()); + assertThat(request.getValue().getQuery()).isEqualTo("What is AI?"); + } + + @Test + public void testSimilaritySearchWithVectorBodyWithoutInputTextFails() { + assertThatThrownBy(() -> fluentTemplate.to("spring-ai-vector-store:test?operation=SIMILARITY_SEARCH") + .withBody(new float[] { 0.1f, 0.2f, 0.3f }) + .request()) + .isInstanceOf(CamelExecutionException.class) + .cause() + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("CamelSpringAiEmbeddingInputText"); + } + + @Test + public void testDeleteAfterSearchDeletesIdsOfBody() { + vectorStore.add(List.of( + new Document("doc-a", "Document A", Map.of()), + new Document("doc-b", "Document B", Map.of()), + new Document("doc-c", "Document C", Map.of()))); + + // SIMILARITY_SEARCH sets the DocumentIds header to all the documents it found + Message found = fluentTemplate.to("spring-ai-vector-store:test?operation=SIMILARITY_SEARCH&topK=10") + .withBody("Document") + .request(Message.class); + assertThat(found.getHeader(SpringAiVectorStoreHeaders.DOCUMENT_IDS, List.class)).hasSize(3); + + // a later DELETE of one id in the body deletes only that id + Message result = fluentTemplate.to("spring-ai-vector-store:test?operation=DELETE") + .withBody("doc-b") + .withHeader(SpringAiVectorStoreHeaders.DOCUMENT_IDS, + found.getHeader(SpringAiVectorStoreHeaders.DOCUMENT_IDS)) + .request(Message.class); + + assertThat(result.getHeader(SpringAiVectorStoreHeaders.DOCUMENTS_DELETED, Integer.class)).isEqualTo(1); + assertThat(allTexts()).containsExactlyInAnyOrder("Document A", "Document C"); + } + + @Test + public void testDeleteDocumentsOfSearchResult() { + vectorStore.add(List.of( + new Document("doc-x", "Outdated X", Map.of()), + new Document("doc-y", "Outdated Y", Map.of()))); + + // the body of SIMILARITY_SEARCH is a List of Document, which DELETE deletes by their ids + Message result = fluentTemplate.to("spring-ai-vector-store:test?operation=DELETE") + .withBody(List.of(new Document("doc-x", "Outdated X", Map.of()))) + .request(Message.class); + + assertThat(result.getHeader(SpringAiVectorStoreHeaders.DOCUMENTS_DELETED, Integer.class)).isEqualTo(1); + assertThat(allTexts()).containsExactly("Outdated Y"); + } + + @Test + public void testDeleteListOfOtherElementsFails() { + assertThatThrownBy(() -> fluentTemplate.to("spring-ai-vector-store:test?operation=DELETE") + .withBody(List.of(42)) + .request()) + .isInstanceOf(CamelExecutionException.class) + .cause() + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("java.lang.Integer"); + } + + private List<String> allTexts() { + return vectorStore.similaritySearch(SearchRequest.builder().query("any").topK(100).build()) + .stream().map(Document::getText).toList(); + } + @Override protected RouteBuilder createRouteBuilder() throws Exception { return new RouteBuilder() { diff --git a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc index 10f5a47c1b89..c7a47af1d20b 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc @@ -804,6 +804,27 @@ CamelContext of that provider. It now closes each of them only when no other `JC it returned the closed cache, whose operations throw `IllegalStateException`. This matters for code that calls a `JCacheManager` (for example one created by a custom `JCacheManagerFactory`) after it was closed. +=== camel-spring-ai-vector-store - DELETE takes the ids from the body first, and the embeddings headers go with a vector body + +The DELETE operation now takes the ids from the body: an id, a `Document`, or a List of ids or of `Document`. The +`CamelSpringAiVectorStoreDocumentIds` header is used only when the message has no body. Before, the header won over +the body, and as ADD and SIMILARITY_SEARCH set this header to their own output ids, a later DELETE in the same route +deleted those documents instead of the ids in the body. A route that sets the header and sends a body that is not the +ids should remove the body (for example `setBody(constant(null))`) or put the ids in the body. Deleting what +SIMILARITY_SEARCH found still works, as its body is the List of `Document` found. A List with an element that is not a +String or a `Document` now fails. + +The ADD operation now uses the `CamelSpringAiEmbeddingInputText` and `CamelSpringAiEmbeddingInputTexts` headers of +spring-ai-embeddings only with a `float[]` or a List of `float[]` body. Before, a header left over from an earlier +spring-ai-embeddings call replaced a `Document` or text body. A `float[]` body without the text header, a List of +`float[]` with a different number of texts, and a List with an element that is not a `Document` or a String now fail +with an `IllegalArgumentException`. Before, a vector without its text was stored as the text of `float[].toString()`, +extra vectors or texts were dropped, and other elements were skipped. + +SIMILARITY_SEARCH with a `float[]` body (after spring-ai-embeddings) now searches for the text of the +`CamelSpringAiEmbeddingInputText` header, and fails without it. Before, it searched for the text of +`float[].toString()`. + === camel-spring-batch - the CamelSpringBatchJobName header needs jobFromHeader=true The producer now takes the job name from the `CamelSpringBatchJobName` header only when the endpoint has diff --git a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/SpringAiVectorStoreEndpointBuilderFactory.java b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/SpringAiVectorStoreEndpointBuilderFactory.java index 73076a2d1135..8cb374da2f5f 100644 --- a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/SpringAiVectorStoreEndpointBuilderFactory.java +++ b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/SpringAiVectorStoreEndpointBuilderFactory.java @@ -364,8 +364,8 @@ public interface SpringAiVectorStoreEndpointBuilderFactory { return "CamelSpringAiVectorStoreFilterExpression"; } /** - * List of document IDs (input for DELETE, output for ADD and - * SIMILARITY_SEARCH). + * List of document IDs (output of ADD and SIMILARITY_SEARCH, and input + * of DELETE when the message has no body). * * The option is a: {@code java.util.List<String>} type. * @@ -405,7 +405,8 @@ public interface SpringAiVectorStoreEndpointBuilderFactory { return "CamelSpringAiVectorStoreDocumentsAdded"; } /** - * Number of documents deleted. + * Number of document IDs that DELETE gave to the vector store, as the + * vector store does not tell how many of them existed. * * The option is a: {@code Integer} type. *
