This is an automated email from the ASF dual-hosted git repository.
oscerd pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
The following commit(s) were added to refs/heads/main by this push:
new d107d3468 Fix #3015: Add LangChain4j document ingestion Kamelets
(#3016)
d107d3468 is described below
commit d107d34685c5d3c72f1d63e505f05ab50c238d31
Author: Jiří Ondrušek <[email protected]>
AuthorDate: Tue Sep 15 11:23:35 2026 +0200
Fix #3015: Add LangChain4j document ingestion Kamelets (#3016)
The ingestion half of a RAG setup over the camel-langchain4j-ingest
component (Camel 4.23): a sink that splits, batch-embeds and stores
documents with identity metadata and optional dedup; generic Tika and
Docling parse actions that capture the document id before the parse, so
a document cannot forge its own identity; and a file source with
knowledge-base semantics (read without consuming, edits re-delivered,
raw bytes for parsers). Citrus tests cover each Kamelet and the
compositions; they need Camel 4.23+ and fail on older versions by
design, gating the merge until the catalog bumps
(run with -Dcamel.version=4.23.0-SNAPSHOT).
Co-authored-by: Claude Fable 5 <[email protected]>
---
docs/modules/ROOT/nav.adoc | 4 +
kamelets/docling-convert-action.kamelet.yaml | 86 ++++++++++++++
.../langchain4j-ingest-file-source.kamelet.yaml | 101 +++++++++++++++++
kamelets/langchain4j-ingest-sink.kamelet.yaml | 123 +++++++++++++++++++++
kamelets/tika-extract-text-action.kamelet.yaml | 86 ++++++++++++++
.../kamelets/docling-convert-action.kamelet.yaml | 86 ++++++++++++++
.../langchain4j-ingest-file-source.kamelet.yaml | 101 +++++++++++++++++
.../kamelets/langchain4j-ingest-sink.kamelet.yaml | 123 +++++++++++++++++++++
.../kamelets/tika-extract-text-action.kamelet.yaml | 86 ++++++++++++++
.../src/test/java/CommonIT.java | 8 ++
.../docling-convert-route.citrus.it.yaml | 89 +++++++++++++++
.../langchain4j-ingest/docling-convert-route.yaml | 61 ++++++++++
...ain4j-ingest-file-register-route.citrus.it.yaml | 84 ++++++++++++++
.../langchain4j-ingest-file-register-route.yaml | 57 ++++++++++
...chain4j-ingest-file-source-route.citrus.it.yaml | 72 ++++++++++++
.../langchain4j-ingest-file-source-route.yaml | 42 +++++++
...ngchain4j-ingest-file-tika-route.citrus.it.yaml | 65 +++++++++++
.../langchain4j-ingest-file-tika-route.yaml | 58 ++++++++++
.../langchain4j-ingest-sink-route.citrus.it.yaml | 62 +++++++++++
.../langchain4j-ingest-sink-route.yaml | 58 ++++++++++
.../resources/langchain4j-ingest/pdfs/pump.pdf | Bin 0 -> 850 bytes
.../tika-extract-text-route.citrus.it.yaml | 62 +++++++++++
.../tika-extract-text-route.yaml | 60 ++++++++++
23 files changed, 1574 insertions(+)
diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc
index 662aa4cc9..7406fe27d 100644
--- a/docs/modules/ROOT/nav.adoc
+++ b/docs/modules/ROOT/nav.adoc
@@ -74,6 +74,7 @@
* xref:dns-dig-action.adoc[]
* xref:dns-ip-action.adoc[]
* xref:dns-lookup-action.adoc[]
+* xref:docling-convert-action.adoc[]
* xref:drop-field-action.adoc[]
* xref:drop-header-action.adoc[]
* xref:drop-headers-action.adoc[]
@@ -164,6 +165,8 @@
* xref:kubernetes-namespaces-source.adoc[]
* xref:kubernetes-nodes-source.adoc[]
* xref:kubernetes-pods-source.adoc[]
+* xref:langchain4j-ingest-file-source.adoc[]
+* xref:langchain4j-ingest-sink.adoc[]
* xref:log-action.adoc[]
* xref:log-sink.adoc[]
* xref:mail-imap-source.adoc[]
@@ -245,6 +248,7 @@
* xref:telegram-sink.adoc[]
* xref:telegram-source.adoc[]
* xref:throttle-action.adoc[]
+* xref:tika-extract-text-action.adoc[]
* xref:timer-source.adoc[]
* xref:timestamp-router-action.adoc[]
* xref:topic-name-matches-filter-action.adoc[]
diff --git a/kamelets/docling-convert-action.kamelet.yaml
b/kamelets/docling-convert-action.kamelet.yaml
new file mode 100644
index 000000000..d1a00f80e
--- /dev/null
+++ b/kamelets/docling-convert-action.kamelet.yaml
@@ -0,0 +1,86 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: docling-convert-action
+ labels:
+ camel.apache.org/kamelet.type: "action"
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.22.1-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2NCA2NCI+PHJlY3QgeD0iOCIgeT0iNiIgd2lkdGg9IjI4IiBoZWlnaHQ9IjM2IiByeD0iMyIgZmlsbD0iIzhmYjRmZSIvPjxyZWN0IHg9IjEyIiB5PSIxMiIgd2lkdGg9IjIwIiBoZWlnaHQ9IjMiIGZpbGw9IiNmZmYiLz48cmVjdCB4PSIxMiIgeT0iMTkiIHdpZHRoPSIyMCIgaGVpZ2h0PSIzIiBmaWxsPSIjZmZmIi8+PHJlY3QgeD0iMTIiIHk9IjI2IiB3aWR0aD0iMTQiIGhlaWdodD0iMyIgZmlsbD0iI2ZmZiIvPjxjaXJjbGUgY3g9IjQ2IiBjeT0iMjAiIHI9IjQiIGZpbG
[...]
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.group: "Actions"
+ camel.apache.org/kamelet.namespace: "AI"
+spec:
+ definition:
+ title: "Docling Convert Action"
+ description: |-
+ Convert PDF, office and similar documents to structure-preserving
markdown with
+ Docling, OCR included.
+
+ Before the conversion, the value of the `documentIdHeader` header is
captured into the
+ `CamelLangChain4jIngestDocumentId` exchange property, so a converted
document cannot
+ forge its own identity in steps that read it later (the
langchain4j-ingest sink does);
+ the body is pinned to bytes and the `CamelDocling*` control headers are
swept, because
+ none of them may be decided by a consumer-delivered payload. By default
the `docling`
+ component executes a local docling CLI; set `doclingServeUrl` to use a
Docling Serve
+ instance instead - the natural fit for self-contained Pipes.
+ type: object
+ properties:
+ documentIdHeader:
+ title: Document Id Header
+ description: Name of the header carrying the stable document id,
captured into the
+ CamelLangChain4jIngestDocumentId exchange property before the
conversion.
+ type: string
+ default: CamelLangChain4jIngestDocumentId
+ doclingServeUrl:
+ title: Docling Serve URL
+ description: Address of a Docling Serve instance to convert through,
such as
+ http://docling-serve:5001; when unset, the docling component's own
configuration
+ applies (a local docling CLI by default).
+ type: string
+ useDoclingServe:
+ title: Use Docling Serve
+ description: Whether to convert through a Docling Serve instance
instead of a local
+ docling CLI; set together with doclingServeUrl.
+ type: boolean
+ dependencies:
+ - "camel:core"
+ - "camel:docling"
+ - "camel:kamelet"
+ template:
+ from:
+ uri: "kamelet:source"
+ steps:
+ # captured before the conversion, so a document cannot forge its own
identity
+ - setProperty:
+ name: "CamelLangChain4jIngestDocumentId"
+ simple: "${header[{{documentIdHeader}}]}"
+ - convertBodyTo:
+ type: "byte[]"
+ - removeHeaders:
+ pattern: "CamelDocling*"
+ - to:
+ uri: "docling:convert"
+ parameters:
+ operation: "CONVERT_TO_MARKDOWN"
+ contentInBody: "true"
+ useDoclingServe: "{{?useDoclingServe}}"
+ doclingServeUrl: "{{?doclingServeUrl}}"
diff --git a/kamelets/langchain4j-ingest-file-source.kamelet.yaml
b/kamelets/langchain4j-ingest-file-source.kamelet.yaml
new file mode 100644
index 000000000..5114b2e42
--- /dev/null
+++ b/kamelets/langchain4j-ingest-file-source.kamelet.yaml
@@ -0,0 +1,101 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: langchain4j-ingest-file-source
+ labels:
+ camel.apache.org/kamelet.type: "source"
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.22.1-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2NCA2NCI+PHJlY3QgeD0iOCIgeT0iNiIgd2lkdGg9IjI4IiBoZWlnaHQ9IjM2IiByeD0iMyIgZmlsbD0iIzhmYjRmZSIvPjxyZWN0IHg9IjEyIiB5PSIxMiIgd2lkdGg9IjIwIiBoZWlnaHQ9IjMiIGZpbGw9IiNmZmYiLz48cmVjdCB4PSIxMiIgeT0iMTkiIHdpZHRoPSIyMCIgaGVpZ2h0PSIzIiBmaWxsPSIjZmZmIi8+PHJlY3QgeD0iMTIiIHk9IjI2IiB3aWR0aD0iMTQiIGhlaWdodD0iMyIgZmlsbD0iI2ZmZiIvPjxjaXJjbGUgY3g9IjQ2IiBjeT0iMjAiIHI9IjQiIGZpbG
[...]
+ camel.apache.org/kamelet.group: "LangChain4j Ingest"
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.namespace: "AI"
+spec:
+ definition:
+ title: "LangChain4j Ingest File Source"
+ description: |-
+ Watch a directory as the source of an ingestion pipeline, with the
defaults a knowledge
+ base needs: documents are left in place (noop), unchanged files are
remembered in a
+ duplicate register keyed on path, modification time and size, a file
still being copied
+ in is waited for (readLock=changed), and the file name is exposed as the
document id in
+ the CamelLangChain4jIngestDocumentId header.
+
+ Mind the two dedup layers when composing: this source re-delivers an
edited file (its
+ register key changes with the modification time), but a
langchain4j-ingest-sink with an
+ idempotentRepository is first-write-wins by document id - the
re-delivered edit would
+ be skipped. Use a version-aware id, or no sink repository, when
documents change.
+ required:
+ - directory
+ type: object
+ properties:
+ directory:
+ title: Directory
+ description: The directory to ingest documents from.
+ type: string
+ pattern: "[^?#]+"
+ recursive:
+ title: Recursive
+ description: Whether subdirectories are ingested too.
+ type: boolean
+ default: true
+ delay:
+ title: Delay
+ description: Milliseconds between directory polls; the file endpoint's
default is
+ 500ms, which is aggressive for a knowledge-base directory.
+ type: integer
+ charset:
+ title: Charset
+ description: Character set for reading text documents, for example
UTF-8. Leave
+ unset when the source feeds a parser action
(tika-extract-text-action or
+ docling-convert-action), which must receive the raw bytes - a charset
+ conversion would corrupt a binary document such as a PDF.
+ type: string
+ idempotentRepository:
+ title: Idempotent Repository
+ description: An IdempotentRepository bean remembering already consumed
files, as a
+ `#bean:name` reference; the register is keyed on path, modification
time and
+ size, so an edited file is consumed again. When unset, the file
endpoint's
+ default in-memory register (1000 entries) applies - eviction on a
large directory
+ re-ingests files, so supply a sized or persistent repository for a
real knowledge
+ base.
+ type: string
+ dependencies:
+ - "camel:core"
+ - "camel:file"
+ - "camel:kamelet"
+ template:
+ from:
+ uri: "file:{{directory}}"
+ parameters:
+ noop: "true"
+ idempotent: "true"
+ # an edited file gets a new key and re-ingests
+ idempotentKey: "${file:absolute.path}:${file:modified}:${file:size}"
+ recursive: "{{recursive}}"
+ readLock: "changed"
+ delay: "{{?delay}}"
+ charset: "{{?charset}}"
+ idempotentRepository: "{{?idempotentRepository}}"
+ steps:
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ simple: "${header.CamelFileName}"
+ - to: "kamelet:sink"
diff --git a/kamelets/langchain4j-ingest-sink.kamelet.yaml
b/kamelets/langchain4j-ingest-sink.kamelet.yaml
new file mode 100644
index 000000000..768c3e1c4
--- /dev/null
+++ b/kamelets/langchain4j-ingest-sink.kamelet.yaml
@@ -0,0 +1,123 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: langchain4j-ingest-sink
+ labels:
+ camel.apache.org/kamelet.type: "sink"
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.22.1-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2NCA2NCI+PHJlY3QgeD0iOCIgeT0iNiIgd2lkdGg9IjI4IiBoZWlnaHQ9IjM2IiByeD0iMyIgZmlsbD0iIzhmYjRmZSIvPjxyZWN0IHg9IjEyIiB5PSIxMiIgd2lkdGg9IjIwIiBoZWlnaHQ9IjMiIGZpbGw9IiNmZmYiLz48cmVjdCB4PSIxMiIgeT0iMTkiIHdpZHRoPSIyMCIgaGVpZ2h0PSIzIiBmaWxsPSIjZmZmIi8+PHJlY3QgeD0iMTIiIHk9IjI2IiB3aWR0aD0iMTQiIGhlaWdodD0iMyIgZmlsbD0iI2ZmZiIvPjxjaXJjbGUgY3g9IjQ2IiBjeT0iMjAiIHI9IjQiIGZpbG
[...]
+ camel.apache.org/kamelet.group: "LangChain4j Ingest"
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.namespace: "AI"
+spec:
+ definition:
+ title: "LangChain4j Ingest Sink"
+ description: |-
+ Ingest documents into a LangChain4j EmbeddingStore: the payload is split
into overlapping
+ segments, embedded in batches and written to the store, each segment
stamped with the
+ pipeline name and the document id.
+
+ The document id is read from the `CamelLangChain4jIngestDocumentId`
exchange property
+ when a prior step captured it (the parser actions do, before the parse),
else from the
+ header named by `documentIdHeader` - the property always wins, so a
parsed document
+ cannot forge its own identity. Bean references are `#bean:name` values;
when
+ `embeddingStore` or `embeddingModel` is not set, the single registry
bean of that type is
+ used. With `idempotentRepository` set, the first write per document id
wins - a
+ re-delivered edited document is skipped, so streams that carry updates
need a
+ version-aware id.
+ type: object
+ properties:
+ pipelineName:
+ title: Pipeline Name
+ description: The pipeline name, stamped on every written segment.
+ type: string
+ pattern: "[A-Za-z0-9._-]+"
+ default: ingest
+ documentIdHeader:
+ title: Document Id Header
+ description: Name of the header carrying the stable document id, such
as CamelAwsS3Key
+ for an S3 source or CamelKafkaKey for a Kafka one; the
+ CamelLangChain4jIngestDocumentId exchange property, when set, takes
precedence.
+ type: string
+ default: CamelLangChain4jIngestDocumentId
+ maxSegmentSize:
+ title: Max Segment Size
+ description: Maximum size of one segment, in characters.
+ type: integer
+ default: 500
+ maxOverlapSize:
+ title: Max Overlap Size
+ description: How much of the previous segment each segment repeats, in
characters.
+ type: integer
+ default: 50
+ embeddingBatchSize:
+ title: Embedding Batch Size
+ description: How many segments are embedded per request to the
embedding model.
+ type: integer
+ default: 32
+ maxDocumentSize:
+ title: Max Document Size
+ description: Maximum size of one document in characters; unset means
no limit. The
+ pipeline holds a document in memory whole, so set the cap when the
source can
+ deliver oversized payloads. An oversized document fails the exchange
cleanly.
+ type: integer
+ documentSplitter:
+ title: Document Splitter
+ description: A DocumentSplitter bean replacing the default recursive
splitting, as a
+ `#bean:name` reference; maxSegmentSize and maxOverlapSize are then
ignored.
+ type: string
+ embeddingStore:
+ title: Embedding Store
+ description: The EmbeddingStore bean to write to, as a `#bean:name`
reference.
+ type: string
+ embeddingModel:
+ title: Embedding Model
+ description: The EmbeddingModel bean to embed with, as a `#bean:name`
reference.
+ type: string
+ idempotentRepository:
+ title: Idempotent Repository
+ description: An IdempotentRepository bean remembering ingested
document ids, as a
+ `#bean:name` reference. A duplicate delivery is answered skipped,
first write wins.
+ type: string
+ dependencies:
+ - "camel:core"
+ - "camel:langchain4j-ingest"
+ - "camel:kamelet"
+ template:
+ from:
+ uri: "kamelet:source"
+ steps:
+ # the endpoint itself resolves the id - the exchange property (set
pre-parse by the
+ # parser actions) wins over the header, so no re-capture happens here:
reading the
+ # header after a parse could hand a crafted document its own identity
+ - to:
+ uri: "langchain4j-ingest:{{pipelineName}}"
+ parameters:
+ documentIdHeader: "{{documentIdHeader}}"
+ maxSegmentSize: "{{maxSegmentSize}}"
+ maxOverlapSize: "{{maxOverlapSize}}"
+ embeddingBatchSize: "{{embeddingBatchSize}}"
+ maxDocumentSize: "{{?maxDocumentSize}}"
+ documentSplitter: "{{?documentSplitter}}"
+ embeddingStore: "{{?embeddingStore}}"
+ embeddingModel: "{{?embeddingModel}}"
+ idempotentRepository: "{{?idempotentRepository}}"
diff --git a/kamelets/tika-extract-text-action.kamelet.yaml
b/kamelets/tika-extract-text-action.kamelet.yaml
new file mode 100644
index 000000000..b3465fbb8
--- /dev/null
+++ b/kamelets/tika-extract-text-action.kamelet.yaml
@@ -0,0 +1,86 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: tika-extract-text-action
+ labels:
+ camel.apache.org/kamelet.type: "action"
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.22.1-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2NCA2NCI+PHJlY3QgeD0iOCIgeT0iNiIgd2lkdGg9IjI4IiBoZWlnaHQ9IjM2IiByeD0iMyIgZmlsbD0iIzhmYjRmZSIvPjxyZWN0IHg9IjEyIiB5PSIxMiIgd2lkdGg9IjIwIiBoZWlnaHQ9IjMiIGZpbGw9IiNmZmYiLz48cmVjdCB4PSIxMiIgeT0iMTkiIHdpZHRoPSIyMCIgaGVpZ2h0PSIzIiBmaWxsPSIjZmZmIi8+PHJlY3QgeD0iMTIiIHk9IjI2IiB3aWR0aD0iMTQiIGhlaWdodD0iMyIgZmlsbD0iI2ZmZiIvPjxjaXJjbGUgY3g9IjQ2IiBjeT0iMjAiIHI9IjQiIGZpbG
[...]
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.group: "Actions"
+ camel.apache.org/kamelet.namespace: "AI"
+spec:
+ definition:
+ title: "Tika Extract Text Action"
+ description: |-
+ Extract plain text from PDF, office and similar documents in-process
with Apache Tika.
+
+ Before the parse, the value of the `documentIdHeader` header is captured
into the
+ `CamelLangChain4jIngestDocumentId` exchange property - Tika copies
document metadata
+ over the headers, so a crafted document could otherwise forge its own
identity in steps
+ that read it later (the langchain4j-ingest sink does). The extracted
text is decoded at
+ a pinned UTF-8 (a document-injected charset header is swept before the
conversion), and
+ after the parse all non-Camel headers are removed: past this action,
parsed document
+ metadata is indistinguishable from caller headers, so it must not travel
further. Each
+ format needs its Tika parser module on the classpath; camel-tika ships
the HTML and
+ text modules, PDF needs org.apache.tika:tika-parser-pdf-module.
+ type: object
+ properties:
+ documentIdHeader:
+ title: Document Id Header
+ description: Name of the header carrying the stable document id,
captured into the
+ CamelLangChain4jIngestDocumentId exchange property before the parse.
+ type: string
+ default: CamelLangChain4jIngestDocumentId
+ dependencies:
+ - "camel:core"
+ - "camel:tika"
+ - "camel:kamelet"
+ template:
+ from:
+ uri: "kamelet:source"
+ steps:
+ # captured before the parse: Tika copies document metadata over the
headers, so a
+ # crafted document could otherwise forge its own identity
+ - setProperty:
+ name: "CamelLangChain4jIngestDocumentId"
+ simple: "${header[{{documentIdHeader}}]}"
+ # tika answers plain text (body only, blocks newline-separated) at the
pinned encoding
+ - to:
"tika:parse?tikaParseOutputFormat=text&tikaParseOutputEncoding=UTF-8"
+ # a message- or document-supplied charset header outranks the charset
the conversion
+ # below sets, so it is swept before the bytes are decoded at the
pinned UTF-8.
+ # These three steps are the declarative twin of the
camel-langchain4j-ingest
+ # component's TikaTextDecode processor, reimplemented here on purpose
so a generic
+ # parse action does not depend on that component - keep the two in step
+ - removeHeaders:
+ pattern: "CamelCharsetName"
+ - convertBodyTo:
+ type: "byte[]"
+ - convertBodyTo:
+ type: "java.lang.String"
+ charset: "UTF-8"
+ # past the parse, non-Camel headers are parsed document metadata
(Content-Type,
+ # dc:* and friends) - Camel-namespace names were already filtered by
camel-tika,
+ # the rest must not leak into later steps as if the caller had set them
+ - removeHeaders:
+ pattern: "*"
+ excludePattern: "Camel*"
diff --git
a/library/camel-kamelets/src/main/resources/kamelets/docling-convert-action.kamelet.yaml
b/library/camel-kamelets/src/main/resources/kamelets/docling-convert-action.kamelet.yaml
new file mode 100644
index 000000000..d1a00f80e
--- /dev/null
+++
b/library/camel-kamelets/src/main/resources/kamelets/docling-convert-action.kamelet.yaml
@@ -0,0 +1,86 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: docling-convert-action
+ labels:
+ camel.apache.org/kamelet.type: "action"
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.22.1-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2NCA2NCI+PHJlY3QgeD0iOCIgeT0iNiIgd2lkdGg9IjI4IiBoZWlnaHQ9IjM2IiByeD0iMyIgZmlsbD0iIzhmYjRmZSIvPjxyZWN0IHg9IjEyIiB5PSIxMiIgd2lkdGg9IjIwIiBoZWlnaHQ9IjMiIGZpbGw9IiNmZmYiLz48cmVjdCB4PSIxMiIgeT0iMTkiIHdpZHRoPSIyMCIgaGVpZ2h0PSIzIiBmaWxsPSIjZmZmIi8+PHJlY3QgeD0iMTIiIHk9IjI2IiB3aWR0aD0iMTQiIGhlaWdodD0iMyIgZmlsbD0iI2ZmZiIvPjxjaXJjbGUgY3g9IjQ2IiBjeT0iMjAiIHI9IjQiIGZpbG
[...]
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.group: "Actions"
+ camel.apache.org/kamelet.namespace: "AI"
+spec:
+ definition:
+ title: "Docling Convert Action"
+ description: |-
+ Convert PDF, office and similar documents to structure-preserving
markdown with
+ Docling, OCR included.
+
+ Before the conversion, the value of the `documentIdHeader` header is
captured into the
+ `CamelLangChain4jIngestDocumentId` exchange property, so a converted
document cannot
+ forge its own identity in steps that read it later (the
langchain4j-ingest sink does);
+ the body is pinned to bytes and the `CamelDocling*` control headers are
swept, because
+ none of them may be decided by a consumer-delivered payload. By default
the `docling`
+ component executes a local docling CLI; set `doclingServeUrl` to use a
Docling Serve
+ instance instead - the natural fit for self-contained Pipes.
+ type: object
+ properties:
+ documentIdHeader:
+ title: Document Id Header
+ description: Name of the header carrying the stable document id,
captured into the
+ CamelLangChain4jIngestDocumentId exchange property before the
conversion.
+ type: string
+ default: CamelLangChain4jIngestDocumentId
+ doclingServeUrl:
+ title: Docling Serve URL
+ description: Address of a Docling Serve instance to convert through,
such as
+ http://docling-serve:5001; when unset, the docling component's own
configuration
+ applies (a local docling CLI by default).
+ type: string
+ useDoclingServe:
+ title: Use Docling Serve
+ description: Whether to convert through a Docling Serve instance
instead of a local
+ docling CLI; set together with doclingServeUrl.
+ type: boolean
+ dependencies:
+ - "camel:core"
+ - "camel:docling"
+ - "camel:kamelet"
+ template:
+ from:
+ uri: "kamelet:source"
+ steps:
+ # captured before the conversion, so a document cannot forge its own
identity
+ - setProperty:
+ name: "CamelLangChain4jIngestDocumentId"
+ simple: "${header[{{documentIdHeader}}]}"
+ - convertBodyTo:
+ type: "byte[]"
+ - removeHeaders:
+ pattern: "CamelDocling*"
+ - to:
+ uri: "docling:convert"
+ parameters:
+ operation: "CONVERT_TO_MARKDOWN"
+ contentInBody: "true"
+ useDoclingServe: "{{?useDoclingServe}}"
+ doclingServeUrl: "{{?doclingServeUrl}}"
diff --git
a/library/camel-kamelets/src/main/resources/kamelets/langchain4j-ingest-file-source.kamelet.yaml
b/library/camel-kamelets/src/main/resources/kamelets/langchain4j-ingest-file-source.kamelet.yaml
new file mode 100644
index 000000000..5114b2e42
--- /dev/null
+++
b/library/camel-kamelets/src/main/resources/kamelets/langchain4j-ingest-file-source.kamelet.yaml
@@ -0,0 +1,101 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: langchain4j-ingest-file-source
+ labels:
+ camel.apache.org/kamelet.type: "source"
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.22.1-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2NCA2NCI+PHJlY3QgeD0iOCIgeT0iNiIgd2lkdGg9IjI4IiBoZWlnaHQ9IjM2IiByeD0iMyIgZmlsbD0iIzhmYjRmZSIvPjxyZWN0IHg9IjEyIiB5PSIxMiIgd2lkdGg9IjIwIiBoZWlnaHQ9IjMiIGZpbGw9IiNmZmYiLz48cmVjdCB4PSIxMiIgeT0iMTkiIHdpZHRoPSIyMCIgaGVpZ2h0PSIzIiBmaWxsPSIjZmZmIi8+PHJlY3QgeD0iMTIiIHk9IjI2IiB3aWR0aD0iMTQiIGhlaWdodD0iMyIgZmlsbD0iI2ZmZiIvPjxjaXJjbGUgY3g9IjQ2IiBjeT0iMjAiIHI9IjQiIGZpbG
[...]
+ camel.apache.org/kamelet.group: "LangChain4j Ingest"
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.namespace: "AI"
+spec:
+ definition:
+ title: "LangChain4j Ingest File Source"
+ description: |-
+ Watch a directory as the source of an ingestion pipeline, with the
defaults a knowledge
+ base needs: documents are left in place (noop), unchanged files are
remembered in a
+ duplicate register keyed on path, modification time and size, a file
still being copied
+ in is waited for (readLock=changed), and the file name is exposed as the
document id in
+ the CamelLangChain4jIngestDocumentId header.
+
+ Mind the two dedup layers when composing: this source re-delivers an
edited file (its
+ register key changes with the modification time), but a
langchain4j-ingest-sink with an
+ idempotentRepository is first-write-wins by document id - the
re-delivered edit would
+ be skipped. Use a version-aware id, or no sink repository, when
documents change.
+ required:
+ - directory
+ type: object
+ properties:
+ directory:
+ title: Directory
+ description: The directory to ingest documents from.
+ type: string
+ pattern: "[^?#]+"
+ recursive:
+ title: Recursive
+ description: Whether subdirectories are ingested too.
+ type: boolean
+ default: true
+ delay:
+ title: Delay
+ description: Milliseconds between directory polls; the file endpoint's
default is
+ 500ms, which is aggressive for a knowledge-base directory.
+ type: integer
+ charset:
+ title: Charset
+ description: Character set for reading text documents, for example
UTF-8. Leave
+ unset when the source feeds a parser action
(tika-extract-text-action or
+ docling-convert-action), which must receive the raw bytes - a charset
+ conversion would corrupt a binary document such as a PDF.
+ type: string
+ idempotentRepository:
+ title: Idempotent Repository
+ description: An IdempotentRepository bean remembering already consumed
files, as a
+ `#bean:name` reference; the register is keyed on path, modification
time and
+ size, so an edited file is consumed again. When unset, the file
endpoint's
+ default in-memory register (1000 entries) applies - eviction on a
large directory
+ re-ingests files, so supply a sized or persistent repository for a
real knowledge
+ base.
+ type: string
+ dependencies:
+ - "camel:core"
+ - "camel:file"
+ - "camel:kamelet"
+ template:
+ from:
+ uri: "file:{{directory}}"
+ parameters:
+ noop: "true"
+ idempotent: "true"
+ # an edited file gets a new key and re-ingests
+ idempotentKey: "${file:absolute.path}:${file:modified}:${file:size}"
+ recursive: "{{recursive}}"
+ readLock: "changed"
+ delay: "{{?delay}}"
+ charset: "{{?charset}}"
+ idempotentRepository: "{{?idempotentRepository}}"
+ steps:
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ simple: "${header.CamelFileName}"
+ - to: "kamelet:sink"
diff --git
a/library/camel-kamelets/src/main/resources/kamelets/langchain4j-ingest-sink.kamelet.yaml
b/library/camel-kamelets/src/main/resources/kamelets/langchain4j-ingest-sink.kamelet.yaml
new file mode 100644
index 000000000..768c3e1c4
--- /dev/null
+++
b/library/camel-kamelets/src/main/resources/kamelets/langchain4j-ingest-sink.kamelet.yaml
@@ -0,0 +1,123 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: langchain4j-ingest-sink
+ labels:
+ camel.apache.org/kamelet.type: "sink"
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.22.1-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2NCA2NCI+PHJlY3QgeD0iOCIgeT0iNiIgd2lkdGg9IjI4IiBoZWlnaHQ9IjM2IiByeD0iMyIgZmlsbD0iIzhmYjRmZSIvPjxyZWN0IHg9IjEyIiB5PSIxMiIgd2lkdGg9IjIwIiBoZWlnaHQ9IjMiIGZpbGw9IiNmZmYiLz48cmVjdCB4PSIxMiIgeT0iMTkiIHdpZHRoPSIyMCIgaGVpZ2h0PSIzIiBmaWxsPSIjZmZmIi8+PHJlY3QgeD0iMTIiIHk9IjI2IiB3aWR0aD0iMTQiIGhlaWdodD0iMyIgZmlsbD0iI2ZmZiIvPjxjaXJjbGUgY3g9IjQ2IiBjeT0iMjAiIHI9IjQiIGZpbG
[...]
+ camel.apache.org/kamelet.group: "LangChain4j Ingest"
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.namespace: "AI"
+spec:
+ definition:
+ title: "LangChain4j Ingest Sink"
+ description: |-
+ Ingest documents into a LangChain4j EmbeddingStore: the payload is split
into overlapping
+ segments, embedded in batches and written to the store, each segment
stamped with the
+ pipeline name and the document id.
+
+ The document id is read from the `CamelLangChain4jIngestDocumentId`
exchange property
+ when a prior step captured it (the parser actions do, before the parse),
else from the
+ header named by `documentIdHeader` - the property always wins, so a
parsed document
+ cannot forge its own identity. Bean references are `#bean:name` values;
when
+ `embeddingStore` or `embeddingModel` is not set, the single registry
bean of that type is
+ used. With `idempotentRepository` set, the first write per document id
wins - a
+ re-delivered edited document is skipped, so streams that carry updates
need a
+ version-aware id.
+ type: object
+ properties:
+ pipelineName:
+ title: Pipeline Name
+ description: The pipeline name, stamped on every written segment.
+ type: string
+ pattern: "[A-Za-z0-9._-]+"
+ default: ingest
+ documentIdHeader:
+ title: Document Id Header
+ description: Name of the header carrying the stable document id, such
as CamelAwsS3Key
+ for an S3 source or CamelKafkaKey for a Kafka one; the
+ CamelLangChain4jIngestDocumentId exchange property, when set, takes
precedence.
+ type: string
+ default: CamelLangChain4jIngestDocumentId
+ maxSegmentSize:
+ title: Max Segment Size
+ description: Maximum size of one segment, in characters.
+ type: integer
+ default: 500
+ maxOverlapSize:
+ title: Max Overlap Size
+ description: How much of the previous segment each segment repeats, in
characters.
+ type: integer
+ default: 50
+ embeddingBatchSize:
+ title: Embedding Batch Size
+ description: How many segments are embedded per request to the
embedding model.
+ type: integer
+ default: 32
+ maxDocumentSize:
+ title: Max Document Size
+ description: Maximum size of one document in characters; unset means
no limit. The
+ pipeline holds a document in memory whole, so set the cap when the
source can
+ deliver oversized payloads. An oversized document fails the exchange
cleanly.
+ type: integer
+ documentSplitter:
+ title: Document Splitter
+ description: A DocumentSplitter bean replacing the default recursive
splitting, as a
+ `#bean:name` reference; maxSegmentSize and maxOverlapSize are then
ignored.
+ type: string
+ embeddingStore:
+ title: Embedding Store
+ description: The EmbeddingStore bean to write to, as a `#bean:name`
reference.
+ type: string
+ embeddingModel:
+ title: Embedding Model
+ description: The EmbeddingModel bean to embed with, as a `#bean:name`
reference.
+ type: string
+ idempotentRepository:
+ title: Idempotent Repository
+ description: An IdempotentRepository bean remembering ingested
document ids, as a
+ `#bean:name` reference. A duplicate delivery is answered skipped,
first write wins.
+ type: string
+ dependencies:
+ - "camel:core"
+ - "camel:langchain4j-ingest"
+ - "camel:kamelet"
+ template:
+ from:
+ uri: "kamelet:source"
+ steps:
+ # the endpoint itself resolves the id - the exchange property (set
pre-parse by the
+ # parser actions) wins over the header, so no re-capture happens here:
reading the
+ # header after a parse could hand a crafted document its own identity
+ - to:
+ uri: "langchain4j-ingest:{{pipelineName}}"
+ parameters:
+ documentIdHeader: "{{documentIdHeader}}"
+ maxSegmentSize: "{{maxSegmentSize}}"
+ maxOverlapSize: "{{maxOverlapSize}}"
+ embeddingBatchSize: "{{embeddingBatchSize}}"
+ maxDocumentSize: "{{?maxDocumentSize}}"
+ documentSplitter: "{{?documentSplitter}}"
+ embeddingStore: "{{?embeddingStore}}"
+ embeddingModel: "{{?embeddingModel}}"
+ idempotentRepository: "{{?idempotentRepository}}"
diff --git
a/library/camel-kamelets/src/main/resources/kamelets/tika-extract-text-action.kamelet.yaml
b/library/camel-kamelets/src/main/resources/kamelets/tika-extract-text-action.kamelet.yaml
new file mode 100644
index 000000000..b3465fbb8
--- /dev/null
+++
b/library/camel-kamelets/src/main/resources/kamelets/tika-extract-text-action.kamelet.yaml
@@ -0,0 +1,86 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: tika-extract-text-action
+ labels:
+ camel.apache.org/kamelet.type: "action"
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.22.1-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2NCA2NCI+PHJlY3QgeD0iOCIgeT0iNiIgd2lkdGg9IjI4IiBoZWlnaHQ9IjM2IiByeD0iMyIgZmlsbD0iIzhmYjRmZSIvPjxyZWN0IHg9IjEyIiB5PSIxMiIgd2lkdGg9IjIwIiBoZWlnaHQ9IjMiIGZpbGw9IiNmZmYiLz48cmVjdCB4PSIxMiIgeT0iMTkiIHdpZHRoPSIyMCIgaGVpZ2h0PSIzIiBmaWxsPSIjZmZmIi8+PHJlY3QgeD0iMTIiIHk9IjI2IiB3aWR0aD0iMTQiIGhlaWdodD0iMyIgZmlsbD0iI2ZmZiIvPjxjaXJjbGUgY3g9IjQ2IiBjeT0iMjAiIHI9IjQiIGZpbG
[...]
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.group: "Actions"
+ camel.apache.org/kamelet.namespace: "AI"
+spec:
+ definition:
+ title: "Tika Extract Text Action"
+ description: |-
+ Extract plain text from PDF, office and similar documents in-process
with Apache Tika.
+
+ Before the parse, the value of the `documentIdHeader` header is captured
into the
+ `CamelLangChain4jIngestDocumentId` exchange property - Tika copies
document metadata
+ over the headers, so a crafted document could otherwise forge its own
identity in steps
+ that read it later (the langchain4j-ingest sink does). The extracted
text is decoded at
+ a pinned UTF-8 (a document-injected charset header is swept before the
conversion), and
+ after the parse all non-Camel headers are removed: past this action,
parsed document
+ metadata is indistinguishable from caller headers, so it must not travel
further. Each
+ format needs its Tika parser module on the classpath; camel-tika ships
the HTML and
+ text modules, PDF needs org.apache.tika:tika-parser-pdf-module.
+ type: object
+ properties:
+ documentIdHeader:
+ title: Document Id Header
+ description: Name of the header carrying the stable document id,
captured into the
+ CamelLangChain4jIngestDocumentId exchange property before the parse.
+ type: string
+ default: CamelLangChain4jIngestDocumentId
+ dependencies:
+ - "camel:core"
+ - "camel:tika"
+ - "camel:kamelet"
+ template:
+ from:
+ uri: "kamelet:source"
+ steps:
+ # captured before the parse: Tika copies document metadata over the
headers, so a
+ # crafted document could otherwise forge its own identity
+ - setProperty:
+ name: "CamelLangChain4jIngestDocumentId"
+ simple: "${header[{{documentIdHeader}}]}"
+ # tika answers plain text (body only, blocks newline-separated) at the
pinned encoding
+ - to:
"tika:parse?tikaParseOutputFormat=text&tikaParseOutputEncoding=UTF-8"
+ # a message- or document-supplied charset header outranks the charset
the conversion
+ # below sets, so it is swept before the bytes are decoded at the
pinned UTF-8.
+ # These three steps are the declarative twin of the
camel-langchain4j-ingest
+ # component's TikaTextDecode processor, reimplemented here on purpose
so a generic
+ # parse action does not depend on that component - keep the two in step
+ - removeHeaders:
+ pattern: "CamelCharsetName"
+ - convertBodyTo:
+ type: "byte[]"
+ - convertBodyTo:
+ type: "java.lang.String"
+ charset: "UTF-8"
+ # past the parse, non-Camel headers are parsed document metadata
(Content-Type,
+ # dc:* and friends) - Camel-namespace names were already filtered by
camel-tika,
+ # the rest must not leak into later steps as if the caller had set them
+ - removeHeaders:
+ pattern: "*"
+ excludePattern: "Camel*"
diff --git a/tests/camel-kamelets-itest/src/test/java/CommonIT.java
b/tests/camel-kamelets-itest/src/test/java/CommonIT.java
index 04124894b..2e14332e9 100644
--- a/tests/camel-kamelets-itest/src/test/java/CommonIT.java
+++ b/tests/camel-kamelets-itest/src/test/java/CommonIT.java
@@ -94,6 +94,14 @@ public class CommonIT {
return
CitrusTestFactorySupport.factory(TestLoader.YAML).packageScan("http");
}
+ @CitrusTestFactory
+ public Stream<DynamicTest> langchain4jIngest() {
+ // these tests need the camel-langchain4j-ingest component (Camel
4.23+) and fail on
+ // an older Camel BY DESIGN: the red build is the guard that keeps the
Kamelets from
+ // being merged into a catalog line whose Camel cannot run them
+ return
CitrusTestFactorySupport.factory(TestLoader.YAML).packageScan("langchain4j-ingest");
+ }
+
@CitrusTestFactory
public Stream<DynamicTest> log() {
return
CitrusTestFactorySupport.factory(TestLoader.YAML).packageScan("log");
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/docling-convert-route.citrus.it.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/docling-convert-route.citrus.it.yaml
new file mode 100644
index 000000000..e44dbfb91
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/docling-convert-route.citrus.it.yaml
@@ -0,0 +1,89 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+name: docling-convert-route-test
+actions:
+ # drain any request a previous test may have left on the shared server
+ - purge:
+ endpoints:
+ - name: "httpServer"
+ timeout: 100
+ - createVariables:
+ variables:
+ - name: "docling.serve.url"
+ value: "http://localhost:${http.server.port}";
+ - name: "report.url"
+ value: "http://localhost:${http.server.port}/result/docling";
+
+ # Run a route feeding one document through the docling-convert-action into
the
+ # sink Kamelet, with the docling component pointed at the Citrus HTTP
server; the
+ # route reports the IngestResult back to the same server afterwards
+ - camel:
+ cli:
+ run:
+ waitForRunningState: false
+ args:
+ - "--dep=camel:groovy,camel:langchain4j-ingest"
+ - "--max-seconds=60"
+ integration:
+ file: "langchain4j-ingest/docling-convert-route.yaml"
+ systemProperties:
+ properties:
+ - name: "docling.serve.url"
+ value: "${docling.serve.url}"
+ - name: "report.url"
+ value: "${report.url}"
+
+ # The conversion request reaches the stub
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/v1/convert/source"
+
+ # Answer the shape of a real docling-serve convert response
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+ headers:
+ - name: "Content-Type"
+ value: "application/json"
+ body:
+ data: |
+ {"document":{"filename":"scan.pdf","md_content":"The EPSILON-2
valve seals at 80
bar.","json_content":null,"html_content":null,"text_content":null,"doctags_content":null},"status":"success","errors":[],"processing_time":0.01,"timings":{}}
+
+ # The reported IngestResult proves the stubbed markdown was embedded and
stored
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/docling"
+ type: "plaintext"
+ body:
+ data: "@contains('documentId=scan.pdf, segmentsWritten=1,
outcome=INGESTED')@"
+
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/docling-convert-route.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/docling-convert-route.yaml
new file mode 100644
index 000000000..bd99c155b
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/docling-convert-route.yaml
@@ -0,0 +1,61 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+# The docling action feeding the sink, against the test's stubbed Docling
Serve instance (the real
+# image is a multi-gigabyte download): the action's own serve parameters point
the
+# Kamelet at the Citrus HTTP server, which answers markdown; an INGESTED
+# outcome proves the converted markdown was split, embedded and stored.
+- beans:
+ - name: store
+ type: dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore
+ # a deterministic in-process stub model - the test asserts the conversion
+ # and pipeline mechanics, not embedding quality
+ - name: model
+ type: dev.langchain4j.model.embedding.EmbeddingModel
+ scriptLanguage: groovy
+ script: |
+ def embed = { String text ->
+ def random = new java.util.Random(text.hashCode())
+ float[] vector = new float[8]
+ for (int i = 0; i < 8; i++) {
+ vector[i] = random.nextFloat()
+ }
+ return dev.langchain4j.data.embedding.Embedding.from(vector)
+ }
+ return [
+ embedAll: { segments ->
+ dev.langchain4j.model.output.Response.from(segments.collect {
embed(it.text()) })
+ }
+ ] as dev.langchain4j.model.embedding.EmbeddingModel
+- route:
+ id: docling-convert-route
+ from:
+ uri: "timer:ingest"
+ parameters:
+ repeatCount: 1
+ steps:
+ - setBody:
+ constant: "opaque binary payload, the stub answers regardless"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "scan.pdf"
+ - to:
"kamelet:docling-convert-action?useDoclingServe=true&doclingServeUrl={{docling.serve.url}}"
+ - to:
"kamelet:langchain4j-ingest-sink?embeddingStore=#bean:store&embeddingModel=#bean:model"
+ - log: "INGEST ${body}"
+ - convertBodyTo:
+ type: "java.lang.String"
+ - to: "{{report.url}}"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-register-route.citrus.it.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-register-route.citrus.it.yaml
new file mode 100644
index 000000000..af50ed9ff
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-register-route.citrus.it.yaml
@@ -0,0 +1,84 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+name: langchain4j-ingest-file-register-route-test
+actions:
+ # drain any request a previous test may have left on the shared server
+ - purge:
+ endpoints:
+ - name: "httpServer"
+ timeout: 100
+ - createVariables:
+ variables:
+ - name: "report.url"
+ value: "http://localhost:${http.server.port}/result/file-register";
+ - name: "ingest.dir"
+ value: "ingest-reg-citrus:randomNumber(5)"
+
+ # Run the register-semantics scenario: v1 is seeded, then overwritten by v2
+ - camel:
+ cli:
+ run:
+ waitForRunningState: false
+ args:
+ - "--max-seconds=60"
+ integration:
+ file:
"langchain4j-ingest/langchain4j-ingest-file-register-route.yaml"
+ systemProperties:
+ properties:
+ - name: "report.url"
+ value: "${report.url}"
+ - name: "ingest.dir"
+ value: "${ingest.dir}"
+
+ # First delivery: the original file, consumed exactly once
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/file-register"
+ type: "plaintext"
+ body:
+ data: "@contains('version one of the document')@"
+
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # Second delivery: the EDITED file is re-delivered (register key includes the
+ # modification time); an unchanged-file re-delivery in between would arrive
+ # here instead and fail the assertion
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/file-register"
+ type: "plaintext"
+ body:
+ data: "@contains('version two, edited in place')@"
+
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-register-route.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-register-route.yaml
new file mode 100644
index 000000000..b136d34da
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-register-route.yaml
@@ -0,0 +1,57 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+# The file source's register semantics: the duplicate register is keyed on
path,
+# modification time and size, so the SAME file is consumed once per version and
+# an EDITED file is re-delivered. A first seed writes v1, a second seed
+# overwrites it with v2 three seconds later - the test receives exactly v1 then
+# v2 (an unchanged-file re-delivery in between would break the v2 assertion).
+# The register is a named bean and the poll delay is set, exercising the
+# idempotentRepository and delay parameters end to end.
+- beans:
+ - name: register
+ type:
org.apache.camel.support.processor.idempotent.MemoryIdempotentRepository
+- route:
+ id: seed-v1
+ from:
+ uri: "timer:seed-v1"
+ parameters:
+ repeatCount: 1
+ steps:
+ - setBody:
+ constant: "version one of the document"
+ - to: "file:{{ingest.dir}}?fileName=doc.txt"
+- route:
+ id: seed-v2
+ from:
+ uri: "timer:seed-v2"
+ parameters:
+ repeatCount: 1
+ delay: 3000
+ steps:
+ - setBody:
+ constant: "version two, edited in place"
+ - to: "file:{{ingest.dir}}?fileName=doc.txt"
+- route:
+ id: langchain4j-ingest-file-register-route
+ from:
+ uri:
"kamelet:langchain4j-ingest-file-source?directory={{ingest.dir}}&charset=UTF-8&idempotentRepository=#bean:register&delay=200"
+ steps:
+ - convertBodyTo:
+ type: "java.lang.String"
+ - log: "FILE-REGISTER ${body}"
+ - to: "{{report.url}}"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-source-route.citrus.it.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-source-route.citrus.it.yaml
new file mode 100644
index 000000000..1eea405c2
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-source-route.citrus.it.yaml
@@ -0,0 +1,72 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+name: langchain4j-ingest-file-source-route-test
+actions:
+ # drain any request a previous test may have left on the shared server
+ - purge:
+ endpoints:
+ - name: "httpServer"
+ timeout: 100
+ - createVariables:
+ variables:
+ - name: "report.url"
+ value: "http://localhost:${http.server.port}/result/file-source";
+ # a fresh directory per test run: a still-terminating integration from
a
+ # previous local run must not re-consume this run's seed file
+ - name: "ingest.dir"
+ value: "ingest-in-citrus:randomNumber(5)"
+
+ # Run a route seeding one file into the directory watched by the
+ # langchain4j-ingest-file-source Kamelet; the source's output is reported
+ # back to the test's HTTP server
+ - camel:
+ cli:
+ run:
+ waitForRunningState: false
+ args:
+ - "--max-seconds=60"
+ integration:
+ file:
"langchain4j-ingest/langchain4j-ingest-file-source-route.yaml"
+ systemProperties:
+ properties:
+ - name: "report.url"
+ value: "${report.url}"
+ - name: "ingest.dir"
+ value: "${ingest.dir}"
+
+ # The file content flows out of the source together with the document id
+ # header the ingest sinks read
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/file-source"
+ headers:
+ - name: "docId"
+ value: "doc.txt"
+ type: "plaintext"
+ body:
+ data: "@contains('Sphinx of black quartz, judge my vow')@"
+
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-source-route.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-source-route.yaml
new file mode 100644
index 000000000..9c6544fed
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-source-route.yaml
@@ -0,0 +1,42 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+# The file source: a seed route drops one file into the watched directory, the
+# Kamelet picks it up, leaves it in place and stamps the document id into the
+# CamelLangChain4jIngestDocumentId header - the contract the ingest sinks read.
+- route:
+ id: seed-file
+ from:
+ uri: "timer:seed"
+ parameters:
+ repeatCount: 1
+ steps:
+ - setBody:
+ constant: "Sphinx of black quartz, judge my vow"
+ - to: "file:{{ingest.dir}}?fileName=doc.txt"
+- route:
+ id: langchain4j-ingest-file-source-route
+ from:
+ uri:
"kamelet:langchain4j-ingest-file-source?directory={{ingest.dir}}&charset=UTF-8"
+ steps:
+ - convertBodyTo:
+ type: "java.lang.String"
+ - log: "FILE-SOURCE ${body}
id=${header.CamelLangChain4jIngestDocumentId}"
+ - setHeader:
+ name: "docId"
+ simple: "${header.CamelLangChain4jIngestDocumentId}"
+ - to: "{{report.url}}"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-tika-route.citrus.it.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-tika-route.citrus.it.yaml
new file mode 100644
index 000000000..93734ccc9
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-tika-route.citrus.it.yaml
@@ -0,0 +1,65 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+name: langchain4j-ingest-file-tika-route-test
+actions:
+ # drain any request a previous test may have left on the shared server
+ - purge:
+ endpoints:
+ - name: "httpServer"
+ timeout: 100
+ - createVariables:
+ variables:
+ - name: "report.url"
+ value: "http://localhost:${http.server.port}/result/file-tika";
+
+ # Run the file-source -> tika-action -> sink composition over a binary PDF;
the
+ # PDF parser module is not part of camel-tika, so it is added to the run -
the
+ # per-format module story the component docs describe. Its version tracks the
+ # Tika of the Camel release (4.0.0 for Camel 4.23): bump the two together
+ - camel:
+ cli:
+ run:
+ waitForRunningState: false
+ args:
+ -
"--dep=camel:groovy,camel:langchain4j-ingest,org.apache.tika:tika-parser-pdf-module:4.0.0"
+ - "--max-seconds=60"
+ integration:
+ file: "langchain4j-ingest/langchain4j-ingest-file-tika-route.yaml"
+ systemProperties:
+ properties:
+ - name: "report.url"
+ value: "${report.url}"
+
+ # INGESTED (not EMPTY) proves the PDF bytes reached Tika uncorrupted - with a
+ # charset conversion in the source this parse would find no text
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/file-tika"
+ type: "plaintext"
+ body:
+ data: "@contains('documentId=pump.pdf, segmentsWritten=1,
outcome=INGESTED')@"
+
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-tika-route.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-tika-route.yaml
new file mode 100644
index 000000000..cd357bc82
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-file-tika-route.yaml
@@ -0,0 +1,58 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+# The composition the Kamelets are designed for: the file source feeds the tika
+# sink with a BINARY document (a checked-in one-page PDF). The source leaves
+# charset unset on purpose, so the sink receives raw bytes - a charset
+# round-trip would corrupt the PDF before Tika ever parses it, and the INGESTED
+# outcome (not EMPTY) proves the bytes arrived intact.
+- beans:
+ - name: store
+ type: dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore
+ # a deterministic in-process stub model - the test asserts the composition
+ # and parse mechanics, not embedding quality
+ - name: model
+ type: dev.langchain4j.model.embedding.EmbeddingModel
+ scriptLanguage: groovy
+ script: |
+ def embed = { String text ->
+ def random = new java.util.Random(text.hashCode())
+ float[] vector = new float[8]
+ for (int i = 0; i < 8; i++) {
+ vector[i] = random.nextFloat()
+ }
+ return dev.langchain4j.data.embedding.Embedding.from(vector)
+ }
+ return [
+ embedAll: { segments ->
+ dev.langchain4j.model.output.Response.from(segments.collect {
embed(it.text()) })
+ }
+ ] as dev.langchain4j.model.embedding.EmbeddingModel
+- route:
+ id: langchain4j-ingest-file-tika-route
+ from:
+ uri: "kamelet:langchain4j-ingest-file-source?directory=pdfs"
+ steps:
+ - to: "kamelet:tika-extract-text-action"
+ - to:
"kamelet:langchain4j-ingest-sink?embeddingStore=#bean:store&embeddingModel=#bean:model"
+ - log: "INGEST ${body}"
+ - convertBodyTo:
+ type: "java.lang.String"
+ # no Content-Type cleanup needed here: the tika action itself sweeps
the parsed
+ # document's non-Camel headers (this report would arrive as
application/pdf otherwise,
+ # so this test doubles as the proof of that sweep)
+ - to: "{{report.url}}"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-sink-route.citrus.it.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-sink-route.citrus.it.yaml
new file mode 100644
index 000000000..962d80bb6
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-sink-route.citrus.it.yaml
@@ -0,0 +1,62 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+name: langchain4j-ingest-sink-route-test
+actions:
+ # drain any request a previous test may have left on the shared server
+ - purge:
+ endpoints:
+ - name: "httpServer"
+ timeout: 100
+ - createVariables:
+ variables:
+ - name: "report.url"
+ value: "http://localhost:${http.server.port}/result/sink";
+
+ # Run a route feeding one document into the langchain4j-ingest-sink Kamelet;
+ # the route reports the IngestResult back to the test's HTTP server
+ - camel:
+ cli:
+ run:
+ waitForRunningState: false
+ args:
+ - "--dep=camel:groovy,camel:langchain4j-ingest"
+ - "--max-seconds=60"
+ integration:
+ file: "langchain4j-ingest/langchain4j-ingest-sink-route.yaml"
+ systemProperties:
+ properties:
+ - name: "report.url"
+ value: "${report.url}"
+
+ # The reported IngestResult proves the document was split, embedded and
stored
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/sink"
+ type: "plaintext"
+ body:
+ data: "@contains('documentId=camels.txt, segmentsWritten=1,
outcome=INGESTED')@"
+
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-sink-route.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-sink-route.yaml
new file mode 100644
index 000000000..c1f1f40a9
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/langchain4j-ingest-sink-route.yaml
@@ -0,0 +1,58 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+# The plain sink: the payload is split, embedded and written to an in-memory
+# store; the IngestResult reply is logged for the test to verify. The model is
+# a deterministic in-process stub - the test asserts the pipeline mechanics,
+# not embedding quality, and needs no ONNX model download.
+- beans:
+ - name: store
+ type: dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore
+ - name: model
+ type: dev.langchain4j.model.embedding.EmbeddingModel
+ scriptLanguage: groovy
+ script: |
+ def embed = { String text ->
+ def random = new java.util.Random(text.hashCode())
+ float[] vector = new float[8]
+ for (int i = 0; i < 8; i++) {
+ vector[i] = random.nextFloat()
+ }
+ return dev.langchain4j.data.embedding.Embedding.from(vector)
+ }
+ return [
+ embedAll: { segments ->
+ dev.langchain4j.model.output.Response.from(segments.collect {
embed(it.text()) })
+ }
+ ] as dev.langchain4j.model.embedding.EmbeddingModel
+- route:
+ id: langchain4j-ingest-sink-route
+ from:
+ uri: "timer:ingest"
+ parameters:
+ repeatCount: 1
+ steps:
+ - setBody:
+ constant: "Camels are resilient desert animals. They store fat in
their humps."
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "camels.txt"
+ - to:
"kamelet:langchain4j-ingest-sink?embeddingStore=#bean:store&embeddingModel=#bean:model"
+ - log: "INGEST ${body}"
+ - convertBodyTo:
+ type: "java.lang.String"
+ - to: "{{report.url}}"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/pdfs/pump.pdf
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/pdfs/pump.pdf
new file mode 100644
index 000000000..110d9e609
Binary files /dev/null and
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/pdfs/pump.pdf
differ
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/tika-extract-text-route.citrus.it.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/tika-extract-text-route.citrus.it.yaml
new file mode 100644
index 000000000..a68d554d7
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/tika-extract-text-route.citrus.it.yaml
@@ -0,0 +1,62 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+name: tika-extract-text-route-test
+actions:
+ # drain any request a previous test may have left on the shared server
+ - purge:
+ endpoints:
+ - name: "httpServer"
+ timeout: 100
+ - createVariables:
+ variables:
+ - name: "report.url"
+ value: "http://localhost:${http.server.port}/result/tika";
+
+ # Run a route feeding one HTML document through the tika-extract-text-action
+ # into the sink Kamelet; the route reports the IngestResult back to the
test's HTTP server
+ - camel:
+ cli:
+ run:
+ waitForRunningState: false
+ args:
+ - "--dep=camel:groovy,camel:langchain4j-ingest"
+ - "--max-seconds=60"
+ integration:
+ file: "langchain4j-ingest/tika-extract-text-route.yaml"
+ systemProperties:
+ properties:
+ - name: "report.url"
+ value: "${report.url}"
+
+ # INGESTED (not EMPTY) proves Tika extracted text from the HTML before the
split
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/tika"
+ type: "plaintext"
+ body:
+ data: "@contains('documentId=notice.html, segmentsWritten=1,
outcome=INGESTED')@"
+
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/tika-extract-text-route.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/tika-extract-text-route.yaml
new file mode 100644
index 000000000..d75aa182a
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/tika-extract-text-route.yaml
@@ -0,0 +1,60 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+# The tika action feeding the sink: an HTML document is parsed to plain text by
+# Apache Tika in the action, then split and embedded by the sink. An INGESTED
outcome (not EMPTY)
+# proves the parse produced text; a failed parse would answer outcome=EMPTY.
+- beans:
+ - name: store
+ type: dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore
+ # a deterministic in-process stub model - the test asserts the parse and
+ # pipeline mechanics, not embedding quality
+ - name: model
+ type: dev.langchain4j.model.embedding.EmbeddingModel
+ scriptLanguage: groovy
+ script: |
+ def embed = { String text ->
+ def random = new java.util.Random(text.hashCode())
+ float[] vector = new float[8]
+ for (int i = 0; i < 8; i++) {
+ vector[i] = random.nextFloat()
+ }
+ return dev.langchain4j.data.embedding.Embedding.from(vector)
+ }
+ return [
+ embedAll: { segments ->
+ dev.langchain4j.model.output.Response.from(segments.collect {
embed(it.text()) })
+ }
+ ] as dev.langchain4j.model.embedding.EmbeddingModel
+- route:
+ id: tika-extract-text-route
+ from:
+ uri: "timer:ingest"
+ parameters:
+ repeatCount: 1
+ steps:
+ - setBody:
+ constant: "<html><head><title>Secret</title></head><body><p>First
paragraph about camels.</p></body></html>"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "notice.html"
+ - to: "kamelet:tika-extract-text-action"
+ - to:
"kamelet:langchain4j-ingest-sink?embeddingStore=#bean:store&embeddingModel=#bean:model"
+ - log: "INGEST ${body}"
+ - convertBodyTo:
+ type: "java.lang.String"
+ - to: "{{report.url}}"
