This is an automated email from the ASF dual-hosted git repository.
oscerd pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
The following commit(s) were added to refs/heads/main by this push:
new ede9d0499 Fix #3121: Parser actions: check maxDocumentSize without a
full read (#3122)
ede9d0499 is described below
commit ede9d049952647debf07c796ea016bf57ef346ff
Author: Jiří Ondrušek <[email protected]>
AuthorDate: Fri Oct 30 15:09:01 2026 +0100
Fix #3121: Parser actions: check maxDocumentSize without a full read (#3122)
tika-extract-text-action and docling-convert-action measured the
payload with length(), which reads a file or uncached stream to its
end and counts in an int.
- A polled file is sized by the CamelFileLength header its consumer
set, without a read. The header is believed only next to that file
(a WrappedFile body); on any other body it may be stale or forged,
as #3086 assumed. Without it, length() still measures the file.
- A plain java.io.File (file-watch) and a cache spooled to disk, the
only one past an int, are sized by their long length(). File sizes
compare as long, so past 2 GiB they no longer wrap and pass.
- An uncached stream is read at most maxDocumentSize + 1 bytes,
clamped to the int readNBytes takes, then closed; the extra byte
rejects it, otherwise the bytes read are the document.
- That stream is held in an exchange property, not a variable. With
caching on, a variable is stream-cached without the exchange, which
fails with StreamCacheException on exchanges that turn caching off
for themselves (netty-http, platform-http-vertx proxy, A2A).
- The error message no longer re-runs length(); a capped stream
reports "more than N bytes".
- camel:bean is declared: the read, the close and the spooled size
are simple OGNL calls.
Citrus tests: a forged header without a file, polled and plain files
and spooled caches under the cap and past 4 GiB, a stream under a cap
past the int range, uncached streams with caching on but off for the
exchange, and Tika and Docling streams rejected with the rest left
unread and the stream closed.
Fixes #3121
Co-authored-by: jiriOndrusek-agent <[email protected]>
Co-authored-by: Claude Sonnet 5.5 <[email protected]>
---
.../camel-kamelets-catalog-sbom.json | 4 +-
kamelets/docling-convert-action.kamelet.yaml | 61 +++++-
kamelets/tika-extract-text-action.kamelet.yaml | 61 +++++-
.../kamelets/docling-convert-action.kamelet.yaml | 61 +++++-
.../kamelets/tika-extract-text-action.kamelet.yaml | 61 +++++-
.../parser-max-size-route.citrus.it.yaml | 174 ++++++++++++++-
.../langchain4j-ingest/parser-max-size-route.yaml | 242 ++++++++++++++++++++-
.../parser-max-size-stream-route.citrus.it.yaml | 77 ++++++-
.../parser-max-size-stream-route.yaml | 115 +++++++++-
9 files changed, 817 insertions(+), 39 deletions(-)
diff --git a/camel-kamelets-sbom/camel-kamelets-catalog-sbom.json
b/camel-kamelets-sbom/camel-kamelets-catalog-sbom.json
index 29f83c377..185abd0cf 100644
--- a/camel-kamelets-sbom/camel-kamelets-catalog-sbom.json
+++ b/camel-kamelets-sbom/camel-kamelets-catalog-sbom.json
@@ -2899,7 +2899,7 @@
"dependsOn" : [ "pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]" ]
}, {
"ref" : "kamelet:docling-convert-action",
- "dependsOn" : [ "pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]" ]
+ "dependsOn" : [ "pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]" ]
}, {
"ref" : "kamelet:drop-field-action",
"dependsOn" : [ "pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]" ]
@@ -3412,7 +3412,7 @@
"dependsOn" : [ "pkg:maven/org.apache.camel/[email protected]" ]
}, {
"ref" : "kamelet:tika-extract-text-action",
- "dependsOn" : [ "pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]" ]
+ "dependsOn" : [ "pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]" ]
}, {
"ref" : "kamelet:timer-source",
"dependsOn" : [ "pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]" ]
diff --git a/kamelets/docling-convert-action.kamelet.yaml
b/kamelets/docling-convert-action.kamelet.yaml
index ac2b15292..e1c82e962 100644
--- a/kamelets/docling-convert-action.kamelet.yaml
+++ b/kamelets/docling-convert-action.kamelet.yaml
@@ -72,6 +72,7 @@ spec:
dependencies:
- "camel:docling"
- "camel:kamelet"
+ - "camel:bean"
template:
from:
uri: kamelet:source
@@ -86,7 +87,7 @@ spec:
# the document, so an oversized one is rejected first. The
precondition is evaluated
# once, when the Kamelet is instantiated - only then does
${properties:maxDocumentSize:0}
# see the parameter, at runtime it resolves to 0 - so the check must
stay a
- # precondition, and the comparison below uses {{maxDocumentSize}}
+ # precondition, and the comparisons below use {{maxDocumentSize}}
- choice:
precondition: true
when:
@@ -94,26 +95,72 @@ spec:
simple:
expression: "${properties:maxDocumentSize:0} > 0"
steps:
- # length() reads a stream to its end and closes it, so a
stream body that
- # stream caching (on by default) left uncached is read into
bytes first; files,
- # byte arrays and cached streams are measured in place,
without a copy
+ # length() would read a stream to its end, so one that
stream caching (on by
+ # default) left uncached is read at most one byte past the
cap, within the int
+ # readNBytes takes, and closed: that byte rejects it below,
else the bytes are
+ # the document. The stream is held in a property: a variable
is stream-cached,
+ # which fails on an exchange that turned caching off for
itself
- choice:
when:
- expression:
simple:
expression: "${body} is 'java.io.InputStream' &&
${body} !is 'org.apache.camel.StreamCache'"
steps:
- - convertBodyTo:
- type: byte[]
+ - setProperty:
+ name: documentStream
+ expression:
+ simple:
+ expression: "${body}"
+ - setBody:
+ expression:
+ simple:
+ expression:
"${body.readNBytes(${min({{maxDocumentSize}},2147483646)}++)}"
+ - script:
+ expression:
+ simple:
+ expression:
"${exchangeProperty.documentStream.close()}"
+ - removeProperty:
+ name: documentStream
+ # a file is sized by the CamelFileLength header its consumer
set, without a read;
+ # the header is believed only next to that file, elsewhere
it may be stale or
+ # caller-supplied. A plain java.io.File, and a cache spooled
to disk, the only one
+ # past an int, are sized by their long length(). Other
bodies are measured; a
+ # capped stream's size is unknown
- choice:
when:
+ - expression:
+ simple:
+ expression: "${body} is
'org.apache.camel.WrappedFile' && ${header.CamelFileLength} != null"
+ steps:
+ - choice:
+ when:
+ - expression:
+ simple:
+ expression: "${header.CamelFileLength}
> {{maxDocumentSize}}"
+ steps:
+ - throwException:
+ exceptionType:
java.lang.IllegalArgumentException
+ message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(${header.CamelFileLength} > {{maxDocumentSize}} bytes)"
+ - expression:
+ simple:
+ expression: "${body} is 'java.io.File' ||
${body} is 'org.apache.camel.converter.stream.FileInputStreamCache'"
+ steps:
+ - choice:
+ when:
+ - expression:
+ simple:
+ expression: "${body.length()} >
{{maxDocumentSize}}"
+ steps:
+ - throwException:
+ exceptionType:
java.lang.IllegalArgumentException
+ message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(${body.length()} > {{maxDocumentSize}} bytes)"
- expression:
simple:
expression: "${length()} > {{maxDocumentSize}}"
steps:
- throwException:
exceptionType:
java.lang.IllegalArgumentException
- message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(${length()} > {{maxDocumentSize}} bytes)"
+ message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(more than {{maxDocumentSize}} bytes)"
- convertBodyTo:
type: byte[]
- removeHeaders:
diff --git a/kamelets/tika-extract-text-action.kamelet.yaml
b/kamelets/tika-extract-text-action.kamelet.yaml
index 4689b0d05..3ca8c2b22 100644
--- a/kamelets/tika-extract-text-action.kamelet.yaml
+++ b/kamelets/tika-extract-text-action.kamelet.yaml
@@ -61,6 +61,7 @@ spec:
dependencies:
- "camel:tika"
- "camel:kamelet"
+ - "camel:bean"
template:
from:
uri: kamelet:source
@@ -76,7 +77,7 @@ spec:
# document, so an oversized one is rejected first. The precondition is
evaluated once,
# when the Kamelet is instantiated - only then does
${properties:maxDocumentSize:0} see
# the parameter, at runtime it resolves to 0 - so the check must stay
a precondition,
- # and the comparison below uses {{maxDocumentSize}}
+ # and the comparisons below use {{maxDocumentSize}}
- choice:
precondition: true
when:
@@ -84,26 +85,72 @@ spec:
simple:
expression: "${properties:maxDocumentSize:0} > 0"
steps:
- # length() reads a stream to its end and closes it, so a
stream body that
- # stream caching (on by default) left uncached is read into
bytes first; files,
- # byte arrays and cached streams are measured in place,
without a copy
+ # length() would read a stream to its end, so one that
stream caching (on by
+ # default) left uncached is read at most one byte past the
cap, within the int
+ # readNBytes takes, and closed: that byte rejects it below,
else the bytes are
+ # the document. The stream is held in a property: a variable
is stream-cached,
+ # which fails on an exchange that turned caching off for
itself
- choice:
when:
- expression:
simple:
expression: "${body} is 'java.io.InputStream' &&
${body} !is 'org.apache.camel.StreamCache'"
steps:
- - convertBodyTo:
- type: byte[]
+ - setProperty:
+ name: documentStream
+ expression:
+ simple:
+ expression: "${body}"
+ - setBody:
+ expression:
+ simple:
+ expression:
"${body.readNBytes(${min({{maxDocumentSize}},2147483646)}++)}"
+ - script:
+ expression:
+ simple:
+ expression:
"${exchangeProperty.documentStream.close()}"
+ - removeProperty:
+ name: documentStream
+ # a file is sized by the CamelFileLength header its consumer
set, without a read;
+ # the header is believed only next to that file, elsewhere
it may be stale or
+ # caller-supplied. A plain java.io.File, and a cache spooled
to disk, the only one
+ # past an int, are sized by their long length(). Other
bodies are measured; a
+ # capped stream's size is unknown
- choice:
when:
+ - expression:
+ simple:
+ expression: "${body} is
'org.apache.camel.WrappedFile' && ${header.CamelFileLength} != null"
+ steps:
+ - choice:
+ when:
+ - expression:
+ simple:
+ expression: "${header.CamelFileLength}
> {{maxDocumentSize}}"
+ steps:
+ - throwException:
+ exceptionType:
java.lang.IllegalArgumentException
+ message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(${header.CamelFileLength} > {{maxDocumentSize}} bytes)"
+ - expression:
+ simple:
+ expression: "${body} is 'java.io.File' ||
${body} is 'org.apache.camel.converter.stream.FileInputStreamCache'"
+ steps:
+ - choice:
+ when:
+ - expression:
+ simple:
+ expression: "${body.length()} >
{{maxDocumentSize}}"
+ steps:
+ - throwException:
+ exceptionType:
java.lang.IllegalArgumentException
+ message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(${body.length()} > {{maxDocumentSize}} bytes)"
- expression:
simple:
expression: "${length()} > {{maxDocumentSize}}"
steps:
- throwException:
exceptionType:
java.lang.IllegalArgumentException
- message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(${length()} > {{maxDocumentSize}} bytes)"
+ message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(more than {{maxDocumentSize}} bytes)"
# tika answers plain text (body only, blocks newline-separated) at the
pinned encoding
- to:
uri: tika
diff --git
a/library/camel-kamelets/src/main/resources/kamelets/docling-convert-action.kamelet.yaml
b/library/camel-kamelets/src/main/resources/kamelets/docling-convert-action.kamelet.yaml
index ac2b15292..e1c82e962 100644
---
a/library/camel-kamelets/src/main/resources/kamelets/docling-convert-action.kamelet.yaml
+++
b/library/camel-kamelets/src/main/resources/kamelets/docling-convert-action.kamelet.yaml
@@ -72,6 +72,7 @@ spec:
dependencies:
- "camel:docling"
- "camel:kamelet"
+ - "camel:bean"
template:
from:
uri: kamelet:source
@@ -86,7 +87,7 @@ spec:
# the document, so an oversized one is rejected first. The
precondition is evaluated
# once, when the Kamelet is instantiated - only then does
${properties:maxDocumentSize:0}
# see the parameter, at runtime it resolves to 0 - so the check must
stay a
- # precondition, and the comparison below uses {{maxDocumentSize}}
+ # precondition, and the comparisons below use {{maxDocumentSize}}
- choice:
precondition: true
when:
@@ -94,26 +95,72 @@ spec:
simple:
expression: "${properties:maxDocumentSize:0} > 0"
steps:
- # length() reads a stream to its end and closes it, so a
stream body that
- # stream caching (on by default) left uncached is read into
bytes first; files,
- # byte arrays and cached streams are measured in place,
without a copy
+ # length() would read a stream to its end, so one that
stream caching (on by
+ # default) left uncached is read at most one byte past the
cap, within the int
+ # readNBytes takes, and closed: that byte rejects it below,
else the bytes are
+ # the document. The stream is held in a property: a variable
is stream-cached,
+ # which fails on an exchange that turned caching off for
itself
- choice:
when:
- expression:
simple:
expression: "${body} is 'java.io.InputStream' &&
${body} !is 'org.apache.camel.StreamCache'"
steps:
- - convertBodyTo:
- type: byte[]
+ - setProperty:
+ name: documentStream
+ expression:
+ simple:
+ expression: "${body}"
+ - setBody:
+ expression:
+ simple:
+ expression:
"${body.readNBytes(${min({{maxDocumentSize}},2147483646)}++)}"
+ - script:
+ expression:
+ simple:
+ expression:
"${exchangeProperty.documentStream.close()}"
+ - removeProperty:
+ name: documentStream
+ # a file is sized by the CamelFileLength header its consumer
set, without a read;
+ # the header is believed only next to that file, elsewhere
it may be stale or
+ # caller-supplied. A plain java.io.File, and a cache spooled
to disk, the only one
+ # past an int, are sized by their long length(). Other
bodies are measured; a
+ # capped stream's size is unknown
- choice:
when:
+ - expression:
+ simple:
+ expression: "${body} is
'org.apache.camel.WrappedFile' && ${header.CamelFileLength} != null"
+ steps:
+ - choice:
+ when:
+ - expression:
+ simple:
+ expression: "${header.CamelFileLength}
> {{maxDocumentSize}}"
+ steps:
+ - throwException:
+ exceptionType:
java.lang.IllegalArgumentException
+ message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(${header.CamelFileLength} > {{maxDocumentSize}} bytes)"
+ - expression:
+ simple:
+ expression: "${body} is 'java.io.File' ||
${body} is 'org.apache.camel.converter.stream.FileInputStreamCache'"
+ steps:
+ - choice:
+ when:
+ - expression:
+ simple:
+ expression: "${body.length()} >
{{maxDocumentSize}}"
+ steps:
+ - throwException:
+ exceptionType:
java.lang.IllegalArgumentException
+ message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(${body.length()} > {{maxDocumentSize}} bytes)"
- expression:
simple:
expression: "${length()} > {{maxDocumentSize}}"
steps:
- throwException:
exceptionType:
java.lang.IllegalArgumentException
- message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(${length()} > {{maxDocumentSize}} bytes)"
+ message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(more than {{maxDocumentSize}} bytes)"
- convertBodyTo:
type: byte[]
- removeHeaders:
diff --git
a/library/camel-kamelets/src/main/resources/kamelets/tika-extract-text-action.kamelet.yaml
b/library/camel-kamelets/src/main/resources/kamelets/tika-extract-text-action.kamelet.yaml
index 4689b0d05..3ca8c2b22 100644
---
a/library/camel-kamelets/src/main/resources/kamelets/tika-extract-text-action.kamelet.yaml
+++
b/library/camel-kamelets/src/main/resources/kamelets/tika-extract-text-action.kamelet.yaml
@@ -61,6 +61,7 @@ spec:
dependencies:
- "camel:tika"
- "camel:kamelet"
+ - "camel:bean"
template:
from:
uri: kamelet:source
@@ -76,7 +77,7 @@ spec:
# document, so an oversized one is rejected first. The precondition is
evaluated once,
# when the Kamelet is instantiated - only then does
${properties:maxDocumentSize:0} see
# the parameter, at runtime it resolves to 0 - so the check must stay
a precondition,
- # and the comparison below uses {{maxDocumentSize}}
+ # and the comparisons below use {{maxDocumentSize}}
- choice:
precondition: true
when:
@@ -84,26 +85,72 @@ spec:
simple:
expression: "${properties:maxDocumentSize:0} > 0"
steps:
- # length() reads a stream to its end and closes it, so a
stream body that
- # stream caching (on by default) left uncached is read into
bytes first; files,
- # byte arrays and cached streams are measured in place,
without a copy
+ # length() would read a stream to its end, so one that
stream caching (on by
+ # default) left uncached is read at most one byte past the
cap, within the int
+ # readNBytes takes, and closed: that byte rejects it below,
else the bytes are
+ # the document. The stream is held in a property: a variable
is stream-cached,
+ # which fails on an exchange that turned caching off for
itself
- choice:
when:
- expression:
simple:
expression: "${body} is 'java.io.InputStream' &&
${body} !is 'org.apache.camel.StreamCache'"
steps:
- - convertBodyTo:
- type: byte[]
+ - setProperty:
+ name: documentStream
+ expression:
+ simple:
+ expression: "${body}"
+ - setBody:
+ expression:
+ simple:
+ expression:
"${body.readNBytes(${min({{maxDocumentSize}},2147483646)}++)}"
+ - script:
+ expression:
+ simple:
+ expression:
"${exchangeProperty.documentStream.close()}"
+ - removeProperty:
+ name: documentStream
+ # a file is sized by the CamelFileLength header its consumer
set, without a read;
+ # the header is believed only next to that file, elsewhere
it may be stale or
+ # caller-supplied. A plain java.io.File, and a cache spooled
to disk, the only one
+ # past an int, are sized by their long length(). Other
bodies are measured; a
+ # capped stream's size is unknown
- choice:
when:
+ - expression:
+ simple:
+ expression: "${body} is
'org.apache.camel.WrappedFile' && ${header.CamelFileLength} != null"
+ steps:
+ - choice:
+ when:
+ - expression:
+ simple:
+ expression: "${header.CamelFileLength}
> {{maxDocumentSize}}"
+ steps:
+ - throwException:
+ exceptionType:
java.lang.IllegalArgumentException
+ message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(${header.CamelFileLength} > {{maxDocumentSize}} bytes)"
+ - expression:
+ simple:
+ expression: "${body} is 'java.io.File' ||
${body} is 'org.apache.camel.converter.stream.FileInputStreamCache'"
+ steps:
+ - choice:
+ when:
+ - expression:
+ simple:
+ expression: "${body.length()} >
{{maxDocumentSize}}"
+ steps:
+ - throwException:
+ exceptionType:
java.lang.IllegalArgumentException
+ message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(${body.length()} > {{maxDocumentSize}} bytes)"
- expression:
simple:
expression: "${length()} > {{maxDocumentSize}}"
steps:
- throwException:
exceptionType:
java.lang.IllegalArgumentException
- message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(${length()} > {{maxDocumentSize}} bytes)"
+ message: "Document
'${exchangeProperty.CamelLangChain4jIngestDocumentId}' exceeds maxDocumentSize
(more than {{maxDocumentSize}} bytes)"
# tika answers plain text (body only, blocks newline-separated) at the
pinned encoding
- to:
uri: tika
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-route.citrus.it.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-route.citrus.it.yaml
index 722ae4800..7cc18e4f2 100644
---
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-route.citrus.it.yaml
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-route.citrus.it.yaml
@@ -74,7 +74,7 @@ actions:
path: "/result/parser-size"
type: "plaintext"
body:
- data: "@matches('rejected: Document .long.pdf. exceeds
maxDocumentSize .100 > 40 bytes.')@"
+ data: "@matches('rejected: Document .long.pdf. exceeds
maxDocumentSize .more than 40 bytes.')@"
- http:
server: "httpServer"
sendResponse:
@@ -83,7 +83,177 @@ actions:
reasonPhrase: "OK"
version: "HTTP/1.1"
- # a file over the cap: rejected before the parse
+ # a CamelFileLength header without the file it describes: not believed, the
payload is measured
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size"
+ type: "plaintext"
+ body:
+ data: "@matches('rejected: Document .forged.txt. exceeds
maxDocumentSize .more than 40 bytes.')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # a polled file under the cap: sized by its header and parsed
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size"
+ type: "plaintext"
+ body:
+ data: "@contains('parsed: Short text.')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # a polled file whose header puts it past 4 GiB: rejected by that size, the
file not read
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size"
+ type: "plaintext"
+ body:
+ data: "@matches('rejected: Document .huge.pdf. exceeds
maxDocumentSize .4294967306 > 40 bytes.')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # a spooled cache under the cap: sized by its long length() and parsed
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size"
+ type: "plaintext"
+ body:
+ data: "@contains('parsed: Short text.')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # a spooled cache past 4 GiB: rejected by its long length(), not wrapped to
an int
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size"
+ type: "plaintext"
+ body:
+ data: "@matches('rejected: Document .huge.cache. exceeds
maxDocumentSize .4294967306 > 40 bytes.')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # a plain java.io.File under the cap: sized by its long length() and parsed
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size"
+ type: "plaintext"
+ body:
+ data: "@contains('parsed: Short text.')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # a plain java.io.File past 4 GiB: rejected by its long length(), not read
and counted in an
+ # int - by Tika, then by Docling
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size"
+ type: "plaintext"
+ body:
+ data: "@matches('rejected: Document .huge.file. exceeds
maxDocumentSize .4294967306 > 40 bytes.')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size"
+ type: "plaintext"
+ body:
+ data: "@matches('rejected: Document .huge-file.pdf. exceeds
maxDocumentSize .4294967306 > 40 bytes.')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # an uncached stream with caching on but off for the exchange: under the
cap, parsed - not
+ # failed by stream-caching it on the side
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size"
+ type: "plaintext"
+ body:
+ data: "@contains('parsed: Short text.')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # the same over the cap, through Docling: rejected by the cap
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size"
+ type: "plaintext"
+ body:
+ data: "@matches('rejected: Document .uncached.pdf. exceeds
maxDocumentSize .more than 40 bytes.')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # a file over the cap from the file source: rejected by the header the
consumer set
- http:
server: "httpServer"
receiveRequest:
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-route.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-route.yaml
index da97a95b7..b4cc9f9de 100644
---
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-route.yaml
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-route.yaml
@@ -16,10 +16,16 @@
# ---------------------------------------------------------------------------
# maxDocumentSize on the parser actions: the cap is checked before the parse
or conversion.
-# One route feeds three documents in turn, each report waiting for the test's
answer, so
-# the order is fixed: a short text through Tika under the cap, an oversized
payload through
+# One route feeds the documents in turn, each report waiting for the test's
answer, so the
+# order is fixed: a short text through Tika under the cap, an oversized
payload through
# Docling (pointed at the test's HTTP server, so a payload leaking past the
cap would show up
-# there as a conversion request), and an oversized file through the file
source and Tika.
+# there as a conversion request), an oversized payload whose CamelFileLength
header claims a
+# small file, two polled files - one under the cap, one whose CamelFileLength
header puts it
+# past 4 GiB - two caches spooled to disk, one under the cap, one past 4 GiB,
three plain
+# java.io.File bodies, one under the cap, two past 4 GiB through Tika and
Docling, two
+# uncached streams on an exchange that turned stream caching off for itself,
one under the cap
+# through Tika, one over it through Docling, and an oversized file through the
file source and
+# Tika.
- route:
id: parser-max-size-route
from:
@@ -62,6 +68,236 @@
- setBody:
simple: "rejected: ${exception.message}"
- to: "{{report.url}}"
+ # 100 bytes again, with a CamelFileLength header claiming 10: without
the file it
+ # describes, the header is not believed
+ - setBody:
+ constant:
"0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"
+ - setHeader:
+ name: "CamelFileLength"
+ constant: "10"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "forged.txt"
+ - doTry:
+ steps:
+ - to: "kamelet:tika-extract-text-action?maxDocumentSize=40"
+ - setBody:
+ simple: "parsed: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}"
+ - removeHeader:
+ name: "CamelFileLength"
+ - to: "{{report.url}}"
+ # a file under the cap, polled into the body as a file consumer
delivers it: sized by
+ # its CamelFileLength header and parsed. Polled from a directory the
file source does
+ # not watch
+ - setBody:
+ constant: "Short text."
+ - to: "file:{{ingest.dir}}-poll?fileName=small.txt"
+ - pollEnrich:
+ expression:
+ constant: "file:{{ingest.dir}}-poll?fileName=small.txt&noop=true"
+ timeout: "10000"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "small.txt"
+ - doTry:
+ steps:
+ - to: "kamelet:tika-extract-text-action?maxDocumentSize=40"
+ - setBody:
+ simple: "parsed: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}"
+ - to: "{{report.url}}"
+ # a file whose CamelFileLength header puts it past 4 GiB, beyond an
int: believed next
+ # to the file, so the file is rejected without being read - through
Docling, where a
+ # leak past the cap would show up as a conversion request
+ - setBody:
+ constant: "Short text."
+ - to: "file:{{ingest.dir}}-poll?fileName=huge.pdf"
+ - pollEnrich:
+ expression:
+ constant: "file:{{ingest.dir}}-poll?fileName=huge.pdf&noop=true"
+ timeout: "10000"
+ - setHeader:
+ name: "CamelFileLength"
+ expression:
+ constant:
+ expression: "4294967306"
+ resultType: "java.lang.Long"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "huge.pdf"
+ - doTry:
+ steps:
+ - to:
"kamelet:docling-convert-action?maxDocumentSize=40&useDoclingServe=true&doclingServeUrl={{docling.serve.url}}"
+ - setBody:
+ simple: "converted: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}"
+ - to: "{{report.url}}"
+ - removeHeader:
+ name: "CamelFileLength"
+ # a cache spooled to disk under the cap: sized by its long length(),
and parsed
+ - setBody:
+ constant: "Short text."
+ - to: "file:{{ingest.dir}}-poll?fileName=small.cache"
+ - setBody:
+ groovy: "new
org.apache.camel.converter.stream.FileInputStreamCache(new
File(headers.CamelFileNameProduced))"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "small.cache"
+ - doTry:
+ steps:
+ - to: "kamelet:tika-extract-text-action?maxDocumentSize=40"
+ - setBody:
+ simple: "parsed: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}"
+ - to: "{{report.url}}"
+ # a cache spooled to disk whose file reports past 4 GiB, which
length() would cut to an
+ # int: rejected by its long length() - through Docling, where a leak
past the cap would
+ # show up as a conversion request
+ - setBody:
+ constant: "Short text."
+ - to: "file:{{ingest.dir}}-poll?fileName=huge.cache"
+ - setBody:
+ groovy: "new
org.apache.camel.converter.stream.FileInputStreamCache(new
File(headers.CamelFileNameProduced) { long length() { 4294967306L } })"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "huge.cache"
+ - doTry:
+ steps:
+ - to:
"kamelet:docling-convert-action?maxDocumentSize=40&useDoclingServe=true&doclingServeUrl={{docling.serve.url}}"
+ - setBody:
+ simple: "converted: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}"
+ - to: "{{report.url}}"
+ # a plain java.io.File under the cap, as file-watch delivers it: sized
by its long
+ # length(), and parsed
+ - setBody:
+ constant: "Short text."
+ - to: "file:{{ingest.dir}}-poll?fileName=small.file"
+ - setBody:
+ groovy: "new File(headers.CamelFileNameProduced)"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "small.file"
+ - doTry:
+ steps:
+ - to: "kamelet:tika-extract-text-action?maxDocumentSize=40"
+ - setBody:
+ simple: "parsed: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}"
+ - to: "{{report.url}}"
+ # a plain java.io.File reporting past 4 GiB, which length() would read
through and count
+ # in an int: rejected by its long length(), by Tika and by Docling
+ - setBody:
+ constant: "Short text."
+ - to: "file:{{ingest.dir}}-poll?fileName=huge.file"
+ - setBody:
+ groovy: "new File(headers.CamelFileNameProduced) { long length() {
4294967306L } }"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "huge.file"
+ - doTry:
+ steps:
+ - to: "kamelet:tika-extract-text-action?maxDocumentSize=40"
+ - setBody:
+ simple: "parsed: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}"
+ - to: "{{report.url}}"
+ - setBody:
+ constant: "Short text."
+ - to: "file:{{ingest.dir}}-poll?fileName=huge-file.pdf"
+ - setBody:
+ groovy: "new File(headers.CamelFileNameProduced) { long length() {
4294967306L } }"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "huge-file.pdf"
+ - doTry:
+ steps:
+ - to:
"kamelet:docling-convert-action?maxDocumentSize=40&useDoclingServe=true&doclingServeUrl={{docling.serve.url}}"
+ - setBody:
+ simple: "converted: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}"
+ - to: "{{report.url}}"
+ # an uncached stream on an exchange that turned stream caching off for
itself, as
+ # netty-http and the platform-http-vertx proxy do, while caching stays
on: under the cap
+ # parsed whole, over it rejected. Wrapped, as a bare
ByteArrayInputStream caches in place
+ - script:
+ groovy: "exchange.exchangeExtension.setStreamCacheDisabled(true)"
+ - setBody:
+ groovy: "new java.io.PushbackInputStream(new
java.io.ByteArrayInputStream('Short text.'.bytes))"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "uncached.txt"
+ - doTry:
+ steps:
+ - to: "kamelet:tika-extract-text-action?maxDocumentSize=40"
+ - setBody:
+ simple: "parsed: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}"
+ - to: "{{report.url}}"
+ # 100 bytes, over the cap of 40, through Docling
+ - setBody:
+ groovy: "new java.io.PushbackInputStream(new
java.io.ByteArrayInputStream(('0123456789' * 10).bytes))"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "uncached.pdf"
+ - doTry:
+ steps:
+ - to:
"kamelet:docling-convert-action?maxDocumentSize=40&useDoclingServe=true&doclingServeUrl={{docling.serve.url}}"
+ - setBody:
+ simple: "converted: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}"
+ - to: "{{report.url}}"
- setBody:
constant:
"0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"
- to: "file:{{ingest.dir}}?fileName=big.txt"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-stream-route.citrus.it.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-stream-route.citrus.it.yaml
index 91fb30293..d13d537cf 100644
---
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-stream-route.citrus.it.yaml
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-stream-route.citrus.it.yaml
@@ -24,6 +24,8 @@ actions:
timeout: 100
- createVariables:
variables:
+ - name: "docling.serve.url"
+ value: "http://localhost:${http.server.port}";
- name: "report.url"
value:
"http://localhost:${http.server.port}/result/parser-size-stream";
@@ -38,6 +40,8 @@ actions:
file: "langchain4j-ingest/parser-max-size-stream-route.yaml"
systemProperties:
properties:
+ - name: "docling.serve.url"
+ value: "${docling.serve.url}"
- name: "report.url"
value: "${report.url}"
- name: "camel.main.streamCachingEnabled"
@@ -60,7 +64,7 @@ actions:
reasonPhrase: "OK"
version: "HTTP/1.1"
- # over the cap: rejected, the size measured once and reported right
+ # over the cap: rejected after one byte past the cap, the other 59 left
unread
- http:
server: "httpServer"
receiveRequest:
@@ -68,7 +72,76 @@ actions:
path: "/result/parser-size-stream"
type: "plaintext"
body:
- data: "@matches('rejected: Document .long.txt. exceeds
maxDocumentSize .100 > 40 bytes.')@"
+ data: "@matches('rejected: Document .long.txt. exceeds
maxDocumentSize .more than 40 bytes., left 59')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # under a cap past the int range: the whole stream reaches the parser, not a
cut of it
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size-stream"
+ type: "plaintext"
+ body:
+ data: "@contains('parsed: Short text.')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # over the cap through Docling: rejected before the conversion, 59 bytes
left unread; a
+ # conversion request instead would fail here
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size-stream"
+ type: "plaintext"
+ body:
+ data: "@matches('rejected: Document .long.pdf. exceeds
maxDocumentSize .more than 40 bytes., left 59')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # rejected by Tika, which closed the stream it read: available() on it then
fails
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size-stream"
+ type: "plaintext"
+ body:
+ data: "@matches('rejected: Document .closed.txt. exceeds
maxDocumentSize .more than 40 bytes., .*IOException: Stream closed')@"
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # rejected by Docling, which closed the stream it read: available() on it
then fails
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result/parser-size-stream"
+ type: "plaintext"
+ body:
+ data: "@matches('rejected: Document .closed.pdf. exceeds
maxDocumentSize .more than 40 bytes., .*IOException: Stream closed')@"
- http:
server: "httpServer"
sendResponse:
diff --git
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-stream-route.yaml
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-stream-route.yaml
index 38a464f50..814015447 100644
---
a/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-stream-route.yaml
+++
b/tests/camel-kamelets-itest/src/test/resources/langchain4j-ingest/parser-max-size-stream-route.yaml
@@ -16,8 +16,10 @@
# ---------------------------------------------------------------------------
# maxDocumentSize with stream caching switched off, so a stream body reaches
the action raw
-# instead of cached. A raw stream under the cap must still be parsed whole -
the measurement
-# must not drain it - and one over the cap is rejected with its exact size.
+# instead of cached. The action reads a raw stream only one byte past the cap
and closes it:
+# one under the cap is still parsed whole, also under a cap past the int
range, and one over
+# the cap is rejected with the rest left unread, by Tika and by Docling
(pointed at the test's
+# HTTP server, so a payload leaking past the cap would show up there as a
conversion request).
- route:
id: parser-max-size-stream-route
from:
@@ -52,6 +54,74 @@
- setHeader:
name: "CamelLangChain4jIngestDocumentId"
constant: "long.txt"
+ - setProperty:
+ name: "rawStream"
+ simple: "${body}"
+ - doTry:
+ steps:
+ - to: "kamelet:tika-extract-text-action?maxDocumentSize=40"
+ - setBody:
+ simple: "parsed: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}, left
${exchangeProperty.rawStream.available()}"
+ - to: "{{report.url}}"
+ # a cap past the int range readNBytes takes: the stream is still read
whole
+ - setBody:
+ constant: "Short text."
+ - setBody:
+ simple: "${bodyAs(java.io.InputStream)}"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "short.txt"
+ - doTry:
+ steps:
+ - to:
"kamelet:tika-extract-text-action?maxDocumentSize=4294967296"
+ - setBody:
+ simple: "parsed: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}"
+ - to: "{{report.url}}"
+ # 100 bytes, over the cap of 40, through Docling
+ - setBody:
+ constant:
"0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"
+ - setBody:
+ simple: "${bodyAs(java.io.InputStream)}"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "long.pdf"
+ - setProperty:
+ name: "rawStream"
+ simple: "${body}"
+ - doTry:
+ steps:
+ - to:
"kamelet:docling-convert-action?maxDocumentSize=40&useDoclingServe=true&doclingServeUrl={{docling.serve.url}}"
+ - setBody:
+ simple: "converted: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}, left
${exchangeProperty.rawStream.available()}"
+ - to: "{{report.url}}"
+ # 100 bytes over the cap of 40, in a stream whose available() fails
once closed: the
+ # action closes the stream it read
+ - setBody:
+ groovy: "new java.io.PushbackInputStream(new
java.io.ByteArrayInputStream(('0123456789' * 10).bytes))"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "closed.txt"
+ - setProperty:
+ name: "rawStream"
+ simple: "${body}"
- doTry:
steps:
- to: "kamelet:tika-extract-text-action?maxDocumentSize=40"
@@ -63,4 +133,45 @@
steps:
- setBody:
simple: "rejected: ${exception.message}"
+ - doTry:
+ steps:
+ - setBody:
+ simple: "${body}, open with
${exchangeProperty.rawStream.available()} left"
+ doCatch:
+ - exception:
+ - "java.lang.Exception"
+ steps:
+ - setBody:
+ simple: "${body}, ${exception.message}"
+ - to: "{{report.url}}"
+ # the same through Docling
+ - setBody:
+ groovy: "new java.io.PushbackInputStream(new
java.io.ByteArrayInputStream(('0123456789' * 10).bytes))"
+ - setHeader:
+ name: "CamelLangChain4jIngestDocumentId"
+ constant: "closed.pdf"
+ - setProperty:
+ name: "rawStream"
+ simple: "${body}"
+ - doTry:
+ steps:
+ - to:
"kamelet:docling-convert-action?maxDocumentSize=40&useDoclingServe=true&doclingServeUrl={{docling.serve.url}}"
+ - setBody:
+ simple: "converted: ${body}"
+ doCatch:
+ - exception:
+ - "java.lang.IllegalArgumentException"
+ steps:
+ - setBody:
+ simple: "rejected: ${exception.message}"
+ - doTry:
+ steps:
+ - setBody:
+ simple: "${body}, open with
${exchangeProperty.rawStream.available()} left"
+ doCatch:
+ - exception:
+ - "java.lang.Exception"
+ steps:
+ - setBody:
+ simple: "${body}, ${exception.message}"
- to: "{{report.url}}"
